<commit_before>package wav

import (
	"encoding/binary"
	"fmt"
	"io"
	"os"

	"github.com/mattetti/audio"
	"github.com/mattetti/audio/riff"
)

// Encoder encodes LPCM data into a wav containter.
type Encoder struct {
	w          io.WriteSeeker
	SampleRate int
	BitDepth   int
	NumChans   int

	// A number indicating the WAVE format category of the file. The content of the
	// <format-specific-fields> portion of the ‘fmt’ chunk, and the interpretation of
	// the waveform data, depend on this value.
	// PCM = 1 (i.e. Linear quantization) Values other than 1 indicate some form of compression.
	WavAudioFormat int

	WrittenBytes    int
	frames          int
	pcmChunkStarted bool
	pcmChunkSizePos int
}

// NewEncoder creates a new encoder to create a new wav file.
// Don't forget to add Frames to the encoder before writing.
func NewEncoder(w io.WriteSeeker, sampleRate, bitDepth, numChans, audioFormat int) *Encoder {
	return &Encoder{
		w:              w,
		SampleRate:     sampleRate,
		BitDepth:       bitDepth,
		NumChans:       numChans,
		WavAudioFormat: audioFormat,
	}
}

// AddLE serializes and adds the passed value using little endian
func (e *Encoder) AddLE(src interface{}) error {
	e.WrittenBytes += binary.Size(src)
	return binary.Write(e.w, binary.LittleEndian, src)
}

// AddBE serializes and adds the passed value using big endian
func (e *Encoder) AddBE(src interface{}) error {
	e.WrittenBytes += binary.Size(src)
	return binary.Write(e.w, binary.BigEndian, src)
}

func (e *Encoder) addFrames(frames []int) error {
	if frames == nil {
		return fmt.Errorf("can't add a nil frames")
	}
	frameSize := e.NumChans

	for i := 0; i+frameSize <= len(frames); {
		for j := 0; j < frameSize; j++ {
			switch e.BitDepth {
			case 8:
				if err := e.AddLE(uint8(frames[i])); err != nil {
					return err
				}
			case 16:
				if err := e.AddLE(int16(frames[i])); err != nil {
					return err
				}
			case 24:
				if err := e.AddLE(audio.Int32toInt24LEBytes(int32(frames[i]))); err != nil {
					return err
				}
			case 32:
				if err := e.AddLE(int32(frames[i])); err != nil {
					return err
				}
			default:
				return fmt.Errorf("can't add frames of bit size %d", e.BitDepth)
			}
			i++
		}
		e.frames++
	}

	return nil
}

func (e *Encoder) writeHeader() error {
	if e == nil {
		return fmt.Errorf("can't write a nil encoder")
	}
	if e.w == nil {
		return fmt.Errorf("can't write to a nil writer")
	}

	if e.WrittenBytes > 0 {
		return nil
	}

	// riff ID
	if err := e.AddLE(riff.RiffID); err != nil {
		return err
	}
	// file size uint32, to update later on.
	if err := e.AddLE(uint32(42)); err != nil {
		return err
	}
	// wave headers
	if err := e.AddLE(riff.WavFormatID); err != nil {
		return err
	}
	// form
	if err := e.AddLE(riff.FmtID); err != nil {
		return err
	}
	// chunk size
	if err := e.AddLE(uint32(16)); err != nil {
		return err
	}
	// wave format
	if err := e.AddLE(uint16(e.WavAudioFormat)); err != nil {
		return err
	}
	// num channels
	if err := e.AddLE(uint16(e.NumChans)); err != nil {
		return fmt.Errorf("error encoding the number of channels - %v", err)
	}
	// samplerate
	if err := e.AddLE(uint32(e.SampleRate)); err != nil {
		return fmt.Errorf("error encoding the sample rate - %v", err)
	}
	// avg bytes per sec
	if err := e.AddLE(uint32(e.SampleRate * e.NumChans * e.BitDepth / 8)); err != nil {
		return fmt.Errorf("error encoding the avg bytes per sec - %v", err)
	}
	// block align
	if err := e.AddLE(uint16(2)); err != nil {
		return err
	}
	// bits per sample
	if err := e.AddLE(uint16(e.BitDepth)); err != nil {
		return fmt.Errorf("error encoding bits per sample - %v", err)
	}

	return nil
}

func (e *Encoder) Write(frames audio.FramesInt) error {
	if err := e.writeHeader(); err != nil {
		return err
	}

	if !e.pcmChunkStarted {
		// sound header
		if err := e.AddLE(riff.DataFormatID); err != nil {
			return fmt.Errorf("error encoding sound header %v", err)
		}

		// write a temporary chunksize
		e.pcmChunkSizePos = e.WrittenBytes
		if err := e.AddLE(uint32(42)); err != nil {
			return fmt.Errorf("%v when writing wav data chunk size header", err)
		}
	}

	return e.addFrames(frames)
}

// Close flushes the content to disk, make sure the headers are up to date
// Note that the underlying writter is NOT being closed.
func (e *Encoder) Close() error {
	if e == nil || e.w == nil {
		return nil
	}

	// go back and write total size in header
	e.w.Seek(4, 0)
	if err := e.AddLE(uint32(e.WrittenBytes) - 8); err != nil {
		return fmt.Errorf("%v when writing the total written bytes", err)
	}
	// rewrite the audio chunk length header
	if e.pcmChunkSizePos > 0 {
		e.w.Seek(int64(e.pcmChunkSizePos), 0)
		chunksize := uint32((int(e.BitDepth) / 8) * int(e.NumChans) * e.frames)
		if err := e.AddLE(uint32(chunksize)); err != nil {
			return fmt.Errorf("%v when writing wav data chunk size header", err)
		}
	}

	// jump back to the end of the file.
	e.w.Seek(0, 2)
	switch e.w.(type) {
	case *os.File:
		return e.w.(*os.File).Sync()
	}
	return nil
}
<commit_msg>wav: check errors when seeking<commit_after>package wav

import (
	"encoding/binary"
	"fmt"
	"io"
	"os"

	"github.com/mattetti/audio"
	"github.com/mattetti/audio/riff"
)

// Encoder encodes LPCM data into a wav containter.
type Encoder struct {
	w          io.WriteSeeker
	SampleRate int
	BitDepth   int
	NumChans   int

	// A number indicating the WAVE format category of the file. The content of the
	// <format-specific-fields> portion of the ‘fmt’ chunk, and the interpretation of
	// the waveform data, depend on this value.
	// PCM = 1 (i.e. Linear quantization) Values other than 1 indicate some form of compression.
	WavAudioFormat int

	WrittenBytes    int
	frames          int
	pcmChunkStarted bool
	pcmChunkSizePos int
}

// NewEncoder creates a new encoder to create a new wav file.
// Don't forget to add Frames to the encoder before writing.
func NewEncoder(w io.WriteSeeker, sampleRate, bitDepth, numChans, audioFormat int) *Encoder {
	return &Encoder{
		w:              w,
		SampleRate:     sampleRate,
		BitDepth:       bitDepth,
		NumChans:       numChans,
		WavAudioFormat: audioFormat,
	}
}

// AddLE serializes and adds the passed value using little endian
func (e *Encoder) AddLE(src interface{}) error {
	e.WrittenBytes += binary.Size(src)
	return binary.Write(e.w, binary.LittleEndian, src)
}

// AddBE serializes and adds the passed value using big endian
func (e *Encoder) AddBE(src interface{}) error {
	e.WrittenBytes += binary.Size(src)
	return binary.Write(e.w, binary.BigEndian, src)
}

func (e *Encoder) addFrames(frames []int) error {
	if frames == nil {
		return fmt.Errorf("can't add a nil frames")
	}
	frameSize := e.NumChans

	for i := 0; i+frameSize <= len(frames); {
		for j := 0; j < frameSize; j++ {
			switch e.BitDepth {
			case 8:
				if err := e.AddLE(uint8(frames[i])); err != nil {
					return err
				}
			case 16:
				if err := e.AddLE(int16(frames[i])); err != nil {
					return err
				}
			case 24:
				if err := e.AddLE(audio.Int32toInt24LEBytes(int32(frames[i]))); err != nil {
					return err
				}
			case 32:
				if err := e.AddLE(int32(frames[i])); err != nil {
					return err
				}
			default:
				return fmt.Errorf("can't add frames of bit size %d", e.BitDepth)
			}
			i++
		}
		e.frames++
	}

	return nil
}

func (e *Encoder) writeHeader() error {
	if e == nil {
		return fmt.Errorf("can't write a nil encoder")
	}
	if e.w == nil {
		return fmt.Errorf("can't write to a nil writer")
	}

	if e.WrittenBytes > 0 {
		return nil
	}

	// riff ID
	if err := e.AddLE(riff.RiffID); err != nil {
		return err
	}
	// file size uint32, to update later on.
	if err := e.AddLE(uint32(42)); err != nil {
		return err
	}
	// wave headers
	if err := e.AddLE(riff.WavFormatID); err != nil {
		return err
	}
	// form
	if err := e.AddLE(riff.FmtID); err != nil {
		return err
	}
	// chunk size
	if err := e.AddLE(uint32(16)); err != nil {
		return err
	}
	// wave format
	if err := e.AddLE(uint16(e.WavAudioFormat)); err != nil {
		return err
	}
	// num channels
	if err := e.AddLE(uint16(e.NumChans)); err != nil {
		return fmt.Errorf("error encoding the number of channels - %v", err)
	}
	// samplerate
	if err := e.AddLE(uint32(e.SampleRate)); err != nil {
		return fmt.Errorf("error encoding the sample rate - %v", err)
	}
	// avg bytes per sec
	if err := e.AddLE(uint32(e.SampleRate * e.NumChans * e.BitDepth / 8)); err != nil {
		return fmt.Errorf("error encoding the avg bytes per sec - %v", err)
	}
	// block align
	if err := e.AddLE(uint16(2)); err != nil {
		return err
	}
	// bits per sample
	if err := e.AddLE(uint16(e.BitDepth)); err != nil {
		return fmt.Errorf("error encoding bits per sample - %v", err)
	}

	return nil
}

func (e *Encoder) Write(frames audio.FramesInt) error {
	if err := e.writeHeader(); err != nil {
		return err
	}

	if !e.pcmChunkStarted {
		// sound header
		if err := e.AddLE(riff.DataFormatID); err != nil {
			return fmt.Errorf("error encoding sound header %v", err)
		}

		// write a temporary chunksize
		e.pcmChunkSizePos = e.WrittenBytes
		if err := e.AddLE(uint32(42)); err != nil {
			return fmt.Errorf("%v when writing wav data chunk size header", err)
		}
	}

	return e.addFrames(frames)
}

// Close flushes the content to disk, make sure the headers are up to date
// Note that the underlying writter is NOT being closed.
func (e *Encoder) Close() error {
	if e == nil || e.w == nil {
		return nil
	}

	// go back and write total size in header
	if _, err := e.w.Seek(4, 0); err != nil {
		return err
	}
	if err := e.AddLE(uint32(e.WrittenBytes) - 8); err != nil {
		return fmt.Errorf("%v when writing the total written bytes", err)
	}
	// rewrite the audio chunk length header
	if e.pcmChunkSizePos > 0 {
		if _, err := e.w.Seek(int64(e.pcmChunkSizePos), 0); err != nil {
			return err
		}
		chunksize := uint32((int(e.BitDepth) / 8) * int(e.NumChans) * e.frames)
		if err := e.AddLE(uint32(chunksize)); err != nil {
			return fmt.Errorf("%v when writing wav data chunk size header", err)
		}
	}

	// jump back to the end of the file.
	if _, err := e.w.Seek(0, 2); err != nil {
		return err
	}
	switch e.w.(type) {
	case *os.File:
		return e.w.(*os.File).Sync()
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/csv"
	"encoding/json"
	"github.com/gin-gonic/gin"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"os"
	"strconv"
	"time"
)

func CampbellTime(myTime time.Time) [3]int {
	hourmin := myTime.Hour()*100 + myTime.Minute()
	if hourmin == 0 {
		hourmin = 2400
	}
	return [3]int{myTime.Year(), myTime.YearDay(), hourmin}
}

type User struct {
	Name     string `json:"name"`
	Password string `json:"password"`
}

func checkErr(err error, msg string) {
	if err != nil {
		log.Fatalln(msg, err)
	}
}

func Index(c *gin.Context) {
	c.String(200, "we are here")
}

type Datum struct {
	Time  time.Time `json:"time"`
	Value float64   `json:"value"`
}

func variatesById(db *sqlx.DB, c *gin.Context) {
	id := c.Params.ByName("id")
	var query string
	db.Exec("set search_path=weather")
	db.Get(&query, "select query from weather.variates where id = $1", id)
	var data = []Datum{}
	db.Select(&data, query)
	c.JSON(200, data)
}

type Table struct {
	Id   int
	Name string
}

type Variate struct {
	Id    int
	Title string
}

func tables(db *sqlx.DB, c *gin.Context) {
	db.Exec("set search_path=weather")
	var tables []Table
	db.Select(&tables, "select id, name from weather.tables")
	c.JSON(200, tables)
}

func variates(db *sqlx.DB, c *gin.Context) {
	db.Exec("set search_path=weather")
	var variates []Variate
	db.Select(&variates, "select id, title from weather.variates")
	c.JSON(200, variates)
}

func tablesById(db *sqlx.DB, c *gin.Context) {
	id := c.Params.ByName("id")
	var query string
	db.Exec("set search_path=weather")
	db.Get(&query, "select query from weather.tables where id = $1", id)
	rows, _ := db.Queryx(query)
	var results []interface{}
	for rows.Next() {
		cols, _ := rows.SliceScan()
		results = append(results, cols)
	}
	rows.Close()
	c.JSON(200, results)
}

type DayObservation struct {
	Year_rtm                     sql.NullInt64
	Day_rtm                      sql.NullInt64
	Hour_rtm                     sql.NullInt64
	Air_temp_107_max             sql.NullFloat64
	Air_temp_107_hour_max        sql.NullInt64
	Air_temp_107_min             sql.NullFloat64
	Air_temp_107_hour_min        sql.NullInt64
	Rh_max                       sql.NullFloat64
	Rh_hour_max                  sql.NullInt64
	Rh_min                       sql.NullFloat64
	Rh_hour_min                  sql.NullInt64
	Solar_radiation              sql.NullFloat64
	Sol_rad_max                  sql.NullFloat64
	Sol_rad_hour_max             sql.NullInt64
	Soil_temp_5_cm_max           sql.NullFloat64
	Soil_temp_5_cm_hour_max      sql.NullInt64
	Soil_temp_5_cm_min           sql.NullFloat64
	Soil_temp_5_cm_hour_min      sql.NullInt64
	Soil_temp_10_cm_max          sql.NullFloat64
	Soil_temp_10_cm_hour_max     sql.NullInt64
	Soil_temp_10_cm_min          sql.NullFloat64
	Soil_temp_10_cm_hour_min     sql.NullInt64
	Soil_moisture_10_cm_max      sql.NullFloat64
	Soil_moisture_10_cm_hour_max sql.NullInt64
	Soil_moisture_10_cm_min      sql.NullFloat64
	Soil_moisture_10_cm_hour_min sql.NullInt64
	Soil_moisture_25_cm_max      sql.NullFloat64
	Soil_moisture_25_cm_hour_max sql.NullInt64
	Soil_moisture_25_cm_min      sql.NullFloat64
	Soil_moisture_25_cm_hour_min sql.NullInt64
	Wind_speed_max               sql.NullFloat64
	Wind_speed_hour_max          sql.NullInt64
	Rain_mm                      sql.NullFloat64
	Battery_voltage_min          sql.NullFloat64
	Date                         time.Time
	Precip                       sql.NullFloat64
}

func (d *DayObservation) toMawn() []string {
	value := []string{
		strconv.FormatInt(d.Year_rtm.Int64, 10),
		strconv.FormatInt(d.Day_rtm.Int64, 10),
		strconv.FormatInt(d.Hour_rtm.Int64, 10),
		strconv.FormatFloat(d.Air_temp_107_max.Float64, 'f', 5, 64),
		// Air_temp_107_hour_max        sql.NullInt64
		// Air_temp_107_min             sql.NullFloat64
		// Air_temp_107_hour_min        sql.NullInt64
		// Rh_max                       sql.NullFloat64
		// Rh_hour_max                  sql.NullInt64
		// Rh_min                       sql.NullFloat64
		// Rh_hour_min                  sql.NullInt64
		// Solar_radiation              sql.NullFloat64
		// Sol_rad_max                  sql.NullFloat64
		// Sol_rad_hour_max             sql.NullInt64
		// Soil_temp_5_cm_max           sql.NullFloat64
		// Soil_temp_5_cm_hour_max      sql.NullInt64
		// Soil_temp_5_cm_min           sql.NullFloat64
		// Soil_temp_5_cm_hour_min      sql.NullInt64
		// Soil_temp_10_cm_max          sql.NullFloat64
		// Soil_temp_10_cm_hour_max     sql.NullInt64
		// Soil_temp_10_cm_min          sql.NullFloat64
		// Soil_temp_10_cm_hour_min     sql.NullInt64
		// Soil_moisture_10_cm_max      sql.NullFloat64
		// Soil_moisture_10_cm_hour_max sql.NullInt64
		// Soil_moisture_10_cm_min      sql.NullFloat64
		// Soil_moisture_10_cm_hour_min sql.NullInt64
		// Soil_moisture_25_cm_max      sql.NullFloat64
		// Soil_moisture_25_cm_hour_max sql.NullInt64
		// Soil_moisture_25_cm_min      sql.NullFloat64
		// Soil_moisture_25_cm_hour_min sql.NullInt64
		// Wind_speed_max               sql.NullFloat64
		// Wind_speed_hour_max          sql.NullInt64
		// Rain_mm                      sql.NullFloat64
		// Battery_voltage_min          sql.NullFloat64
		// Date                         time.Time
		// Precip                       sql.NullFloat64
	}
	return value
}

func day_observations(db *sqlx.DB, c *gin.Context) {
	rows, err := db.Queryx("select * from weather.day_observations_cache")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	for rows.Next() {
		observation := DayObservation{}
		if err := rows.StructScan(&observation); err != nil {
			log.Fatal(err)
		}

		observation.Sol_rad_max.Float64 = observation.Sol_rad_max.Float64 * (0.6977 * 60)
		observation.Rh_max.Float64 = observation.Rh_max.Float64 * 100
		observation.Rh_min.Float64 = observation.Rh_min.Float64 * 100
		writer := csv.NewWriter(c.Writer)
		writer.Write(observation.toMawn())
		writer.Flush()

	}
	/* solar_radiation = nil */
	/* if solar_radiation */
	/*   solar_radiation = self.solar_radiation * 86.4 */
	/* end */

	/* sprintf("24,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, %s, %s,%s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s, %s,%s, %s, %s, %s", */
	/*         year_rtm, day_rtm,hour_rtm, */
	/*         self.air_temp_107_max, */
	/*         self.air_temp_107_hour_max,  self.air_temp_107_min, */
	/*         self.air_temp_107_hour_min, */
	/*         self.rh_max * 100, self.rh_hour_max, self.rh_min * 100, */
	/*         self.rh_hour_min, solar_radiation, */
	/*         self.sol_rad_max * (0.6977 * 60) , self.sol_rad_hour_max, */
	/*         self.soil_temp_5_cm_max,soil_temp_5_cm_hour_max, */
	/*         soil_temp_5_cm_min, soil_temp_5_cm_hour_min, */
	/*         self.soil_temp_10_cm_max, */
	/*         soil_temp_10_cm_hour_max, */
	/*         self.soil_temp_10_cm_min, */
	/*         soil_temp_10_cm_hour_min, */
	/*         self.soil_moisture_10_cm_max, self.soil_moisture_10_cm_hour_max, */
	/*         self.soil_moisture_10_cm_min, */
	/*         self.soil_moisture_10_cm_hour_min, self.soil_moisture_25_cm_max, */
	/*         self.soil_moisture_25_cm_hour_max, */
	/*         self.soil_moisture_25_cm_min, self.soil_moisture_25_cm_hour_min, */
	/*         self.wind_speed_max, */
	/*         self.wind_speed_hour_max, self.rain_mm, self.battery_voltage_min, */
	/*         self.date) */
}

func hour_observations(db *sqlx.DB, c *gin.Context) {
}

func five_minute_observations(db *sqlx.DB, c *gin.Context) {
}
func Router(db *sqlx.DB) *gin.Engine {
	router := gin.Default()
	router.GET("/tables", func(c *gin.Context) {
		tables(db, c)
	})
	router.GET("/tables/:id", func(c *gin.Context) {
		tablesById(db, c)
	})
	router.GET("/variates", func(c *gin.Context) {
		variates(db, c)
	})
	router.GET("/variates/:id", func(c *gin.Context) {
		variatesById(db, c)
	})
	router.GET("/day_observations", func(c *gin.Context) {
		day_observations(db, c)
	})
	router.GET("/hour_observations", func(c *gin.Context) {
		hour_observations(db, c)
	})
	router.GET("/five_minute_observations", func(c *gin.Context) {
		five_minute_observations(db, c)
	})

	return router
}

func main() {
	configFile, err := os.Open(".env")
	checkErr(err, "can't read .env file")

	u := User{}
	jsonParser := json.NewDecoder(configFile)
	if err = jsonParser.Decode(&u); err != nil {
		checkErr(err, "parsing config file")
	}

	connection := "user=" + u.Name + " password=" + u.Password + " dbname=metadata host=127.0.0.1 port=5430"
	db, err := sqlx.Open("postgres", connection)
	checkErr(err, "sql.Open failed")
	defer db.Close()

	Router(db).Run("127.0.0.1:9000")
}
<commit_msg>finished the daily observation<commit_after>package main

import (
	"database/sql"
	"encoding/csv"
	"encoding/json"
	"github.com/gin-gonic/gin"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"os"
	"strconv"
	"time"
)

func CampbellTime(myTime time.Time) [3]int {
	hourmin := myTime.Hour()*100 + myTime.Minute()
	if hourmin == 0 {
		hourmin = 2400
	}
	return [3]int{myTime.Year(), myTime.YearDay(), hourmin}
}

type User struct {
	Name     string `json:"name"`
	Password string `json:"password"`
}

func checkErr(err error, msg string) {
	if err != nil {
		log.Fatalln(msg, err)
	}
}

func Index(c *gin.Context) {
	c.String(200, "we are here")
}

type Datum struct {
	Time  time.Time `json:"time"`
	Value float64   `json:"value"`
}

func variatesById(db *sqlx.DB, c *gin.Context) {
	id := c.Params.ByName("id")
	var query string
	db.Exec("set search_path=weather")
	db.Get(&query, "select query from weather.variates where id = $1", id)
	var data = []Datum{}
	db.Select(&data, query)
	c.JSON(200, data)
}

type Table struct {
	Id   int
	Name string
}

type Variate struct {
	Id    int
	Title string
}

func tables(db *sqlx.DB, c *gin.Context) {
	db.Exec("set search_path=weather")
	var tables []Table
	db.Select(&tables, "select id, name from weather.tables")
	c.JSON(200, tables)
}

func variates(db *sqlx.DB, c *gin.Context) {
	db.Exec("set search_path=weather")
	var variates []Variate
	db.Select(&variates, "select id, title from weather.variates")
	c.JSON(200, variates)
}

func tablesById(db *sqlx.DB, c *gin.Context) {
	id := c.Params.ByName("id")
	var query string
	db.Exec("set search_path=weather")
	db.Get(&query, "select query from weather.tables where id = $1", id)
	rows, _ := db.Queryx(query)
	var results []interface{}
	for rows.Next() {
		cols, _ := rows.SliceScan()
		results = append(results, cols)
	}
	rows.Close()
	c.JSON(200, results)
}

func floatToString(value sql.NullFloat64) string {
	return strconv.FormatFloat(value.Float64, 'f', 5, 64)
}

func intToString(value sql.NullInt64) string {
	return strconv.FormatInt(value.Int64, 10)
}

type DayObservation struct {
	Year_rtm                     sql.NullInt64
	Day_rtm                      sql.NullInt64
	Hour_rtm                     sql.NullInt64
	Air_temp_107_max             sql.NullFloat64
	Air_temp_107_hour_max        sql.NullInt64
	Air_temp_107_min             sql.NullFloat64
	Air_temp_107_hour_min        sql.NullInt64
	Rh_max                       sql.NullFloat64
	Rh_hour_max                  sql.NullInt64
	Rh_min                       sql.NullFloat64
	Rh_hour_min                  sql.NullInt64
	Solar_radiation              sql.NullFloat64
	Sol_rad_max                  sql.NullFloat64
	Sol_rad_hour_max             sql.NullInt64
	Soil_temp_5_cm_max           sql.NullFloat64
	Soil_temp_5_cm_hour_max      sql.NullInt64
	Soil_temp_5_cm_min           sql.NullFloat64
	Soil_temp_5_cm_hour_min      sql.NullInt64
	Soil_temp_10_cm_max          sql.NullFloat64
	Soil_temp_10_cm_hour_max     sql.NullInt64
	Soil_temp_10_cm_min          sql.NullFloat64
	Soil_temp_10_cm_hour_min     sql.NullInt64
	Soil_moisture_10_cm_max      sql.NullFloat64
	Soil_moisture_10_cm_hour_max sql.NullInt64
	Soil_moisture_10_cm_min      sql.NullFloat64
	Soil_moisture_10_cm_hour_min sql.NullInt64
	Soil_moisture_25_cm_max      sql.NullFloat64
	Soil_moisture_25_cm_hour_max sql.NullInt64
	Soil_moisture_25_cm_min      sql.NullFloat64
	Soil_moisture_25_cm_hour_min sql.NullInt64
	Wind_speed_max               sql.NullFloat64
	Wind_speed_hour_max          sql.NullInt64
	Rain_mm                      sql.NullFloat64
	Battery_voltage_min          sql.NullFloat64
	Date                         time.Time
	Precip                       sql.NullFloat64
}

func (d *DayObservation) toMawn() []string {
	value := []string{
		"24",
		intToString(d.Year_rtm),
		intToString(d.Day_rtm),
		intToString(d.Hour_rtm),
		floatToString(d.Air_temp_107_max),
		intToString(d.Air_temp_107_hour_max),
		floatToString(d.Air_temp_107_min),
		intToString(d.Air_temp_107_hour_min),
		floatToString(d.Rh_max),
		intToString(d.Rh_hour_max),
		floatToString(d.Rh_min),
		intToString(d.Rh_hour_min),
		floatToString(d.Solar_radiation),
		floatToString(d.Sol_rad_max),
		intToString(d.Sol_rad_hour_max),
		floatToString(d.Soil_temp_5_cm_max),
		intToString(d.Soil_temp_5_cm_hour_max),
		floatToString(d.Soil_temp_5_cm_min),
		intToString(d.Soil_temp_5_cm_hour_min),
		floatToString(d.Soil_temp_10_cm_max),
		intToString(d.Soil_temp_10_cm_hour_max),
		floatToString(d.Soil_temp_10_cm_min),
		intToString(d.Soil_temp_10_cm_hour_min),
		floatToString(d.Soil_moisture_10_cm_max),
		intToString(d.Soil_moisture_10_cm_hour_max),
		floatToString(d.Soil_moisture_10_cm_min),
		intToString(d.Soil_moisture_10_cm_hour_min),
		floatToString(d.Soil_moisture_25_cm_max),
		intToString(d.Soil_moisture_25_cm_hour_max),
		floatToString(d.Soil_moisture_25_cm_min),
		intToString(d.Soil_moisture_25_cm_hour_min),
		floatToString(d.Wind_speed_max),
		intToString(d.Wind_speed_hour_max),
		floatToString(d.Rain_mm),
		floatToString(d.Battery_voltage_min),
		d.Date.String(),
	}
	return value
}

func (d *DayObservation) mawnHeader() []string {
	value := []string{
		"#code",
		"year",
		"day",
		"report time",
		"air temperature maximum",
		"time of air temperature maximum",
		"air temperature minimum",
		"time of air temperature minimum",
		"relative humidity maximum",
		"time of relative humidity maximum",
		"relative humidity minimum",
		"time of relative humidity minimum",
		"solar radiation",
		"solar radiation density maximum",
		"time  of solar radiation desnsity",
		"soil temperature maximum at 5 cm under bare soil",
		"time  of soil temperature maximum at 5 cm under bare soil",
		"soil temperature minimum at 5 cm under bare soil",
		"time of soil temperature minimum at 5 cm under bare soil",
		"soil temperature maximum at 10 cm under bare soil",
		"time of soil temperature at 10 cm  under bare soil",
		"soil temperature minimum at 10 cm under bare soil",
		"time of soil temperature minimum at 10 cm under bare soil",
		"soil moisture maximum at 5 cm under sod",
		"time of soil moisture maximum at 5 cm under sod",
		"soil moisture minimum at 5 cm under sod",
		"time of soil moisture minimum at 5 cm under sod",
		"soil moisture maximum at 20 cm under sod",
		"time of soil moisture maximum at 20 cm under sod",
		"soil moisture minimum at  20 cm under sod",
		"time of soil moisture minimum at 20 cm under sod",
		"maximum wind speed",
		"time of maximum wind speed",
		"total precipitation",
		"minimum data logger battery voltage",
		"timestamp",
	}
	return value
}

func day_observations(db *sqlx.DB, c *gin.Context) {
	rows, err := db.Queryx("select * from weather.day_observations_cache")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	i := 0
	writer := csv.NewWriter(c.Writer)

	observation := DayObservation{}
	writer.Write(observation.mawnHeader())
	for rows.Next() {
		if err := rows.StructScan(&observation); err != nil {
			log.Fatal(err)
		}

		observation.Solar_radiation.Float64 = observation.Solar_radiation.Float64 * 86.4
		observation.Sol_rad_max.Float64 = observation.Sol_rad_max.Float64 * (0.6977 * 60)
		observation.Rh_max.Float64 = observation.Rh_max.Float64 * 100
		observation.Rh_min.Float64 = observation.Rh_min.Float64 * 100

		writer.Write(observation.toMawn())

		if i%500 == 0 {
			writer.Flush()
		}
		i = i + 1

	}
	writer.Flush()
}

func hour_observations(db *sqlx.DB, c *gin.Context) {

}

func five_minute_observations(db *sqlx.DB, c *gin.Context) {
	rows, err := db.Queryx("select * from weather.lter_five_minute_a")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
}

func Router(db *sqlx.DB) *gin.Engine {
	router := gin.Default()
	router.GET("/tables", func(c *gin.Context) {
		tables(db, c)
	})
	router.GET("/tables/:id", func(c *gin.Context) {
		tablesById(db, c)
	})
	router.GET("/variates", func(c *gin.Context) {
		variates(db, c)
	})
	router.GET("/variates/:id", func(c *gin.Context) {
		variatesById(db, c)
	})
	router.GET("/day_observations", func(c *gin.Context) {
		day_observations(db, c)
	})
	router.GET("/hour_observations", func(c *gin.Context) {
		hour_observations(db, c)
	})
	router.GET("/five_minute_observations", func(c *gin.Context) {
		five_minute_observations(db, c)
	})

	return router
}

func main() {
	configFile, err := os.Open(".env")
	checkErr(err, "can't read .env file")

	u := User{}
	jsonParser := json.NewDecoder(configFile)
	if err = jsonParser.Decode(&u); err != nil {
		checkErr(err, "parsing config file")
	}

	connection := "user=" + u.Name + " password=" + u.Password + " dbname=metadata host=127.0.0.1 port=5430"
	db, err := sqlx.Open("postgres", connection)
	checkErr(err, "sql.Open failed")
	defer db.Close()

	Router(db).Run("127.0.0.1:9000")
}
<|endoftext|>


<commit_before>package context

// The use of //export prevents being able to define any C code in the preamble of that file.  Export
// defines additional C code necessary for the context<->sass_context bridge.
// See: http://golang.org/cmd/cgo/#hdr-C_references_to_Go

// #cgo LDFLAGS: -lsass -lstdc++ -lm
// #cgo CFLAGS:
//
// #include "sass_context.h"
import "C"
import "unsafe"

// goBridge is exported to C for providing a Go function reference that can
// be executed by C.
//export goBridge
func goBridge(cargs UnionSassValue, ptr unsafe.Pointer) UnionSassValue {
	// Recover the Cookie struct passed in
	ck := *(*Cookie)(ptr)
	usv := ck.fn(ck.ctx, cargs)
	return usv
}

// CookieCb defines the callback libsass eventually executes in sprite_sass
type SassCallback func(ctx *Context, csv UnionSassValue) UnionSassValue

type handler struct {
	sign     string
	callback func(ctx *Context, csv UnionSassValue) UnionSassValue
}

// handlers is the list of registered sass handlers
var handlers []handler

func SampleCB(ctx *Context, usv UnionSassValue) UnionSassValue {
	var sv []interface{}
	Unmarshal(usv, &sv)
	return C.sass_make_boolean(false)
}

// RegisterHandler sets the passed signature and callback to the
// handlers array.
func RegisterHandler(sign string,
	callback func(ctx *Context, csv UnionSassValue) UnionSassValue) {
	handlers = append(handlers, handler{sign, callback})
}
<commit_msg>helper for creating UnionSassValue errors, not that they work<commit_after>package context

// The use of //export prevents being able to define any C code in the preamble of that file.  Export
// defines additional C code necessary for the context<->sass_context bridge.
// See: http://golang.org/cmd/cgo/#hdr-C_references_to_Go

// #cgo LDFLAGS: -lsass -lstdc++ -lm
// #cgo CFLAGS:
//
// #include "sass_context.h"
import "C"
import "unsafe"

// goBridge is exported to C for providing a Go function reference that can
// be executed by C.
//export goBridge
func goBridge(cargs UnionSassValue, ptr unsafe.Pointer) UnionSassValue {
	// Recover the Cookie struct passed in
	ck := *(*Cookie)(ptr)
	usv := ck.fn(ck.ctx, cargs)
	return usv
}

// CookieCb defines the callback libsass eventually executes in sprite_sass
type SassCallback func(ctx *Context, csv UnionSassValue) UnionSassValue

type handler struct {
	sign     string
	callback func(ctx *Context, csv UnionSassValue) UnionSassValue
}

// handlers is the list of registered sass handlers
var handlers []handler

func SampleCB(ctx *Context, usv UnionSassValue) UnionSassValue {
	var sv []interface{}
	Unmarshal(usv, &sv)
	return C.sass_make_boolean(false)
}

func Error(err error) UnionSassValue {
	return C.sass_make_error(C.CString(err.Error()))
}

// RegisterHandler sets the passed signature and callback to the
// handlers array.
func RegisterHandler(sign string,
	callback func(ctx *Context, csv UnionSassValue) UnionSassValue) {
	handlers = append(handlers, handler{sign, callback})
}
<|endoftext|>


<commit_before>package server

import (
	"strings"

	"github.com/bmorton/deployster/fleet"
)

type ExtractableUnit fleet.Unit

func (eu *ExtractableUnit) ExtractBaseName() string {
	s := strings.Split(eu.Name, "-")
	return s[0]
}

func (eu *ExtractableUnit) ExtractVersion() string {
	s := strings.Split(eu.Name, "-")
	end := strings.Index(s[1], "@")
	return s[1][:end]
}

func (eu *ExtractableUnit) ExtractInstance() string {
	s := strings.Split(eu.Name, "@")
	end := strings.Index(s[1], ".")
	return s[1][:end]
}
<commit_msg>Add documentation to ExtractableUnit in the server package.<commit_after>package server

import (
	"strings"

	"github.com/bmorton/deployster/fleet"
)

// ExtractableUnit is the local struct for a fleet.Unit with added functions
// for extracting the name, version, and instance that deployster encodes into
// the Fleet unit name.
type ExtractableUnit fleet.Unit

// ExtractBaseName returns the name of the service from the Fleet unit name.
// Given "railsapp-cf2e8ac@1.service" this returns "railsapp"
func (eu *ExtractableUnit) ExtractBaseName() string {
	s := strings.Split(eu.Name, "-")
	return s[0]
}

// ExtractVersion returns the version of the service from the Fleet unit name.
// Given "railsapp-cf2e8ac@1.service" this returns "cf2e8ac"
func (eu *ExtractableUnit) ExtractVersion() string {
	s := strings.Split(eu.Name, "-")
	end := strings.Index(s[1], "@")
	return s[1][:end]
}

// ExtractInstance returns the instance of the service from the Fleet unit name.
// Given "railsapp-cf2e8ac@1.service" this returns "1"
func (eu *ExtractableUnit) ExtractInstance() string {
	s := strings.Split(eu.Name, "@")
	end := strings.Index(s[1], ".")
	return s[1][:end]
}
<|endoftext|>


<commit_before>package main

import (
	"math"
	"regexp"
	"time"

	"github.com/elastic/libbeat/beat"
	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/common"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/publisher"
)

type ProcsMap map[int]*Process

type Topbeat struct {
	isAlive      bool
	period       time.Duration
	procs        []string
	procsMap     ProcsMap
	lastCpuTimes *CpuTimes
	TbConfig     ConfigSettings
	events       chan common.MapStr
}

func (tb *Topbeat) Config(b *beat.Beat) error {

	err := cfgfile.Read(&tb.TbConfig, "")
	if err != nil {
		logp.Err("Error reading configuration file: %v", err)
		return err
	}

	if tb.TbConfig.Input.Period != nil {
		tb.period = time.Duration(*tb.TbConfig.Input.Period) * time.Second
	} else {
		tb.period = 1 * time.Second
	}
	if tb.TbConfig.Input.Procs != nil {
		tb.procs = *tb.TbConfig.Input.Procs
	} else {
		tb.procs = []string{".*"} //all processes
	}

	logp.Debug("topbeat", "Init toppbeat")
	logp.Debug("topbeat", "Follow processes %q\n", tb.procs)
	logp.Debug("topbeat", "Period %v\n", tb.period)

	return nil
}

func (tb *Topbeat) Setup(b *beat.Beat) error {

	tb.events = publisher.Publisher.Queue
	return nil
}

func (t *Topbeat) Run(b *beat.Beat) error {

	t.isAlive = true

	t.initProcStats()

	var err error

	for t.isAlive {
		time.Sleep(t.period)

		err = t.exportSystemStats()
		if err != nil {
			logp.Err("Error reading system stats: %v", err)
		}
		err = t.exportProcStats()
		if err != nil {
			logp.Err("Error reading proc stats: %v", err)
		}
		err = t.exportFileSystemStats()
		if err != nil {
			logp.Err("Error reading fs stats: %v", err)
		}
	}

	return err
}

func (tb *Topbeat) Cleanup(b *beat.Beat) error {
	return nil
}

func (t *Topbeat) Stop() {

	t.isAlive = false
}

func (t *Topbeat) initProcStats() {

	t.procsMap = make(ProcsMap)

	if len(t.procs) == 0 {
		return
	}

	pids, err := Pids()
	if err != nil {
		logp.Warn("Getting the list of pids: %v", err)
	}

	for _, pid := range pids {
		process, err := GetProcess(pid)
		if err != nil {
			logp.Debug("topbeat", "Skip process %d: %v", pid, err)
			continue
		}
		t.procsMap[process.Pid] = process
	}
}

func (t *Topbeat) exportProcStats() error {

	if len(t.procs) == 0 {
		return nil
	}

	pids, err := Pids()
	if err != nil {
		logp.Warn("Getting the list of pids: %v", err)
		return err
	}

	for _, pid := range pids {
		process, err := GetProcess(pid)
		if err != nil {
			logp.Debug("topbeat", "Skip process %d: %v", pid, err)
			continue
		}

		if t.MatchProcess(process.Name) {

			t.addProcCpuPercentage(process)
			t.addProcMemPercentage(process, 0 /*read total mem usage */)

			t.procsMap[process.Pid] = process

			event := common.MapStr{
				"timestamp":  common.Time(time.Now()),
				"type":       "proc",
				"proc.pid":   process.Pid,
				"proc.ppid":  process.Ppid,
				"proc.name":  process.Name,
				"proc.state": process.State,
				"proc.mem":   process.Mem,
				"proc.cpu":   process.Cpu,
			}
			t.events <- event
		}
	}
	return nil
}

func (t *Topbeat) exportSystemStats() error {

	load_stat, err := GetSystemLoad()
	if err != nil {
		logp.Warn("Getting load statistics: %v", err)
		return err
	}
	cpu_stat, err := GetCpuTimes()
	if err != nil {
		logp.Warn("Getting cpu times: %v", err)
		return err
	}

	t.addCpuPercentage(cpu_stat)

	mem_stat, err := GetMemory()
	if err != nil {
		logp.Warn("Getting memory details: %v", err)
		return err
	}
	t.addMemPercentage(mem_stat)

	swap_stat, err := GetSwap()
	if err != nil {
		logp.Warn("Getting swap details: %v", err)
		return err
	}
	t.addMemPercentage(swap_stat)

	event := common.MapStr{
		"timestamp": common.Time(time.Now()),
		"type":      "system",
		"load":      load_stat,
		"cpu":       cpu_stat,
		"mem":       mem_stat,
		"swap":      swap_stat,
	}

	t.events <- event

	return nil
}

func (t *Topbeat) exportFileSystemStats() error {

	fss, err := GetFileSystemList()
	if err != nil {
		logp.Warn("Getting filesystem list: %v", err)
		return err
	}

	for _, fs := range fss {
		fs_stat, err := GetFileSystemStat(fs)
		if err != nil {
			logp.Debug("topbeat", "Skip filesystem %d: %v", fs_stat, err)
			continue
		}
		t.addFileSystemUsedPercentage(fs_stat)

		event := common.MapStr{
			"timestamp": common.Time(time.Now()),
			"type":      "filesystem",
			"fs":        fs_stat,
		}
		t.events <- event
	}

	return nil
}

func (t *Topbeat) MatchProcess(name string) bool {

	for _, reg := range t.procs {
		matched, _ := regexp.MatchString(reg, name)
		if matched {
			return true
		}
	}
	return false
}

func (t *Topbeat) addMemPercentage(m *MemStat) {

	if m.Total == 0 {
		return
	}

	perc := float64(m.Used) / float64(m.Total)
	m.UsedPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addFileSystemUsedPercentage(f *FileSystemStat) {

	if f.Total == 0 {
		return
	}

	perc := float64(f.Used) / float64(f.Total)
	f.UsedPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addCpuPercentage(t2 *CpuTimes) {

	t1 := t.lastCpuTimes

	if t1 != nil && t2 != nil {
		all_delta := t2.sum() - t1.sum()

		calculate := func(field2 uint64, field1 uint64) float64 {

			perc := 0.0
			delta := field2 - field1
			perc = float64(delta) / float64(all_delta)
			return Round(perc, .5, 2)
		}

		t2.UserPercent = calculate(t2.User, t1.User)
		t2.SystemPercent = calculate(t2.System, t1.System)
	}

	t.lastCpuTimes = t2

}

func (t *Topbeat) addProcMemPercentage(proc *Process, total_phymem uint64) {

	// in unit tests, total_phymem is set to a value greater than zero

	if total_phymem == 0 {
		mem_stat, err := GetMemory()
		if err != nil {
			logp.Warn("Getting memory details: %v", err)
			return
		}
		total_phymem = mem_stat.Total
	}

	perc := (float64(proc.Mem.Rss) / float64(total_phymem))

	proc.Mem.RssPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addProcCpuPercentage(proc *Process) {

	oproc, ok := t.procsMap[proc.Pid]
	if ok {

		delta_proc := (proc.Cpu.User - oproc.Cpu.User) + (proc.Cpu.System - oproc.Cpu.System)
		delta_time := proc.ctime.Sub(oproc.ctime).Nanoseconds() / 1e6 // in milliseconds
		perc := float64(delta_proc) / float64(delta_time)

		t.procsMap[proc.Pid] = proc

		proc.Cpu.UserPercent = Round(perc, .5, 2)

	}
}

func Round(val float64, roundOn float64, places int) (newVal float64) {
	var round float64
	pow := math.Pow(10, float64(places))
	digit := pow * val
	_, div := math.Modf(digit)
	if div >= roundOn {
		round = math.Ceil(digit)
	} else {
		round = math.Floor(digit)
	}
	newVal = round / pow
	return
}

func (t *CpuTimes) sum() uint64 {
	return t.User + t.Nice + t.System + t.Idle + t.IOWait + t.Irq + t.SoftIrq + t.Steal
}
<commit_msg>Remove dots from the exported fields<commit_after>package main

import (
	"math"
	"regexp"
	"time"

	"github.com/elastic/libbeat/beat"
	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/common"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/publisher"
)

type ProcsMap map[int]*Process

type Topbeat struct {
	isAlive      bool
	period       time.Duration
	procs        []string
	procsMap     ProcsMap
	lastCpuTimes *CpuTimes
	TbConfig     ConfigSettings
	events       chan common.MapStr
}

func (tb *Topbeat) Config(b *beat.Beat) error {

	err := cfgfile.Read(&tb.TbConfig, "")
	if err != nil {
		logp.Err("Error reading configuration file: %v", err)
		return err
	}

	if tb.TbConfig.Input.Period != nil {
		tb.period = time.Duration(*tb.TbConfig.Input.Period) * time.Second
	} else {
		tb.period = 1 * time.Second
	}
	if tb.TbConfig.Input.Procs != nil {
		tb.procs = *tb.TbConfig.Input.Procs
	} else {
		tb.procs = []string{".*"} //all processes
	}

	logp.Debug("topbeat", "Init toppbeat")
	logp.Debug("topbeat", "Follow processes %q\n", tb.procs)
	logp.Debug("topbeat", "Period %v\n", tb.period)

	return nil
}

func (tb *Topbeat) Setup(b *beat.Beat) error {

	tb.events = publisher.Publisher.Queue
	return nil
}

func (t *Topbeat) Run(b *beat.Beat) error {

	t.isAlive = true

	t.initProcStats()

	var err error

	for t.isAlive {
		time.Sleep(t.period)

		err = t.exportSystemStats()
		if err != nil {
			logp.Err("Error reading system stats: %v", err)
		}
		err = t.exportProcStats()
		if err != nil {
			logp.Err("Error reading proc stats: %v", err)
		}
		err = t.exportFileSystemStats()
		if err != nil {
			logp.Err("Error reading fs stats: %v", err)
		}
	}

	return err
}

func (tb *Topbeat) Cleanup(b *beat.Beat) error {
	return nil
}

func (t *Topbeat) Stop() {

	t.isAlive = false
}

func (t *Topbeat) initProcStats() {

	t.procsMap = make(ProcsMap)

	if len(t.procs) == 0 {
		return
	}

	pids, err := Pids()
	if err != nil {
		logp.Warn("Getting the list of pids: %v", err)
	}

	for _, pid := range pids {
		process, err := GetProcess(pid)
		if err != nil {
			logp.Debug("topbeat", "Skip process %d: %v", pid, err)
			continue
		}
		t.procsMap[process.Pid] = process
	}
}

func (t *Topbeat) exportProcStats() error {

	if len(t.procs) == 0 {
		return nil
	}

	pids, err := Pids()
	if err != nil {
		logp.Warn("Getting the list of pids: %v", err)
		return err
	}

	for _, pid := range pids {
		process, err := GetProcess(pid)
		if err != nil {
			logp.Debug("topbeat", "Skip process %d: %v", pid, err)
			continue
		}

		if t.MatchProcess(process.Name) {

			t.addProcCpuPercentage(process)
			t.addProcMemPercentage(process, 0 /*read total mem usage */)

			t.procsMap[process.Pid] = process

			event := common.MapStr{
				"timestamp": common.Time(time.Now()),
				"type":      "proc",
				"proc": common.MapStr{
					"pid":   process.Pid,
					"ppid":  process.Ppid,
					"name":  process.Name,
					"state": process.State,
					"mem":   process.Mem,
					"cpu":   process.Cpu,
				},
			}
			t.events <- event
		}
	}
	return nil
}

func (t *Topbeat) exportSystemStats() error {

	load_stat, err := GetSystemLoad()
	if err != nil {
		logp.Warn("Getting load statistics: %v", err)
		return err
	}
	cpu_stat, err := GetCpuTimes()
	if err != nil {
		logp.Warn("Getting cpu times: %v", err)
		return err
	}

	t.addCpuPercentage(cpu_stat)

	mem_stat, err := GetMemory()
	if err != nil {
		logp.Warn("Getting memory details: %v", err)
		return err
	}
	t.addMemPercentage(mem_stat)

	swap_stat, err := GetSwap()
	if err != nil {
		logp.Warn("Getting swap details: %v", err)
		return err
	}
	t.addMemPercentage(swap_stat)

	event := common.MapStr{
		"timestamp": common.Time(time.Now()),
		"type":      "system",
		"load":      load_stat,
		"cpu":       cpu_stat,
		"mem":       mem_stat,
		"swap":      swap_stat,
	}

	t.events <- event

	return nil
}

func (t *Topbeat) exportFileSystemStats() error {

	fss, err := GetFileSystemList()
	if err != nil {
		logp.Warn("Getting filesystem list: %v", err)
		return err
	}

	for _, fs := range fss {
		fs_stat, err := GetFileSystemStat(fs)
		if err != nil {
			logp.Debug("topbeat", "Skip filesystem %d: %v", fs_stat, err)
			continue
		}
		t.addFileSystemUsedPercentage(fs_stat)

		event := common.MapStr{
			"timestamp": common.Time(time.Now()),
			"type":      "filesystem",
			"fs":        fs_stat,
		}
		t.events <- event
	}

	return nil
}

func (t *Topbeat) MatchProcess(name string) bool {

	for _, reg := range t.procs {
		matched, _ := regexp.MatchString(reg, name)
		if matched {
			return true
		}
	}
	return false
}

func (t *Topbeat) addMemPercentage(m *MemStat) {

	if m.Total == 0 {
		return
	}

	perc := float64(m.Used) / float64(m.Total)
	m.UsedPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addFileSystemUsedPercentage(f *FileSystemStat) {

	if f.Total == 0 {
		return
	}

	perc := float64(f.Used) / float64(f.Total)
	f.UsedPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addCpuPercentage(t2 *CpuTimes) {

	t1 := t.lastCpuTimes

	if t1 != nil && t2 != nil {
		all_delta := t2.sum() - t1.sum()

		calculate := func(field2 uint64, field1 uint64) float64 {

			perc := 0.0
			delta := field2 - field1
			perc = float64(delta) / float64(all_delta)
			return Round(perc, .5, 2)
		}

		t2.UserPercent = calculate(t2.User, t1.User)
		t2.SystemPercent = calculate(t2.System, t1.System)
	}

	t.lastCpuTimes = t2

}

func (t *Topbeat) addProcMemPercentage(proc *Process, total_phymem uint64) {

	// in unit tests, total_phymem is set to a value greater than zero

	if total_phymem == 0 {
		mem_stat, err := GetMemory()
		if err != nil {
			logp.Warn("Getting memory details: %v", err)
			return
		}
		total_phymem = mem_stat.Total
	}

	perc := (float64(proc.Mem.Rss) / float64(total_phymem))

	proc.Mem.RssPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addProcCpuPercentage(proc *Process) {

	oproc, ok := t.procsMap[proc.Pid]
	if ok {

		delta_proc := (proc.Cpu.User - oproc.Cpu.User) + (proc.Cpu.System - oproc.Cpu.System)
		delta_time := proc.ctime.Sub(oproc.ctime).Nanoseconds() / 1e6 // in milliseconds
		perc := float64(delta_proc) / float64(delta_time)

		t.procsMap[proc.Pid] = proc

		proc.Cpu.UserPercent = Round(perc, .5, 2)

	}
}

func Round(val float64, roundOn float64, places int) (newVal float64) {
	var round float64
	pow := math.Pow(10, float64(places))
	digit := pow * val
	_, div := math.Modf(digit)
	if div >= roundOn {
		round = math.Ceil(digit)
	} else {
		round = math.Floor(digit)
	}
	newVal = round / pow
	return
}

func (t *CpuTimes) sum() uint64 {
	return t.User + t.Nice + t.System + t.Idle + t.IOWait + t.Irq + t.SoftIrq + t.Steal
}
<|endoftext|>


<commit_before>package brimutil

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"hash"
	"io"
)

// ChecksummedReader reads content written by ChecksummedWriter, verifying
// checksums when requested. Implements the io.ReadSeeker and io.Closer
// interfaces.
//
// Any errors from Read or Verify should make no assumptions about any
// resulting position and should Seek before continuing to use the
// ChecksummedReader.
type ChecksummedReader interface {
	// Read implements the io.Reader interface.
	//
	// Any error should make no assumption about any resulting position and
	// should Seek before continuing to use the ChecksummedReader.
	Read(v []byte) (n int, err error)
	// Seek implements the io.Seeker interface.
	Seek(offset int64, whence int) (n int64, err error)
	// Verify verifies the checksum for the section of the content containing
	// the current read position.
	//
	// If there is an error, whether the section is checksum valid is
	// indeterminate by this routine and the caller should decide what to do
	// based on the error. For example, if the underlying i/o causes a timeout
	// error, the content may be fine and just temporarily unreachable.
	//
	// Any error should also make no assumption about any resulting position
	// and should Seek before continuing to use the ChecksummedReader.
	//
	// With no error, the bool indicates whether the content is checksum valid
	// and the position within the ChecksummedReader will not have changed.
	Verify() (bool, error)
	// Close implements the io.Closer interface.
	Close() error
}

// NewChecksummedReader returns a ChecksummedReader that delegates requests to
// an underlying io.ReadSeeker expecting checksums of the content at given
// intervals using the hashing function given.
func NewChecksummedReader(delegate io.ReadSeeker, interval int, newHash func() hash.Hash32) ChecksummedReader {
	return newChecksummedReaderImpl(delegate, interval, newHash)
}

// ChecksummedWriter writes content with additional checksums embedded in the
// underlying content.
//
// Implements the io.WriteCloser interface.
//
// Note that this generally only works for brand new writers starting at offset
// 0. Appending to existing files or starting at offsets other than 0 requires
// special care when working with ChecksummedReader later and is beyond the
// basic usage described here.
//
// Also, do not forget to Close the ChecksummedWriter to ensure the final
// checksum is emitted.
type ChecksummedWriter interface {
	// Write implements the io.Writer interface.
	Write(v []byte) (n int, err error)
	// Close implements the io.Closer interface.
	Close() error
}

// NewChecksummedWriter returns a ChecksummedWriter that delegates requests to
// an underlying io.Writer and embeds checksums of the content at given
// intervals using the hashing function given.
func NewChecksummedWriter(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32) ChecksummedWriter {
	return newChecksummedWriterImpl(delegate, checksumInterval, newHash)
}

type checksummedReaderImpl struct {
	delegate         io.ReadSeeker
	checksumInterval int
	checksumOffset   int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	checksum         []byte
}

func newChecksummedReaderImpl(delegate io.ReadSeeker, interval int, newHash func() hash.Hash32) ChecksummedReader {
	return &checksummedReaderImpl{
		delegate:         delegate,
		checksumInterval: interval,
		newHash:          newHash,
		checksum:         make([]byte, 4),
	}
}

func (cri *checksummedReaderImpl) Read(v []byte) (int, error) {
	if cri.checksumOffset+len(v) > cri.checksumInterval {
		v = v[:cri.checksumInterval-cri.checksumOffset]
	}
	n, err := cri.delegate.Read(v)
	cri.checksumOffset += n
	if err == io.EOF {
		n -= 4
		if n < 0 {
			n = 0
		}
	} else if err == nil {
		if cri.checksumOffset == cri.checksumInterval {
			n2, err := io.ReadFull(cri.delegate, cri.checksum)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				n -= 4 - n2
				err = io.EOF
			}
			cri.checksumOffset = 0
		} else {
			n2, err := io.ReadFull(cri.delegate, cri.checksum)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				n -= 4 - n2
				err = io.EOF
			} else {
				_, err = cri.delegate.Seek(-int64(n2), 1)
			}
		}
	}
	return n, err
}

func (cri *checksummedReaderImpl) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case 0:
	case 1:
		o, err := cri.delegate.Seek(0, 1)
		cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
		if err != nil {
			return o - (o / int64(cri.checksumInterval) * 4), err
		}
		offset = o - (o / int64(cri.checksumInterval) * 4) + offset
	case 2:
		o, err := cri.delegate.Seek(0, 2)
		cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
		if err != nil {
			return o - (o / int64(cri.checksumInterval) * 4), err
		}
		offset = o - (o / int64(cri.checksumInterval) * 4) + offset
	default:
		o, _ := cri.delegate.Seek(0, 1)
		return o, fmt.Errorf("invalid whence %d", whence)
	}
	o, err := cri.delegate.Seek(offset+(offset/int64(cri.checksumInterval)*4), 0)
	cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
	return o - (o / int64(cri.checksumInterval) * 4), err
}

func (cri *checksummedReaderImpl) Verify() (bool, error) {
	originalOffset, err := cri.delegate.Seek(0, 1)
	if err != nil {
		return false, err
	}
	if cri.checksumOffset > 0 {
		_, err = cri.delegate.Seek(-int64(cri.checksumOffset), 1)
		if err != nil {
			return false, err
		}
	}
	block := make([]byte, cri.checksumInterval+4)
	checksum := block[cri.checksumInterval:]
	n, err := io.ReadFull(cri.delegate, block)
	if err == io.ErrUnexpectedEOF {
		checksum = block[n-4 : n]
		block = block[:n-4]
	} else if err != nil {
		return false, err
	} else {
		block = block[:cri.checksumInterval]
	}
	hash := cri.newHash()
	hash.Write(block)
	verified := bytes.Equal(checksum, hash.Sum(cri.checksum[:0]))
	_, err = cri.delegate.Seek(originalOffset, 0)
	if err != nil {
		return verified, err
	}
	return verified, nil
}

func (cri *checksummedReaderImpl) Close() error {
	var err error
	if c, ok := cri.delegate.(io.Closer); ok {
		err = c.Close()
	}
	cri.delegate = _ERR_DELEGATE
	return err
}

type checksummedWriterImpl struct {
	delegate         io.Writer
	checksumInterval int
	checksumOffset   int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	checksum         []byte
}

func newChecksummedWriterImpl(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32) *checksummedWriterImpl {
	return &checksummedWriterImpl{
		delegate:         delegate,
		checksumInterval: checksumInterval,
		newHash:          newHash,
		hash:             newHash(),
		checksum:         make([]byte, 4),
	}
}

func (cwi *checksummedWriterImpl) Write(v []byte) (int, error) {
	var n int
	var n2 int
	var err error
	for cwi.checksumOffset+len(v) >= cwi.checksumInterval {
		n2, err = cwi.delegate.Write(v[:cwi.checksumInterval-cwi.checksumOffset])
		n += n2
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash.Write(v[:cwi.checksumInterval-cwi.checksumOffset])
		v = v[cwi.checksumInterval-cwi.checksumOffset:]
		binary.BigEndian.PutUint32(cwi.checksum, cwi.hash.Sum32())
		_, err = cwi.delegate.Write(cwi.checksum)
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash = cwi.newHash()
		cwi.checksumOffset = 0
	}
	if len(v) > 0 {
		n2, err = cwi.delegate.Write(v)
		n += n2
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash.Write(v)
		cwi.checksumOffset += n2
	}
	return n, err
}

func (cwi *checksummedWriterImpl) Close() error {
	var err error
	if cwi.checksumOffset > 0 {
		binary.BigEndian.PutUint32(cwi.checksum, cwi.hash.Sum32())
		_, err = cwi.delegate.Write(cwi.checksum)
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return err
		}
	}
	if c, ok := cwi.delegate.(io.Closer); ok {
		err = c.Close()
	}
	cwi.delegate = _ERR_DELEGATE
	return err
}

type multiCoreChecksummedWriter struct {
	delegate         io.Writer
	checksumInterval int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	buffer           *multiCoreChecksummedWriterBuffer
	freeChan         chan *multiCoreChecksummedWriterBuffer
	checksumChan     chan *multiCoreChecksummedWriterBuffer
	writeChan        chan *multiCoreChecksummedWriterBuffer
	doneChan         chan struct{}
	err              error
	closed           bool
}

type multiCoreChecksummedWriterBuffer struct {
	seq int64
	buf []byte
}

// NewMultiCoreChecksummedWriter returns a ChecksummedWriter that delegates
// requests to an underlying io.Writer and embeds checksums of the content at
// given intervals using the hashing function given; it will use multiple cores
// for computing the checksums.
//
// Note that this is generally only faster for large files and reasonably sized
// checksum intervals (e.g. 65532). It can be quite a bit slower on single core
// systems or when using tiny checksum intervals.
func NewMultiCoreChecksummedWriter(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32, cores int) ChecksummedWriter {
	cwi := &multiCoreChecksummedWriter{
		delegate:         delegate,
		checksumInterval: checksumInterval,
		newHash:          newHash,
		freeChan:         make(chan *multiCoreChecksummedWriterBuffer, cores),
		checksumChan:     make(chan *multiCoreChecksummedWriterBuffer, cores),
		writeChan:        make(chan *multiCoreChecksummedWriterBuffer, cores),
		doneChan:         make(chan struct{}),
	}
	for i := 0; i < cores; i++ {
		cwi.freeChan <- &multiCoreChecksummedWriterBuffer{0, make([]byte, 0, checksumInterval+4)}
	}
	cwi.buffer = <-cwi.freeChan
	go cwi.writer()
	for i := 0; i < cores; i++ {
		go cwi.checksummer()
	}
	return cwi
}

func (cwi *multiCoreChecksummedWriter) Write(v []byte) (int, error) {
	var n int
	for len(cwi.buffer.buf)+len(v) >= cwi.checksumInterval {
		n2 := cwi.checksumInterval - len(cwi.buffer.buf)
		cwi.buffer.buf = append(cwi.buffer.buf, v[:n2]...)
		s := cwi.buffer.seq + 1
		cwi.checksumChan <- cwi.buffer
		n += n2
		v = v[n2:]
		cwi.buffer = <-cwi.freeChan
		cwi.buffer.seq = s
	}
	if len(v) > 0 {
		cwi.buffer.buf = append(cwi.buffer.buf, v...)
		n += len(v)
	}
	return n, cwi.err
}

func (cwi *multiCoreChecksummedWriter) Close() error {
	if cwi.closed {
		return cwi.err
	}
	cwi.closed = true
	if len(cwi.buffer.buf) > 0 {
		cwi.checksumChan <- cwi.buffer
	}
	close(cwi.checksumChan)
	for i := 0; i < cap(cwi.checksumChan); i++ {
		<-cwi.doneChan
	}
	cwi.writeChan <- nil
	<-cwi.doneChan
	return cwi.err
}

func (cwi *multiCoreChecksummedWriter) checksummer() {
	for {
		b := <-cwi.checksumChan
		if b == nil {
			break
		}
		h := cwi.newHash()
		h.Write(b.buf)
		b.buf = b.buf[:len(b.buf)+4]
		binary.BigEndian.PutUint32(b.buf[len(b.buf)-4:], h.Sum32())
		cwi.writeChan <- b
	}
	cwi.doneChan <- struct{}{}
}

func (cwi *multiCoreChecksummedWriter) writer() {
	var seq int64
	var lastWasNil bool
	for {
		b := <-cwi.writeChan
		if b == nil {
			if lastWasNil {
				break
			}
			lastWasNil = true
			cwi.writeChan <- nil
			continue
		}
		if b.seq != seq {
			cwi.writeChan <- b
			continue
		}
		_, cwi.err = cwi.delegate.Write(b.buf)
		b.buf = b.buf[:0]
		cwi.freeChan <- b
		seq++
	}
	if c, ok := cwi.delegate.(io.Closer); ok {
		cwi.err = c.Close()
	}
	cwi.doneChan <- struct{}{}
}

type errDelegate struct {
}

var _ERR_DELEGATE = &errDelegate{}

func (ed *errDelegate) Read(v []byte) (int, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Write(v []byte) (int, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Seek(offset int64, whence int) (int64, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Close() error {
	return fmt.Errorf("already closed")
}
<commit_msg>Fixed bug on multicore writer<commit_after>package brimutil

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"hash"
	"io"
)

// ChecksummedReader reads content written by ChecksummedWriter, verifying
// checksums when requested. Implements the io.ReadSeeker and io.Closer
// interfaces.
//
// Any errors from Read or Verify should make no assumptions about any
// resulting position and should Seek before continuing to use the
// ChecksummedReader.
type ChecksummedReader interface {
	// Read implements the io.Reader interface.
	//
	// Any error should make no assumption about any resulting position and
	// should Seek before continuing to use the ChecksummedReader.
	Read(v []byte) (n int, err error)
	// Seek implements the io.Seeker interface.
	Seek(offset int64, whence int) (n int64, err error)
	// Verify verifies the checksum for the section of the content containing
	// the current read position.
	//
	// If there is an error, whether the section is checksum valid is
	// indeterminate by this routine and the caller should decide what to do
	// based on the error. For example, if the underlying i/o causes a timeout
	// error, the content may be fine and just temporarily unreachable.
	//
	// Any error should also make no assumption about any resulting position
	// and should Seek before continuing to use the ChecksummedReader.
	//
	// With no error, the bool indicates whether the content is checksum valid
	// and the position within the ChecksummedReader will not have changed.
	Verify() (bool, error)
	// Close implements the io.Closer interface.
	Close() error
}

// NewChecksummedReader returns a ChecksummedReader that delegates requests to
// an underlying io.ReadSeeker expecting checksums of the content at given
// intervals using the hashing function given.
func NewChecksummedReader(delegate io.ReadSeeker, interval int, newHash func() hash.Hash32) ChecksummedReader {
	return newChecksummedReaderImpl(delegate, interval, newHash)
}

// ChecksummedWriter writes content with additional checksums embedded in the
// underlying content.
//
// Implements the io.WriteCloser interface.
//
// Note that this generally only works for brand new writers starting at offset
// 0. Appending to existing files or starting at offsets other than 0 requires
// special care when working with ChecksummedReader later and is beyond the
// basic usage described here.
//
// Also, do not forget to Close the ChecksummedWriter to ensure the final
// checksum is emitted.
type ChecksummedWriter interface {
	// Write implements the io.Writer interface.
	Write(v []byte) (n int, err error)
	// Close implements the io.Closer interface.
	Close() error
}

// NewChecksummedWriter returns a ChecksummedWriter that delegates requests to
// an underlying io.Writer and embeds checksums of the content at given
// intervals using the hashing function given.
func NewChecksummedWriter(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32) ChecksummedWriter {
	return newChecksummedWriterImpl(delegate, checksumInterval, newHash)
}

type checksummedReaderImpl struct {
	delegate         io.ReadSeeker
	checksumInterval int
	checksumOffset   int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	checksum         []byte
}

func newChecksummedReaderImpl(delegate io.ReadSeeker, interval int, newHash func() hash.Hash32) ChecksummedReader {
	return &checksummedReaderImpl{
		delegate:         delegate,
		checksumInterval: interval,
		newHash:          newHash,
		checksum:         make([]byte, 4),
	}
}

func (cri *checksummedReaderImpl) Read(v []byte) (int, error) {
	if cri.checksumOffset+len(v) > cri.checksumInterval {
		v = v[:cri.checksumInterval-cri.checksumOffset]
	}
	n, err := cri.delegate.Read(v)
	cri.checksumOffset += n
	if err == io.EOF {
		n -= 4
		if n < 0 {
			n = 0
		}
	} else if err == nil {
		if cri.checksumOffset == cri.checksumInterval {
			n2, err := io.ReadFull(cri.delegate, cri.checksum)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				n -= 4 - n2
				err = io.EOF
			}
			cri.checksumOffset = 0
		} else {
			n2, err := io.ReadFull(cri.delegate, cri.checksum)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				n -= 4 - n2
				err = io.EOF
			} else {
				_, err = cri.delegate.Seek(-int64(n2), 1)
			}
		}
	}
	return n, err
}

func (cri *checksummedReaderImpl) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case 0:
	case 1:
		o, err := cri.delegate.Seek(0, 1)
		cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
		if err != nil {
			return o - (o / int64(cri.checksumInterval) * 4), err
		}
		offset = o - (o / int64(cri.checksumInterval) * 4) + offset
	case 2:
		o, err := cri.delegate.Seek(0, 2)
		cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
		if err != nil {
			return o - (o / int64(cri.checksumInterval) * 4), err
		}
		offset = o - (o / int64(cri.checksumInterval) * 4) + offset
	default:
		o, _ := cri.delegate.Seek(0, 1)
		return o, fmt.Errorf("invalid whence %d", whence)
	}
	o, err := cri.delegate.Seek(offset+(offset/int64(cri.checksumInterval)*4), 0)
	cri.checksumOffset = int(o % (int64(cri.checksumInterval) + 4))
	return o - (o / int64(cri.checksumInterval) * 4), err
}

func (cri *checksummedReaderImpl) Verify() (bool, error) {
	originalOffset, err := cri.delegate.Seek(0, 1)
	if err != nil {
		return false, err
	}
	if cri.checksumOffset > 0 {
		_, err = cri.delegate.Seek(-int64(cri.checksumOffset), 1)
		if err != nil {
			return false, err
		}
	}
	block := make([]byte, cri.checksumInterval+4)
	checksum := block[cri.checksumInterval:]
	n, err := io.ReadFull(cri.delegate, block)
	if err == io.ErrUnexpectedEOF {
		checksum = block[n-4 : n]
		block = block[:n-4]
	} else if err != nil {
		return false, err
	} else {
		block = block[:cri.checksumInterval]
	}
	hash := cri.newHash()
	hash.Write(block)
	verified := bytes.Equal(checksum, hash.Sum(cri.checksum[:0]))
	_, err = cri.delegate.Seek(originalOffset, 0)
	if err != nil {
		return verified, err
	}
	return verified, nil
}

func (cri *checksummedReaderImpl) Close() error {
	var err error
	if c, ok := cri.delegate.(io.Closer); ok {
		err = c.Close()
	}
	cri.delegate = _ERR_DELEGATE
	return err
}

type checksummedWriterImpl struct {
	delegate         io.Writer
	checksumInterval int
	checksumOffset   int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	checksum         []byte
}

func newChecksummedWriterImpl(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32) *checksummedWriterImpl {
	return &checksummedWriterImpl{
		delegate:         delegate,
		checksumInterval: checksumInterval,
		newHash:          newHash,
		hash:             newHash(),
		checksum:         make([]byte, 4),
	}
}

func (cwi *checksummedWriterImpl) Write(v []byte) (int, error) {
	var n int
	var n2 int
	var err error
	for cwi.checksumOffset+len(v) >= cwi.checksumInterval {
		n2, err = cwi.delegate.Write(v[:cwi.checksumInterval-cwi.checksumOffset])
		n += n2
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash.Write(v[:cwi.checksumInterval-cwi.checksumOffset])
		v = v[cwi.checksumInterval-cwi.checksumOffset:]
		binary.BigEndian.PutUint32(cwi.checksum, cwi.hash.Sum32())
		_, err = cwi.delegate.Write(cwi.checksum)
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash = cwi.newHash()
		cwi.checksumOffset = 0
	}
	if len(v) > 0 {
		n2, err = cwi.delegate.Write(v)
		n += n2
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return n, err
		}
		cwi.hash.Write(v)
		cwi.checksumOffset += n2
	}
	return n, err
}

func (cwi *checksummedWriterImpl) Close() error {
	var err error
	if cwi.checksumOffset > 0 {
		binary.BigEndian.PutUint32(cwi.checksum, cwi.hash.Sum32())
		_, err = cwi.delegate.Write(cwi.checksum)
		if err != nil {
			cwi.delegate = _ERR_DELEGATE
			return err
		}
	}
	if c, ok := cwi.delegate.(io.Closer); ok {
		err = c.Close()
	}
	cwi.delegate = _ERR_DELEGATE
	return err
}

type multiCoreChecksummedWriter struct {
	delegate         io.Writer
	checksumInterval int
	newHash          func() hash.Hash32
	hash             hash.Hash32
	buffer           *multiCoreChecksummedWriterBuffer
	freeChan         chan *multiCoreChecksummedWriterBuffer
	checksumChan     chan *multiCoreChecksummedWriterBuffer
	writeChan        chan *multiCoreChecksummedWriterBuffer
	doneChan         chan struct{}
	err              error
	closed           bool
}

type multiCoreChecksummedWriterBuffer struct {
	seq int64
	buf []byte
}

// NewMultiCoreChecksummedWriter returns a ChecksummedWriter that delegates
// requests to an underlying io.Writer and embeds checksums of the content at
// given intervals using the hashing function given; it will use multiple cores
// for computing the checksums.
//
// Note that this is generally only faster for large files and reasonably sized
// checksum intervals (e.g. 65532). It can be quite a bit slower on single core
// systems or when using tiny checksum intervals.
func NewMultiCoreChecksummedWriter(delegate io.Writer, checksumInterval int, newHash func() hash.Hash32, cores int) ChecksummedWriter {
	cwi := &multiCoreChecksummedWriter{
		delegate:         delegate,
		checksumInterval: checksumInterval,
		newHash:          newHash,
		freeChan:         make(chan *multiCoreChecksummedWriterBuffer, cores),
		checksumChan:     make(chan *multiCoreChecksummedWriterBuffer, cores),
		writeChan:        make(chan *multiCoreChecksummedWriterBuffer, cores),
		doneChan:         make(chan struct{}),
	}
	for i := 0; i < cores; i++ {
		cwi.freeChan <- &multiCoreChecksummedWriterBuffer{0, make([]byte, 0, checksumInterval+4)}
	}
	cwi.buffer = <-cwi.freeChan
	go cwi.writer()
	for i := 0; i < cores; i++ {
		go cwi.checksummer()
	}
	return cwi
}

func (cwi *multiCoreChecksummedWriter) Write(v []byte) (int, error) {
	var n int
	for len(cwi.buffer.buf)+len(v) >= cwi.checksumInterval {
		n2 := cwi.checksumInterval - len(cwi.buffer.buf)
		cwi.buffer.buf = append(cwi.buffer.buf, v[:n2]...)
		s := cwi.buffer.seq + 1
		cwi.checksumChan <- cwi.buffer
		n += n2
		v = v[n2:]
		cwi.buffer = <-cwi.freeChan
		cwi.buffer.seq = s
	}
	if len(v) > 0 {
		cwi.buffer.buf = append(cwi.buffer.buf, v...)
		n += len(v)
	}
	return n, cwi.err
}

func (cwi *multiCoreChecksummedWriter) Close() error {
	if cwi.closed {
		return cwi.err
	}
	cwi.closed = true
	if len(cwi.buffer.buf) > 0 {
		cwi.checksumChan <- cwi.buffer
	}
	close(cwi.checksumChan)
	for i := 0; i < cap(cwi.checksumChan); i++ {
		<-cwi.doneChan
	}
	cwi.writeChan <- nil
	<-cwi.doneChan
	return cwi.err
}

func (cwi *multiCoreChecksummedWriter) checksummer() {
	for {
		b := <-cwi.checksumChan
		if b == nil {
			break
		}
		h := cwi.newHash()
		h.Write(b.buf)
		b.buf = b.buf[:len(b.buf)+4]
		binary.BigEndian.PutUint32(b.buf[len(b.buf)-4:], h.Sum32())
		cwi.writeChan <- b
	}
	cwi.doneChan <- struct{}{}
}

func (cwi *multiCoreChecksummedWriter) writer() {
	var seq int64
	var lastWasNil bool
	for {
		b := <-cwi.writeChan
		if b == nil {
			if lastWasNil {
				break
			}
			lastWasNil = true
			cwi.writeChan <- nil
			continue
		}
		lastWasNil = false
		if b.seq != seq {
			cwi.writeChan <- b
			continue
		}
		_, cwi.err = cwi.delegate.Write(b.buf)
		b.buf = b.buf[:0]
		cwi.freeChan <- b
		seq++
	}
	if c, ok := cwi.delegate.(io.Closer); ok {
		cwi.err = c.Close()
	}
	cwi.doneChan <- struct{}{}
}

type errDelegate struct {
}

var _ERR_DELEGATE = &errDelegate{}

func (ed *errDelegate) Read(v []byte) (int, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Write(v []byte) (int, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Seek(offset int64, whence int) (int64, error) {
	return 0, fmt.Errorf("closed")
}

func (ed *errDelegate) Close() error {
	return fmt.Errorf("already closed")
}
<|endoftext|>


<commit_before>package termite

import (
	"fmt"
	"os"
	"log"
	"sync"
	"rpc"
	"github.com/hanwen/go-fuse/fuse"
	"strings"
	"path/filepath"
)

type RpcFs struct {
	fuse.DefaultFileSystem
	cache *ContentCache

	client *rpc.Client

	// Roots that we should try to fetch locally.
	localRoots []string

	// Should be acquired before attrMutex if applicable.
	dirMutex    sync.Mutex
	directories map[string]*DirResponse

	attrMutex    sync.RWMutex
	attrResponse map[string]*FileAttr

	fetchMutex sync.Mutex
	fetchCond  sync.Cond
	fetchMap   map[string]bool
}

type UpdateRequest struct {
	Files []FileAttr
}

type UpdateResponse struct {

}

func NewRpcFs(server *rpc.Client, cache *ContentCache) *RpcFs {
	me := &RpcFs{}
	me.client = server
	me.directories = make(map[string]*DirResponse)
	me.cache = cache
	me.attrResponse = make(map[string]*FileAttr)

	me.fetchMap = make(map[string]bool)
	me.fetchCond.L = &me.fetchMutex
	return me
}

func (me *RpcFs) Update(req *UpdateRequest, resp *UpdateResponse) os.Error {
	me.updateFiles(req.Files)
	return nil
}

func (me *RpcFs) updateFiles(files []FileAttr) {
	me.dirMutex.Lock()
	defer me.dirMutex.Unlock()

	me.attrMutex.Lock()
	defer me.attrMutex.Unlock()

	flushDirs := []string{}
	for _, r := range files {
		p := strings.TrimLeft(r.Path, string(filepath.Separator))

		d, _ := filepath.Split(p)
		a, existed := me.attrResponse[p]

		if r.Deletion() || (existed && a.Status != fuse.ENOENT) {
			flushDirs = append(flushDirs, d)
		}

		if r.Deletion() {
			me.attrResponse[p] = nil, false
		} else {
			newVal := r
			me.attrResponse[p] = &newVal
		}
	}

	for _, d := range flushDirs {
		me.directories[d] = nil, false
	}
}

func (me *RpcFs) GetDir(name string) *DirResponse {
	me.dirMutex.Lock()
	defer me.dirMutex.Unlock()

	r, ok := me.directories[name]
	if ok {
		return r
	}

	// TODO - asynchronous.
	// TODO - eliminate cut & paste
	req := &DirRequest{Name: "/" + name}
	rep := &DirResponse{}
	err := me.client.Call("FsServer.ReadDir", req, rep)
	if err != nil {
		log.Println("GetDir error:", err)
		return nil
	}

	me.directories[name] = rep
	return rep
}

func (me *RpcFs) OpenDir(name string) (chan fuse.DirEntry, fuse.Status) {
	r := me.GetDir(name)

	if r == nil {
		return nil, fuse.ENOENT
	}
	c := make(chan fuse.DirEntry, len(r.NameModeMap))
	for k, mode := range r.NameModeMap {
		c <- fuse.DirEntry{
			Name: k,
			Mode: mode,
		}
	}
	close(c)
	return c, fuse.OK
}

func (me *RpcFs) Open(name string, flags uint32) (fuse.File, fuse.Status) {
	if flags&fuse.O_ANYWRITE != 0 {
		return nil, fuse.EPERM
	}
	a := me.getFileAttr(name)
	if a == nil {
		return nil, fuse.ENOENT
	}
	if !a.Status.Ok() {
		return nil, a.Status
	}

	p := me.cache.Path(a.Hash)
	if _, err := os.Lstat(p); fuse.OsErrorToErrno(err) == fuse.ENOENT {
		log.Printf("Fetching contents for file %s", name)
		err = me.FetchHash(a.FileInfo.Size, a.Hash)
		// should return something else?
		if err != nil {
			return nil, fuse.ENOENT
		}
	}

	f, err := os.Open(p)
	if err != nil {
		return nil, fuse.OsErrorToErrno(err)
	}

	return &fuse.LoopbackFile{File: f}, fuse.OK
}

func (me *RpcFs) FetchHash(size int64, hash []byte) os.Error {
	key := string(hash)
	me.fetchMutex.Lock()
	defer me.fetchMutex.Unlock()
	for me.fetchMap[key] && !me.cache.HasHash(hash) {
		me.fetchCond.Wait()
	}

	if me.cache.HasHash(hash) {
		return nil
	}
	me.fetchMap[key] = true
	defer func() {
		me.fetchMap[key] = false, false
		me.fetchCond.Signal()
	}()
	me.fetchMutex.Unlock()
	err := me.fetchOnce(size, hash)
	me.fetchMutex.Lock()
	return err
}

func (me *RpcFs) fetchOnce(size int64, hash []byte) os.Error {
	// TODO - should save in smaller chunks.
	return FetchBetweenContentServers(me.client, "FsServer.FileContent", size, hash,
		me.cache)
}

func (me *RpcFs) Readlink(name string) (string, fuse.Status) {
	a := me.getFileAttr(name)
	if a == nil {
		return "", fuse.ENOENT
	}

	if !a.Status.Ok() {
		return "", a.Status
	}
	if !a.FileInfo.IsSymlink() {
		return "", fuse.EINVAL
	}

	return a.Link, fuse.OK
}

func EncodeFileInfo(fi os.FileInfo) string {
	fi.Atime_ns = 0
	fi.Ino = 0
	fi.Dev = 0
	fi.Name = ""
	return fmt.Sprintf("%v", fi)
}

func (me *RpcFs) getFileAttr(name string) *FileAttr {
	me.attrMutex.RLock()
	result, ok := me.attrResponse[name]
	me.attrMutex.RUnlock()

	if ok {
		return result
	}

	me.attrMutex.Lock()
	defer me.attrMutex.Unlock()
	result, ok = me.attrResponse[name]
	if ok {
		return result
	}

	abs := "/" + name

	req := &AttrRequest{Name: abs}
	rep := &AttrResponse{}
	err := me.client.Call("FsServer.GetAttr", req, rep)
	if err != nil {
		log.Println("GetAttr error:", err)
		return nil
	}
	var wanted *FileAttr
	for _, attr := range rep.Attrs {
		if attr.Content != nil {
			me.cache.Save(attr.Content)
		}

		me.considerSaveLocal(attr)
		fa := attr
		me.attrResponse[strings.TrimLeft(attr.Path, "/")] = &fa
		if attr.Path == abs {
			wanted = &fa
		}
	}
	return wanted
}

func (me *RpcFs) considerSaveLocal(attr FileAttr) {
	absPath := attr.Path
	if !attr.Status.Ok() || !attr.FileInfo.IsRegular() {
		return
	}
	found := false
	for _, root := range me.localRoots {
		if strings.HasPrefix(absPath, root+"/") {
			found = true
		}
	}
	if !found {
		return
	}

	fi, _ := os.Lstat(absPath)
	if fi == nil {
		return
	}
	if EncodeFileInfo(*fi) != EncodeFileInfo(*attr.FileInfo) {
		return
	}

	// Avoid fetching local data; this assumes that most paths
	// will be the same between master and worker.
	me.cache.SaveImmutablePath(absPath)
}

func (me *RpcFs) GetAttr(name string) (*os.FileInfo, fuse.Status) {
	if name == "" {
		return &os.FileInfo{
			Mode: fuse.S_IFDIR | 0755,
		}, fuse.OK
	}

	r := me.getFileAttr(name)
	if r == nil {
		return nil, fuse.ENOENT
	}

	return r.FileInfo, r.Status
}
<commit_msg>In updateFiles(), do not discard ENOENT replies, but store them.<commit_after>package termite

import (
	"fmt"
	"os"
	"log"
	"sync"
	"rpc"
	"github.com/hanwen/go-fuse/fuse"
	"strings"
	"path/filepath"
)

type RpcFs struct {
	fuse.DefaultFileSystem
	cache *ContentCache

	client *rpc.Client

	// Roots that we should try to fetch locally.
	localRoots []string

	// Should be acquired before attrMutex if applicable.
	dirMutex    sync.Mutex
	directories map[string]*DirResponse

	attrMutex    sync.RWMutex
	attrResponse map[string]*FileAttr

	fetchMutex sync.Mutex
	fetchCond  sync.Cond
	fetchMap   map[string]bool
}

type UpdateRequest struct {
	Files []FileAttr
}

type UpdateResponse struct {

}

func NewRpcFs(server *rpc.Client, cache *ContentCache) *RpcFs {
	me := &RpcFs{}
	me.client = server
	me.directories = make(map[string]*DirResponse)
	me.cache = cache
	me.attrResponse = make(map[string]*FileAttr)

	me.fetchMap = make(map[string]bool)
	me.fetchCond.L = &me.fetchMutex
	return me
}

func (me *RpcFs) Update(req *UpdateRequest, resp *UpdateResponse) os.Error {
	me.updateFiles(req.Files)
	return nil
}

func (me *RpcFs) updateFiles(files []FileAttr) {
	me.dirMutex.Lock()
	defer me.dirMutex.Unlock()

	me.attrMutex.Lock()
	defer me.attrMutex.Unlock()

	flushDirs := []string{}
	for _, r := range files {
		p := strings.TrimLeft(r.Path, string(filepath.Separator))

		d, _ := filepath.Split(p)
		a, existed := me.attrResponse[p]

		if r.Deletion() || (existed && a.Status != fuse.ENOENT) {
			flushDirs = append(flushDirs, d)
		}

		newVal := r
		me.attrResponse[p] = &newVal
	}

	for _, d := range flushDirs {
		me.directories[d] = nil, false
	}
}

func (me *RpcFs) GetDir(name string) *DirResponse {
	me.dirMutex.Lock()
	defer me.dirMutex.Unlock()

	r, ok := me.directories[name]
	if ok {
		return r
	}

	// TODO - asynchronous.
	// TODO - eliminate cut & paste
	req := &DirRequest{Name: "/" + name}
	rep := &DirResponse{}
	err := me.client.Call("FsServer.ReadDir", req, rep)
	if err != nil {
		log.Println("GetDir error:", err)
		return nil
	}

	me.directories[name] = rep
	return rep
}

func (me *RpcFs) OpenDir(name string) (chan fuse.DirEntry, fuse.Status) {
	r := me.GetDir(name)

	if r == nil {
		return nil, fuse.ENOENT
	}
	c := make(chan fuse.DirEntry, len(r.NameModeMap))
	for k, mode := range r.NameModeMap {
		c <- fuse.DirEntry{
			Name: k,
			Mode: mode,
		}
	}
	close(c)
	return c, fuse.OK
}

func (me *RpcFs) Open(name string, flags uint32) (fuse.File, fuse.Status) {
	if flags&fuse.O_ANYWRITE != 0 {
		return nil, fuse.EPERM
	}
	a := me.getFileAttr(name)
	if a == nil {
		return nil, fuse.ENOENT
	}
	if !a.Status.Ok() {
		return nil, a.Status
	}

	p := me.cache.Path(a.Hash)
	if _, err := os.Lstat(p); fuse.OsErrorToErrno(err) == fuse.ENOENT {
		log.Printf("Fetching contents for file %s", name)
		err = me.FetchHash(a.FileInfo.Size, a.Hash)
		// should return something else?
		if err != nil {
			return nil, fuse.ENOENT
		}
	}

	f, err := os.Open(p)
	if err != nil {
		return nil, fuse.OsErrorToErrno(err)
	}

	return &fuse.LoopbackFile{File: f}, fuse.OK
}

func (me *RpcFs) FetchHash(size int64, hash []byte) os.Error {
	key := string(hash)
	me.fetchMutex.Lock()
	defer me.fetchMutex.Unlock()
	for me.fetchMap[key] && !me.cache.HasHash(hash) {
		me.fetchCond.Wait()
	}

	if me.cache.HasHash(hash) {
		return nil
	}
	me.fetchMap[key] = true
	defer func() {
		me.fetchMap[key] = false, false
		me.fetchCond.Signal()
	}()
	me.fetchMutex.Unlock()
	err := me.fetchOnce(size, hash)
	me.fetchMutex.Lock()
	return err
}

func (me *RpcFs) fetchOnce(size int64, hash []byte) os.Error {
	// TODO - should save in smaller chunks.
	return FetchBetweenContentServers(me.client, "FsServer.FileContent", size, hash,
		me.cache)
}

func (me *RpcFs) Readlink(name string) (string, fuse.Status) {
	a := me.getFileAttr(name)
	if a == nil {
		return "", fuse.ENOENT
	}

	if !a.Status.Ok() {
		return "", a.Status
	}
	if !a.FileInfo.IsSymlink() {
		return "", fuse.EINVAL
	}

	return a.Link, fuse.OK
}

func EncodeFileInfo(fi os.FileInfo) string {
	fi.Atime_ns = 0
	fi.Ino = 0
	fi.Dev = 0
	fi.Name = ""
	return fmt.Sprintf("%v", fi)
}

func (me *RpcFs) getFileAttr(name string) *FileAttr {
	me.attrMutex.RLock()
	result, ok := me.attrResponse[name]
	me.attrMutex.RUnlock()

	if ok {
		return result
	}

	me.attrMutex.Lock()
	defer me.attrMutex.Unlock()
	result, ok = me.attrResponse[name]
	if ok {
		return result
	}

	abs := "/" + name

	req := &AttrRequest{Name: abs}
	rep := &AttrResponse{}
	err := me.client.Call("FsServer.GetAttr", req, rep)
	if err != nil {
		log.Println("GetAttr error:", err)
		return nil
	}
	var wanted *FileAttr
	for _, attr := range rep.Attrs {
		if attr.Content != nil {
			me.cache.Save(attr.Content)
		}

		me.considerSaveLocal(attr)
		fa := attr
		me.attrResponse[strings.TrimLeft(attr.Path, "/")] = &fa
		if attr.Path == abs {
			wanted = &fa
		}
	}
	return wanted
}

func (me *RpcFs) considerSaveLocal(attr FileAttr) {
	absPath := attr.Path
	if !attr.Status.Ok() || !attr.FileInfo.IsRegular() {
		return
	}
	found := false
	for _, root := range me.localRoots {
		if strings.HasPrefix(absPath, root+"/") {
			found = true
		}
	}
	if !found {
		return
	}

	fi, _ := os.Lstat(absPath)
	if fi == nil {
		return
	}
	if EncodeFileInfo(*fi) != EncodeFileInfo(*attr.FileInfo) {
		return
	}

	// Avoid fetching local data; this assumes that most paths
	// will be the same between master and worker.
	me.cache.SaveImmutablePath(absPath)
}

func (me *RpcFs) GetAttr(name string) (*os.FileInfo, fuse.Status) {
	if name == "" {
		return &os.FileInfo{
			Mode: fuse.S_IFDIR | 0755,
		}, fuse.OK
	}

	r := me.getFileAttr(name)
	if r == nil {
		return nil, fuse.ENOENT
	}

	return r.FileInfo, r.Status
}
<|endoftext|>


<commit_before>package terraform

import (
	"bytes"
	"encoding/json"
	"errors"
	"reflect"

	"github.com/EngineerBetter/concourse-up/util"
	"github.com/asaskevich/govalidator"
)

// InputVars holds all the parameters AWS IAAS needs
type AWSInputVars struct {
	AllowIPs               string
	AvailabilityZone       string
	ConfigBucket           string
	Deployment             string
	HostedZoneID           string
	HostedZoneRecordPrefix string
	MultiAZRDS             bool
	Namespace              string
	NetworkCIDR            string
	PrivateCIDR            string
	Project                string
	PublicCIDR             string
	PublicKey              string
	RDSDefaultDatabaseName string
	RDSInstanceClass       string
	RDSPassword            string
	RDSUsername            string
	Rds1CIDR               string
	Rds2CIDR               string
	Region                 string
	SourceAccessIP         string
	TFStatePath            string
}

// ConfigureTerraform interpolates terraform contents and returns terraform config
func (v *AWSInputVars) ConfigureTerraform(terraformContents string) (string, error) {
	terraformConfig, err := util.RenderTemplate("terraform", terraformContents, v)
	if terraformConfig == nil {
		return "", err
	}
	return string(terraformConfig), err
}

// MetadataStringValue is a terraform output string variable
type MetadataStringValue struct {
	Value string `json:"value"`
}

// Metadata represents output from terraform on AWS or GCP
type AWSOutputs struct {
	ATCPublicIP              MetadataStringValue `json:"atc_public_ip" valid:"required"`
	ATCSecurityGroupID       MetadataStringValue `json:"atc_security_group_id" valid:"required"`
	BlobstoreBucket          MetadataStringValue `json:"blobstore_bucket" valid:"required"`
	BlobstoreSecretAccessKey MetadataStringValue `json:"blobstore_user_secret_access_key" valid:"required"`
	BlobstoreUserAccessKeyID MetadataStringValue `json:"blobstore_user_access_key_id" valid:"required"`
	BoshDBAddress            MetadataStringValue `json:"bosh_db_address" valid:"required"`
	BoshDBPort               MetadataStringValue `json:"bosh_db_port" valid:"required"`
	BoshSecretAccessKey      MetadataStringValue `json:"bosh_user_secret_access_key" valid:"required"`
	BoshUserAccessKeyID      MetadataStringValue `json:"bosh_user_access_key_id" valid:"required"`
	DirectorKeyPair          MetadataStringValue `json:"director_key_pair" valid:"required"`
	DirectorPublicIP         MetadataStringValue `json:"director_public_ip" valid:"required"`
	DirectorSecurityGroupID  MetadataStringValue `json:"director_security_group_id" valid:"required"`
	NatGatewayIP             MetadataStringValue `json:"nat_gateway_ip" valid:"required"`
	PrivateSubnetID          MetadataStringValue `json:"private_subnet_id" valid:"required"`
	PublicSubnetID           MetadataStringValue `json:"public_subnet_id" valid:"required"`
	SourceAccessIP           MetadataStringValue `json:"source_access_ip"`
	VMsSecurityGroupID       MetadataStringValue `json:"vms_security_group_id" valid:"required"`
	VPCID                    MetadataStringValue `json:"vpc_id" valid:"required"`
}

// AssertValid returns an error if the struct contains any missing fields
func (outputs *AWSOutputs) AssertValid() error {
	_, err := govalidator.ValidateStruct(outputs)
	return err
}

// Init populates outputs struct with values from the buffer
func (outputs *AWSOutputs) Init(buffer *bytes.Buffer) error {
	if err := json.NewDecoder(buffer).Decode(&outputs); err != nil {
		return err
	}

	return nil
}

// Get returns a the specified value from the outputs struct
func (outputs *AWSOutputs) Get(key string) (string, error) {
	reflectValue := reflect.ValueOf(outputs)
	reflectStruct := reflectValue.Elem()
	value := reflectStruct.FieldByName(key)
	if !value.IsValid() {
		return "", errors.New(key + " key not found")
	}

	return value.FieldByName("Value").String(), nil
}
<commit_msg>remove leftover MultiAZRDS field that was not used<commit_after>package terraform

import (
	"bytes"
	"encoding/json"
	"errors"
	"reflect"

	"github.com/EngineerBetter/concourse-up/util"
	"github.com/asaskevich/govalidator"
)

// InputVars holds all the parameters AWS IAAS needs
type AWSInputVars struct {
	AllowIPs               string
	AvailabilityZone       string
	ConfigBucket           string
	Deployment             string
	HostedZoneID           string
	HostedZoneRecordPrefix string
	Namespace              string
	NetworkCIDR            string
	PrivateCIDR            string
	Project                string
	PublicCIDR             string
	PublicKey              string
	RDSDefaultDatabaseName string
	RDSInstanceClass       string
	RDSPassword            string
	RDSUsername            string
	Rds1CIDR               string
	Rds2CIDR               string
	Region                 string
	SourceAccessIP         string
	TFStatePath            string
}

// ConfigureTerraform interpolates terraform contents and returns terraform config
func (v *AWSInputVars) ConfigureTerraform(terraformContents string) (string, error) {
	terraformConfig, err := util.RenderTemplate("terraform", terraformContents, v)
	if terraformConfig == nil {
		return "", err
	}
	return string(terraformConfig), err
}

// MetadataStringValue is a terraform output string variable
type MetadataStringValue struct {
	Value string `json:"value"`
}

// Metadata represents output from terraform on AWS or GCP
type AWSOutputs struct {
	ATCPublicIP              MetadataStringValue `json:"atc_public_ip" valid:"required"`
	ATCSecurityGroupID       MetadataStringValue `json:"atc_security_group_id" valid:"required"`
	BlobstoreBucket          MetadataStringValue `json:"blobstore_bucket" valid:"required"`
	BlobstoreSecretAccessKey MetadataStringValue `json:"blobstore_user_secret_access_key" valid:"required"`
	BlobstoreUserAccessKeyID MetadataStringValue `json:"blobstore_user_access_key_id" valid:"required"`
	BoshDBAddress            MetadataStringValue `json:"bosh_db_address" valid:"required"`
	BoshDBPort               MetadataStringValue `json:"bosh_db_port" valid:"required"`
	BoshSecretAccessKey      MetadataStringValue `json:"bosh_user_secret_access_key" valid:"required"`
	BoshUserAccessKeyID      MetadataStringValue `json:"bosh_user_access_key_id" valid:"required"`
	DirectorKeyPair          MetadataStringValue `json:"director_key_pair" valid:"required"`
	DirectorPublicIP         MetadataStringValue `json:"director_public_ip" valid:"required"`
	DirectorSecurityGroupID  MetadataStringValue `json:"director_security_group_id" valid:"required"`
	NatGatewayIP             MetadataStringValue `json:"nat_gateway_ip" valid:"required"`
	PrivateSubnetID          MetadataStringValue `json:"private_subnet_id" valid:"required"`
	PublicSubnetID           MetadataStringValue `json:"public_subnet_id" valid:"required"`
	SourceAccessIP           MetadataStringValue `json:"source_access_ip"`
	VMsSecurityGroupID       MetadataStringValue `json:"vms_security_group_id" valid:"required"`
	VPCID                    MetadataStringValue `json:"vpc_id" valid:"required"`
}

// AssertValid returns an error if the struct contains any missing fields
func (outputs *AWSOutputs) AssertValid() error {
	_, err := govalidator.ValidateStruct(outputs)
	return err
}

// Init populates outputs struct with values from the buffer
func (outputs *AWSOutputs) Init(buffer *bytes.Buffer) error {
	if err := json.NewDecoder(buffer).Decode(&outputs); err != nil {
		return err
	}

	return nil
}

// Get returns a the specified value from the outputs struct
func (outputs *AWSOutputs) Get(key string) (string, error) {
	reflectValue := reflect.ValueOf(outputs)
	reflectStruct := reflectValue.Elem()
	value := reflectStruct.FieldByName(key)
	if !value.IsValid() {
		return "", errors.New(key + " key not found")
	}

	return value.FieldByName("Value").String(), nil
}
<|endoftext|>


<commit_before>package terse

import (
	"bufio"
	"fmt"
	"strings"
)

type rawTree struct {
	Children []*rawNode
}

func (rt rawTree) String() string {
	switch len(rt.Children) {
	case 0:
		return ""
	case 1:
		return rt.Children[0].Print("")
	}
	content := rt.Children[0].Print("")
	for _, node := range rt.Children[1:] {
		content += "\n" + node.Print("")
	}
	return content
}

type rawNode struct {
	Code     string
	Children []*rawNode
	Pos      int
}

func (rn rawNode) Print(prefix string) string {
	content := prefix + rn.Code
	for _, node := range rn.Children {
		content += "\n" + node.Print(prefix+"  ")
	}
	return content
}

func scan(src string) rawTree {
	s := bufio.NewScanner(strings.NewReader(src))
	rt := rawTree{}
	for s.Scan() {
		line := s.Text()
		if blankLine(line) {
			continue
		}
		if unindentedLine(line) || len(rt.Children) == 0 {
			rt.Children = append(rt.Children, &rawNode{Code: line})
			continue
		}
		current := rt.Children[len(rt.Children)-1]
		line = unindentLine(line)
		fmt.Println(line)
		for len(current.Children) != 0 && !unindentedLine(line) {
			current = current.Children[len(current.Children)-1]
			line = unindentLine(line)
			fmt.Println(line)
		}
		current.Children = append(current.Children, &rawNode{Code: line})
	}
	return rt
}

func blankLine(line string) bool {
	return strings.TrimSpace(line) == ""
}

func unindentedLine(line string) bool {
	return line[0:2] != "  " && line[0] != '\t'
}

func unindentLine(line string) string {
	if line[0:2] == "  " {
		return line[2:]
	}
	if line[0] == '/' {
		return line[1:]
	}
	return line
}
<commit_msg>Remove debugging code<commit_after>package terse

import (
	"bufio"
	"strings"
)

type rawTree struct {
	Children []*rawNode
}

func (rt rawTree) String() string {
	switch len(rt.Children) {
	case 0:
		return ""
	case 1:
		return rt.Children[0].Print("")
	}
	content := rt.Children[0].Print("")
	for _, node := range rt.Children[1:] {
		content += "\n" + node.Print("")
	}
	return content
}

type rawNode struct {
	Code     string
	Children []*rawNode
	Pos      int
}

func (rn rawNode) Print(prefix string) string {
	content := prefix + rn.Code
	for _, node := range rn.Children {
		content += "\n" + node.Print(prefix+"  ")
	}
	return content
}

func scan(src string) rawTree {
	s := bufio.NewScanner(strings.NewReader(src))
	rt := rawTree{}
	for s.Scan() {
		line := s.Text()
		if blankLine(line) {
			continue
		}
		if unindentedLine(line) || len(rt.Children) == 0 {
			rt.Children = append(rt.Children, &rawNode{Code: line})
			continue
		}
		current := rt.Children[len(rt.Children)-1]
		line = unindentLine(line)
		for len(current.Children) != 0 && !unindentedLine(line) {
			current = current.Children[len(current.Children)-1]
			line = unindentLine(line)
		}
		current.Children = append(current.Children, &rawNode{Code: line})
	}
	return rt
}

func blankLine(line string) bool {
	return strings.TrimSpace(line) == ""
}

func unindentedLine(line string) bool {
	return line[0:2] != "  " && line[0] != '\t'
}

func unindentLine(line string) string {
	if line[0:2] == "  " {
		return line[2:]
	}
	if line[0] == '/' {
		return line[1:]
	}
	return line
}
<|endoftext|>


<commit_before><commit_msg>Added a way of removing the cluster ID from a context<commit_after><|endoftext|>


<commit_before>package circularqueue

// Queue represents the underlying job queue
type Queue interface {
  Pop() (Job,error)
}

// Conn is what all circular queues require
type Conn interface {
	Do(commandName string, args... interface{}) (interface{}, error)
}

// ConnQueue represents the underlying job queue
type ConnQueue struct {
	conn Conn
}

// New constructs a new instance of the circular queue
func New(conn Conn) (*ConnQueue) {
	return &ConnQueue{conn: conn}
}

// Pop returns a job from the queue and auto-readds it
func (queue *ConnQueue) Pop() (Job,error) {
  result, err := queue.conn.Do("RPOPLPUSH", "circular-job-queue", "circular-job-queue")
  if err != nil {
    return nil, err
  }

  key := string(result.([]uint8))
  return NewJob(key), nil
}
<commit_msg>BRPOPLPUSH<commit_after>package circularqueue

// Queue represents the underlying job queue
type Queue interface {
  Pop() (Job,error)
}

// Conn is what all circular queues require
type Conn interface {
	Do(commandName string, args... interface{}) (interface{}, error)
}

// ConnQueue represents the underlying job queue
type ConnQueue struct {
	conn Conn
}

// New constructs a new instance of the circular queue
func New(conn Conn) (*ConnQueue) {
	return &ConnQueue{conn: conn}
}

// Pop returns a job from the queue and auto-readds it
func (queue *ConnQueue) Pop() (Job,error) {
  result, err := queue.conn.Do("BRPOPLPUSH", "circular-job-queue", "circular-job-queue", 1000)
  if err != nil {
    return nil, err
  }

  key := string(result.([]uint8))
  return NewJob(key), nil
}
<|endoftext|>