<commit_before>package main

import (
	"bytes"
	"encoding/gob"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"time"
	"unicode/utf8"
)

// Buffer stores the text for files that are loaded into the text editor
// It uses a rope to efficiently store the string and contains some
// simple functions for saving and wrapper functions for modifying the rope
type Buffer struct {
	// The eventhandler for undo/redo
	*EventHandler
	// This stores all the text in the buffer as an array of lines
	*LineArray

	Cursor Cursor

	// Path to the file on disk
	Path string
	// Name of the buffer on the status line
	Name string

	// Whether or not the buffer has been modified since it was opened
	IsModified bool

	// Stores the last modification time of the file the buffer is pointing to
	ModTime time.Time

	NumLines int

	// Syntax highlighting rules
	rules []SyntaxRule

	// Buffer local settings
	Settings map[string]interface{}
}

// The SerializedBuffer holds the types that get serialized when a buffer is saved
// These are used for the savecursor and saveundo options
type SerializedBuffer struct {
	EventHandler *EventHandler
	Cursor       Cursor
	ModTime      time.Time
}

// NewBuffer creates a new buffer from `txt` with path and name `path`
func NewBuffer(txt []byte, path string) *Buffer {
	b := new(Buffer)
	b.LineArray = NewLineArray(txt)

	b.Settings = DefaultLocalSettings()
	for k, v := range globalSettings {
		if _, ok := b.Settings[k]; ok {
			b.Settings[k] = v
		}
	}

	b.Path = path
	b.Name = path

	// If the file doesn't have a path to disk then we give it no name
	if path == "" {
		b.Name = "No name"
	}

	// The last time this file was modified
	b.ModTime, _ = GetModTime(b.Path)

	b.EventHandler = NewEventHandler(b)

	b.Update()
	b.FindFileType()
	b.UpdateRules()

	if _, err := os.Stat(configDir + "/buffers/"); os.IsNotExist(err) {
		os.Mkdir(configDir+"/buffers/", os.ModePerm)
	}

	// Put the cursor at the first spot
	b.Cursor = Cursor{
		Loc: Loc{
			X: 0,
			Y: 0,
		},
		buf: b,
	}

	InitLocalSettings(b)

	if b.Settings["savecursor"].(bool) || b.Settings["saveundo"].(bool) {
		// If either savecursor or saveundo is turned on, we need to load the serialized information
		// from ~/.config/micro/buffers
		absPath, _ := filepath.Abs(b.Path)
		file, err := os.Open(configDir + "/buffers/" + EscapePath(absPath))
		if err == nil {
			var buffer SerializedBuffer
			decoder := gob.NewDecoder(file)
			gob.Register(TextEvent{})
			err = decoder.Decode(&buffer)
			if err != nil {
				TermMessage(err.Error(), "\n", "You may want to remove the files in ~/.config/micro/buffers (these files store the information for the 'saveundo' and 'savecursor' options) if this problem persists.")
			}
			if b.Settings["savecursor"].(bool) {
				b.Cursor = buffer.Cursor
				b.Cursor.buf = b
				b.Cursor.Relocate()
			}

			if b.Settings["saveundo"].(bool) {
				// We should only use last time's eventhandler if the file wasn't by someone else in the meantime
				if b.ModTime == buffer.ModTime {
					b.EventHandler = buffer.EventHandler
					b.EventHandler.buf = b
				}
			}
		}
		file.Close()
	}

	return b
}

// UpdateRules updates the syntax rules and filetype for this buffer
// This is called when the colorscheme changes
func (b *Buffer) UpdateRules() {
	b.rules = GetRules(b)
}

// FindFileType identifies this buffer's filetype based on the extension or header
func (b *Buffer) FindFileType() {
	b.Settings["filetype"] = FindFileType(b)
}

// FileType returns the buffer's filetype
func (b *Buffer) FileType() string {
	return b.Settings["filetype"].(string)
}

// CheckModTime makes sure that the file this buffer points to hasn't been updated
// by an external program since it was last read
// If it has, we ask the user if they would like to reload the file
func (b *Buffer) CheckModTime() {
	modTime, ok := GetModTime(b.Path)
	if ok {
		if modTime != b.ModTime {
			choice, canceled := messenger.YesNoPrompt("The file has changed since it was last read. Reload file? (y,n)")
			messenger.Reset()
			messenger.Clear()
			if !choice || canceled {
				// Don't load new changes -- do nothing
				b.ModTime, _ = GetModTime(b.Path)
			} else {
				// Load new changes
				b.ReOpen()
			}
		}
	}
}

// ReOpen reloads the current buffer from disk
func (b *Buffer) ReOpen() {
	data, err := ioutil.ReadFile(b.Path)
	txt := string(data)

	if err != nil {
		messenger.Error(err.Error())
		return
	}
	b.EventHandler.ApplyDiff(txt)

	b.ModTime, _ = GetModTime(b.Path)
	b.IsModified = false
	b.Update()
	b.Cursor.Relocate()
}

// Update fetches the string from the rope and updates the `text` and `lines` in the buffer
func (b *Buffer) Update() {
	b.NumLines = len(b.lines)
}

// Save saves the buffer to its default path
func (b *Buffer) Save() error {
	return b.SaveAs(b.Path)
}

// SaveWithSudo saves the buffer to the default path with sudo
func (b *Buffer) SaveWithSudo() error {
	return b.SaveAsWithSudo(b.Path)
}

// Serialize serializes the buffer to configDir/buffers
func (b *Buffer) Serialize() error {
	if b.Settings["savecursor"].(bool) || b.Settings["saveundo"].(bool) {
		absPath, _ := filepath.Abs(b.Path)
		file, err := os.Create(configDir + "/buffers/" + EscapePath(absPath))
		if err == nil {
			enc := gob.NewEncoder(file)
			gob.Register(TextEvent{})
			err = enc.Encode(SerializedBuffer{
				b.EventHandler,
				b.Cursor,
				b.ModTime,
			})
		}
		file.Close()
		return err
	}
	return nil
}

// SaveAs saves the buffer to a specified path (filename), creating the file if it does not exist
func (b *Buffer) SaveAs(filename string) error {
	b.UpdateRules()
	b.Name = filename
	b.Path = filename
	data := []byte(b.String())
	err := ioutil.WriteFile(filename, data, 0644)
	if err == nil {
		b.IsModified = false
		b.ModTime, _ = GetModTime(filename)
		return b.Serialize()
	}
	return err
}

// SaveAsWithSudo is the same as SaveAs except it uses a neat trick
// with tee to use sudo so the user doesn't have to reopen micro with sudo
func (b *Buffer) SaveAsWithSudo(filename string) error {
	b.UpdateRules()
	b.Name = filename
	b.Path = filename

	// The user may have already used sudo in which case we won't need the password
	// It's a bit nicer for them if they don't have to enter the password every time
	_, err := RunShellCommand("sudo -v")
	needPassword := err != nil

	// If we need the password, we have to close the screen and ask using the shell
	if needPassword {
		// Shut down the screen because we're going to interact directly with the shell
		screen.Fini()
		screen = nil
	}

	// Set up everything for the command
	cmd := exec.Command("sudo", "tee", filename)
	cmd.Stdin = bytes.NewBufferString(b.String())

	// This is a trap for Ctrl-C so that it doesn't kill micro
	// Instead we trap Ctrl-C to kill the program we're running
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for range c {
			cmd.Process.Kill()
		}
	}()

	// Start the command
	cmd.Start()
	err = cmd.Wait()

	// If we needed the password, we closed the screen, so we have to initialize it again
	if needPassword {
		// Start the screen back up
		InitScreen()
	}
	if err == nil {
		b.IsModified = false
		b.ModTime, _ = GetModTime(filename)
		b.Serialize()
	}
	return err
}

func (b *Buffer) insert(pos Loc, value []byte) {
	b.IsModified = true
	b.LineArray.insert(pos, value)
	b.Update()
}
func (b *Buffer) remove(start, end Loc) string {
	b.IsModified = true
	sub := b.LineArray.remove(start, end)
	b.Update()
	return sub
}

// Start returns the location of the first character in the buffer
func (b *Buffer) Start() Loc {
	return Loc{0, 0}
}

// End returns the location of the last character in the buffer
func (b *Buffer) End() Loc {
	return Loc{utf8.RuneCount(b.lines[b.NumLines-1]), b.NumLines - 1}
}

// Line returns a single line
func (b *Buffer) Line(n int) string {
	return string(b.lines[n])
}

// Lines returns an array of strings containing the lines from start to end
func (b *Buffer) Lines(start, end int) []string {
	lines := b.lines[start:end]
	var slice []string
	for _, line := range lines {
		slice = append(slice, string(line))
	}
	return slice
}

// Len gives the length of the buffer
func (b *Buffer) Len() int {
	return Count(b.String())
}
<commit_msg>Check for filetype after saving an untitled buffer<commit_after>package main

import (
	"bytes"
	"encoding/gob"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"time"
	"unicode/utf8"
)

// Buffer stores the text for files that are loaded into the text editor
// It uses a rope to efficiently store the string and contains some
// simple functions for saving and wrapper functions for modifying the rope
type Buffer struct {
	// The eventhandler for undo/redo
	*EventHandler
	// This stores all the text in the buffer as an array of lines
	*LineArray

	Cursor Cursor

	// Path to the file on disk
	Path string
	// Name of the buffer on the status line
	Name string

	// Whether or not the buffer has been modified since it was opened
	IsModified bool

	// Stores the last modification time of the file the buffer is pointing to
	ModTime time.Time

	NumLines int

	// Syntax highlighting rules
	rules []SyntaxRule

	// Buffer local settings
	Settings map[string]interface{}
}

// The SerializedBuffer holds the types that get serialized when a buffer is saved
// These are used for the savecursor and saveundo options
type SerializedBuffer struct {
	EventHandler *EventHandler
	Cursor       Cursor
	ModTime      time.Time
}

// NewBuffer creates a new buffer from `txt` with path and name `path`
func NewBuffer(txt []byte, path string) *Buffer {
	b := new(Buffer)
	b.LineArray = NewLineArray(txt)

	b.Settings = DefaultLocalSettings()
	for k, v := range globalSettings {
		if _, ok := b.Settings[k]; ok {
			b.Settings[k] = v
		}
	}

	b.Path = path
	b.Name = path

	// If the file doesn't have a path to disk then we give it no name
	if path == "" {
		b.Name = "No name"
	}

	// The last time this file was modified
	b.ModTime, _ = GetModTime(b.Path)

	b.EventHandler = NewEventHandler(b)

	b.Update()
	b.FindFileType()
	b.UpdateRules()

	if _, err := os.Stat(configDir + "/buffers/"); os.IsNotExist(err) {
		os.Mkdir(configDir+"/buffers/", os.ModePerm)
	}

	// Put the cursor at the first spot
	b.Cursor = Cursor{
		Loc: Loc{
			X: 0,
			Y: 0,
		},
		buf: b,
	}

	InitLocalSettings(b)

	if b.Settings["savecursor"].(bool) || b.Settings["saveundo"].(bool) {
		// If either savecursor or saveundo is turned on, we need to load the serialized information
		// from ~/.config/micro/buffers
		absPath, _ := filepath.Abs(b.Path)
		file, err := os.Open(configDir + "/buffers/" + EscapePath(absPath))
		if err == nil {
			var buffer SerializedBuffer
			decoder := gob.NewDecoder(file)
			gob.Register(TextEvent{})
			err = decoder.Decode(&buffer)
			if err != nil {
				TermMessage(err.Error(), "\n", "You may want to remove the files in ~/.config/micro/buffers (these files store the information for the 'saveundo' and 'savecursor' options) if this problem persists.")
			}
			if b.Settings["savecursor"].(bool) {
				b.Cursor = buffer.Cursor
				b.Cursor.buf = b
				b.Cursor.Relocate()
			}

			if b.Settings["saveundo"].(bool) {
				// We should only use last time's eventhandler if the file wasn't by someone else in the meantime
				if b.ModTime == buffer.ModTime {
					b.EventHandler = buffer.EventHandler
					b.EventHandler.buf = b
				}
			}
		}
		file.Close()
	}

	return b
}

// UpdateRules updates the syntax rules and filetype for this buffer
// This is called when the colorscheme changes
func (b *Buffer) UpdateRules() {
	b.rules = GetRules(b)
}

// FindFileType identifies this buffer's filetype based on the extension or header
func (b *Buffer) FindFileType() {
	b.Settings["filetype"] = FindFileType(b)
}

// FileType returns the buffer's filetype
func (b *Buffer) FileType() string {
	return b.Settings["filetype"].(string)
}

// CheckModTime makes sure that the file this buffer points to hasn't been updated
// by an external program since it was last read
// If it has, we ask the user if they would like to reload the file
func (b *Buffer) CheckModTime() {
	modTime, ok := GetModTime(b.Path)
	if ok {
		if modTime != b.ModTime {
			choice, canceled := messenger.YesNoPrompt("The file has changed since it was last read. Reload file? (y,n)")
			messenger.Reset()
			messenger.Clear()
			if !choice || canceled {
				// Don't load new changes -- do nothing
				b.ModTime, _ = GetModTime(b.Path)
			} else {
				// Load new changes
				b.ReOpen()
			}
		}
	}
}

// ReOpen reloads the current buffer from disk
func (b *Buffer) ReOpen() {
	data, err := ioutil.ReadFile(b.Path)
	txt := string(data)

	if err != nil {
		messenger.Error(err.Error())
		return
	}
	b.EventHandler.ApplyDiff(txt)

	b.ModTime, _ = GetModTime(b.Path)
	b.IsModified = false
	b.Update()
	b.Cursor.Relocate()
}

// Update fetches the string from the rope and updates the `text` and `lines` in the buffer
func (b *Buffer) Update() {
	b.NumLines = len(b.lines)
}

// Save saves the buffer to its default path
func (b *Buffer) Save() error {
	return b.SaveAs(b.Path)
}

// SaveWithSudo saves the buffer to the default path with sudo
func (b *Buffer) SaveWithSudo() error {
	return b.SaveAsWithSudo(b.Path)
}

// Serialize serializes the buffer to configDir/buffers
func (b *Buffer) Serialize() error {
	if b.Settings["savecursor"].(bool) || b.Settings["saveundo"].(bool) {
		absPath, _ := filepath.Abs(b.Path)
		file, err := os.Create(configDir + "/buffers/" + EscapePath(absPath))
		if err == nil {
			enc := gob.NewEncoder(file)
			gob.Register(TextEvent{})
			err = enc.Encode(SerializedBuffer{
				b.EventHandler,
				b.Cursor,
				b.ModTime,
			})
		}
		file.Close()
		return err
	}
	return nil
}

// SaveAs saves the buffer to a specified path (filename), creating the file if it does not exist
func (b *Buffer) SaveAs(filename string) error {
	b.FindFileType()
	b.UpdateRules()
	b.Name = filename
	b.Path = filename
	data := []byte(b.String())
	err := ioutil.WriteFile(filename, data, 0644)
	if err == nil {
		b.IsModified = false
		b.ModTime, _ = GetModTime(filename)
		return b.Serialize()
	}
	return err
}

// SaveAsWithSudo is the same as SaveAs except it uses a neat trick
// with tee to use sudo so the user doesn't have to reopen micro with sudo
func (b *Buffer) SaveAsWithSudo(filename string) error {
	b.FindFileType()
	b.UpdateRules()
	b.Name = filename
	b.Path = filename

	// The user may have already used sudo in which case we won't need the password
	// It's a bit nicer for them if they don't have to enter the password every time
	_, err := RunShellCommand("sudo -v")
	needPassword := err != nil

	// If we need the password, we have to close the screen and ask using the shell
	if needPassword {
		// Shut down the screen because we're going to interact directly with the shell
		screen.Fini()
		screen = nil
	}

	// Set up everything for the command
	cmd := exec.Command("sudo", "tee", filename)
	cmd.Stdin = bytes.NewBufferString(b.String())

	// This is a trap for Ctrl-C so that it doesn't kill micro
	// Instead we trap Ctrl-C to kill the program we're running
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for range c {
			cmd.Process.Kill()
		}
	}()

	// Start the command
	cmd.Start()
	err = cmd.Wait()

	// If we needed the password, we closed the screen, so we have to initialize it again
	if needPassword {
		// Start the screen back up
		InitScreen()
	}
	if err == nil {
		b.IsModified = false
		b.ModTime, _ = GetModTime(filename)
		b.Serialize()
	}
	return err
}

func (b *Buffer) insert(pos Loc, value []byte) {
	b.IsModified = true
	b.LineArray.insert(pos, value)
	b.Update()
}
func (b *Buffer) remove(start, end Loc) string {
	b.IsModified = true
	sub := b.LineArray.remove(start, end)
	b.Update()
	return sub
}

// Start returns the location of the first character in the buffer
func (b *Buffer) Start() Loc {
	return Loc{0, 0}
}

// End returns the location of the last character in the buffer
func (b *Buffer) End() Loc {
	return Loc{utf8.RuneCount(b.lines[b.NumLines-1]), b.NumLines - 1}
}

// Line returns a single line
func (b *Buffer) Line(n int) string {
	return string(b.lines[n])
}

// Lines returns an array of strings containing the lines from start to end
func (b *Buffer) Lines(start, end int) []string {
	lines := b.lines[start:end]
	var slice []string
	for _, line := range lines {
		slice = append(slice, string(line))
	}
	return slice
}

// Len gives the length of the buffer
func (b *Buffer) Len() int {
	return Count(b.String())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/freman/go-aurora"

	"github.com/BurntSushi/toml"
	log "github.com/Sirupsen/logrus"
	"github.com/matryer/try"
	"github.com/tarm/serial"
)

type result struct {
	Time                time.Time
	Address             byte
	BoosterTemperature  float32
	InverterTemperature float32
	Frequency           float32
	GridVoltage         float32
	GridCurrent         float32
	GridPower           float32
	GridRunTime         duration
	Input1Voltage       float32
	Input1Current       float32
	Input2Voltage       float32
	Input2Current       float32
	Joules              uint16
	DailyEnergy         uint32
	WeeklyEnergy        uint32
	MonthlyEnergy       uint32
	YearlyEnergy        uint32
	TotalEnergy         uint32
	TotalRunTime        duration
	SerialNumber        string
}

type results struct {
	sync.RWMutex
	Results map[string]*result
}

type serialConfig struct {
	serial.Config
	ReadTimeout duration
}

type duration struct {
	time.Duration
}

func (o *serialConfig) Normalise() *serial.Config {
	o.Config.ReadTimeout = o.ReadTimeout.Duration
	return &o.Config
}

func (d *duration) UnmarshalText(text []byte) (err error) {
	d.Duration, err = time.ParseDuration(string(text))
	return
}

func (d duration) MarshalJSON() ([]byte, error) {
	return json.Marshal(int64(d.Duration.Seconds()))
}

func withDeadline(deadline time.Duration, f func() error) error {
	c := make(chan error, 1)
	defer close(c)
	go func() {
		c <- try.Do(func(attempt int) (bool, error) {
			time.Sleep(time.Duration(attempt) * time.Millisecond)
			err := f()
			return attempt < 3, err
		})
	}()
	select {
	case err := <-c:
		log.WithError(err).Error("Call to f() failed with error")
		return err
	case <-time.After(deadline):
		log.WithField("deadline", deadline).Warning("Timeout while reading from inverter")
		return errors.New("Timeout while waiting for operation to complete")
	}
}

type configStruct struct {
	Name          string
	Comms         serialConfig
	UpdateRate    duration
	Deadline      duration
	UnitAddresses []byte
}

func main() {
	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		log.Fatal(err)
	}

	config := struct {
		LogPath    string
		UpdateRate duration
		Deadline   duration
		Listen     string
		Devices    []configStruct
	}{
		LogPath: filepath.Join(dir, "main.log"),
		UpdateRate: duration{
			Duration: time.Minute,
		},
		Deadline: duration{
			Duration: 5 * time.Second,
		},
		Listen: ":8080",
	}

	fConfig := flag.String("config", "config.toml", "Path to the configuration file")
	flag.Parse()

	_, err = toml.DecodeFile(*fConfig, &config)
	if err != nil {
		log.Fatalf("Unable to parse configuration file due to %v.", err)
	}

	f, err := os.OpenFile(config.LogPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		log.Fatalf("error opening file: %v", err)
	}
	defer f.Close()

	log.Infof("Logging to %s, there should be no further output", config.LogPath)
	log.SetFormatter(&log.JSONFormatter{})
	log.SetOutput(f)
	log.SetLevel(log.DebugLevel)

	log.WithField("config", config).Debug("Configuration")

	buffer := results{
		Results: map[string]*result{},
	}

	for _, device := range config.Devices {
		go func(device configStruct) {
			logger := log.WithField("coms", device.Comms.Name)
			deadline := device.Deadline.Duration
			if deadline == 0 {
				deadline = config.Deadline.Duration
			}

			updateRate := device.UpdateRate.Duration
			if updateRate == 0 {
				updateRate = config.UpdateRate.Duration
			}

			port, err := serial.OpenPort(device.Comms.Normalise())
			if err != nil {
				log.WithError(err).Fatal("Startup error: Unable to open serial port")
			}

			inverter := &aurora.Inverter{
				Conn: port,
			}

			for _, address := range device.UnitAddresses {
				logger := logger.WithField("address", address)
				name := fmt.Sprintf("%s::%d", device.Comms.Name, address)
				inverter.Address = address
				buffer.Results[name] = &result{
					Address: address,
				}

				err := withDeadline(deadline, func() (err error) {
					buffer.Results[name].SerialNumber, err = inverter.SerialNumber()
					return
				})

				if err != nil {
					logger.WithError(err).Fatal("Startup error: Unable to communicate with inverter")
				}
			}

			ticker := time.NewTicker(updateRate)
			now := time.Now()
			for {
				for _, address := range device.UnitAddresses {
					name := fmt.Sprintf("%s::%d", device.Comms.Name, address)
					buffer.RLock()
					logger := logger.WithFields(log.Fields{
						"address": address,
						"serial":  buffer.Results[name].SerialNumber,
					})
					inverter.Address = address
					r := &result{
						Address:      address,
						SerialNumber: buffer.Results[name].SerialNumber,
						Time:         now,
					}
					buffer.RUnlock()

					err := withDeadline(deadline, func() error {
						var err error
						if r.BoosterTemperature, err = inverter.BoosterTemperature(); err != nil {
							logger.WithError(err).Warning("Unable to read BoosterTemperature")
							return err
						}
						if r.InverterTemperature, err = inverter.InverterTemperature(); err != nil {
							logger.WithError(err).Warning("Unable to read InverterTemperature")
							return err
						}
						if r.Frequency, err = inverter.Frequency(); err != nil {
							logger.WithError(err).Warning("Unable to read Frequency")
							return err
						}
						if r.GridVoltage, err = inverter.GridVoltage(); err != nil {
							logger.WithError(err).Warning("Unable to read GridVoltage")
							return err
						}
						if r.GridCurrent, err = inverter.GridCurrent(); err != nil {
							logger.WithError(err).Warning("Unable to read GridCurrent")
							return err
						}
						if r.GridPower, err = inverter.GridPower(); err != nil {
							logger.WithError(err).Warning("Unable to read GridPower")
							return err
						}
						if r.GridRunTime.Duration, err = inverter.GridRunTime(); err != nil {
							logger.WithError(err).Warning("Unable to read GridRunTime")
							return err
						}
						if r.Input1Voltage, err = inverter.Input1Voltage(); err != nil {
							logger.WithError(err).Warning("Unable to read Input1Voltage")
							return err
						}
						if r.Input1Current, err = inverter.Input1Current(); err != nil {
							logger.WithError(err).Warning("Unable to read Input1Current")
							return err
						}
						if r.Input2Voltage, err = inverter.Input2Voltage(); err != nil {
							logger.WithError(err).Warning("Unable to read Input2Voltage")
							return err
						}
						if r.Input2Current, err = inverter.Input2Current(); err != nil {
							logger.WithError(err).Warning("Unable to read Input2Current")
							return err
						}
						if r.Joules, err = inverter.Joules(); err != nil {
							logger.WithError(err).Warning("Unable to read Joules")
							return err
						}
						if r.DailyEnergy, err = inverter.DailyEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read DailyEnergy")
							return err
						}
						if r.WeeklyEnergy, err = inverter.WeeklyEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read WeeklyEnergy")
							return err
						}
						if r.MonthlyEnergy, err = inverter.MonthlyEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read MonthlyEnergy")
							return err
						}
						if r.YearlyEnergy, err = inverter.YearlyEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read YearlyEnergy")
							return err
						}
						if r.TotalEnergy, err = inverter.TotalEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read TotalEnergy")
							return err
						}
						r.TotalRunTime.Duration, err = inverter.TotalRunTime()
						if err != nil {
							logger.WithError(err).Warning("Unable to read TotalRunTime")
						}
						return err
					})

					if err == nil {
						buffer.Lock()
						buffer.Results[name] = r
						buffer.Unlock()
					}
				}
				now = <-ticker.C
			}
		}(device)
	}

	http.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		logger := log.WithField("remoteaddr", r.RemoteAddr)
		logger.Info("GET /json")
		buffer.RLock()
		defer buffer.RUnlock()
		js, err := json.Marshal(buffer.Results)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Cache-Control", "no-cache")
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Content-Type", "application/json")
		w.Write(js)
	})

	log.Fatal(http.ListenAndServe(config.Listen, nil))
}
<commit_msg>Error log doesn't log err?<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/freman/go-aurora"

	"github.com/BurntSushi/toml"
	log "github.com/Sirupsen/logrus"
	"github.com/matryer/try"
	"github.com/tarm/serial"
)

type result struct {
	Time                time.Time
	Address             byte
	BoosterTemperature  float32
	InverterTemperature float32
	Frequency           float32
	GridVoltage         float32
	GridCurrent         float32
	GridPower           float32
	GridRunTime         duration
	Input1Voltage       float32
	Input1Current       float32
	Input2Voltage       float32
	Input2Current       float32
	Joules              uint16
	DailyEnergy         uint32
	WeeklyEnergy        uint32
	MonthlyEnergy       uint32
	YearlyEnergy        uint32
	TotalEnergy         uint32
	TotalRunTime        duration
	SerialNumber        string
}

type results struct {
	sync.RWMutex
	Results map[string]*result
}

type serialConfig struct {
	serial.Config
	ReadTimeout duration
}

type duration struct {
	time.Duration
}

func (o *serialConfig) Normalise() *serial.Config {
	o.Config.ReadTimeout = o.ReadTimeout.Duration
	return &o.Config
}

func (d *duration) UnmarshalText(text []byte) (err error) {
	d.Duration, err = time.ParseDuration(string(text))
	return
}

func (d duration) MarshalJSON() ([]byte, error) {
	return json.Marshal(int64(d.Duration.Seconds()))
}

func withDeadline(deadline time.Duration, f func() error) error {
	c := make(chan error, 1)
	defer close(c)
	go func() {
		c <- try.Do(func(attempt int) (bool, error) {
			time.Sleep(time.Duration(attempt) * time.Millisecond)
			err := f()
			return attempt < 3, err
		})
	}()
	select {
	case err := <-c:
		log.WithError(err).Errorf("Call to f() failed with error, %s", err.Error())
		return err
	case <-time.After(deadline):
		log.WithField("deadline", deadline).Warning("Timeout while reading from inverter")
		return errors.New("Timeout while waiting for operation to complete")
	}
}

type configStruct struct {
	Name          string
	Comms         serialConfig
	UpdateRate    duration
	Deadline      duration
	UnitAddresses []byte
}

func main() {
	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		log.Fatal(err)
	}

	config := struct {
		LogPath    string
		UpdateRate duration
		Deadline   duration
		Listen     string
		Devices    []configStruct
	}{
		LogPath: filepath.Join(dir, "main.log"),
		UpdateRate: duration{
			Duration: time.Minute,
		},
		Deadline: duration{
			Duration: 5 * time.Second,
		},
		Listen: ":8080",
	}

	fConfig := flag.String("config", "config.toml", "Path to the configuration file")
	flag.Parse()

	_, err = toml.DecodeFile(*fConfig, &config)
	if err != nil {
		log.Fatalf("Unable to parse configuration file due to %v.", err)
	}

	f, err := os.OpenFile(config.LogPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		log.Fatalf("error opening file: %v", err)
	}
	defer f.Close()

	log.Infof("Logging to %s, there should be no further output", config.LogPath)
	log.SetFormatter(&log.JSONFormatter{})
	log.SetOutput(f)
	log.SetLevel(log.DebugLevel)

	log.WithField("config", config).Debug("Configuration")

	buffer := results{
		Results: map[string]*result{},
	}

	for _, device := range config.Devices {
		go func(device configStruct) {
			logger := log.WithField("coms", device.Comms.Name)
			deadline := device.Deadline.Duration
			if deadline == 0 {
				deadline = config.Deadline.Duration
			}

			updateRate := device.UpdateRate.Duration
			if updateRate == 0 {
				updateRate = config.UpdateRate.Duration
			}

			port, err := serial.OpenPort(device.Comms.Normalise())
			if err != nil {
				log.WithError(err).Fatal("Startup error: Unable to open serial port")
			}

			inverter := &aurora.Inverter{
				Conn: port,
			}

			for _, address := range device.UnitAddresses {
				logger := logger.WithField("address", address)
				name := fmt.Sprintf("%s::%d", device.Comms.Name, address)
				inverter.Address = address
				buffer.Results[name] = &result{
					Address: address,
				}

				err := withDeadline(deadline, func() (err error) {
					buffer.Results[name].SerialNumber, err = inverter.SerialNumber()
					return
				})

				if err != nil {
					logger.WithError(err).Fatal("Startup error: Unable to communicate with inverter")
				}
			}

			ticker := time.NewTicker(updateRate)
			now := time.Now()
			for {
				for _, address := range device.UnitAddresses {
					name := fmt.Sprintf("%s::%d", device.Comms.Name, address)
					buffer.RLock()
					logger := logger.WithFields(log.Fields{
						"address": address,
						"serial":  buffer.Results[name].SerialNumber,
					})
					inverter.Address = address
					r := &result{
						Address:      address,
						SerialNumber: buffer.Results[name].SerialNumber,
						Time:         now,
					}
					buffer.RUnlock()

					err := withDeadline(deadline, func() error {
						var err error
						if r.BoosterTemperature, err = inverter.BoosterTemperature(); err != nil {
							logger.WithError(err).Warning("Unable to read BoosterTemperature")
							return err
						}
						if r.InverterTemperature, err = inverter.InverterTemperature(); err != nil {
							logger.WithError(err).Warning("Unable to read InverterTemperature")
							return err
						}
						if r.Frequency, err = inverter.Frequency(); err != nil {
							logger.WithError(err).Warning("Unable to read Frequency")
							return err
						}
						if r.GridVoltage, err = inverter.GridVoltage(); err != nil {
							logger.WithError(err).Warning("Unable to read GridVoltage")
							return err
						}
						if r.GridCurrent, err = inverter.GridCurrent(); err != nil {
							logger.WithError(err).Warning("Unable to read GridCurrent")
							return err
						}
						if r.GridPower, err = inverter.GridPower(); err != nil {
							logger.WithError(err).Warning("Unable to read GridPower")
							return err
						}
						if r.GridRunTime.Duration, err = inverter.GridRunTime(); err != nil {
							logger.WithError(err).Warning("Unable to read GridRunTime")
							return err
						}
						if r.Input1Voltage, err = inverter.Input1Voltage(); err != nil {
							logger.WithError(err).Warning("Unable to read Input1Voltage")
							return err
						}
						if r.Input1Current, err = inverter.Input1Current(); err != nil {
							logger.WithError(err).Warning("Unable to read Input1Current")
							return err
						}
						if r.Input2Voltage, err = inverter.Input2Voltage(); err != nil {
							logger.WithError(err).Warning("Unable to read Input2Voltage")
							return err
						}
						if r.Input2Current, err = inverter.Input2Current(); err != nil {
							logger.WithError(err).Warning("Unable to read Input2Current")
							return err
						}
						if r.Joules, err = inverter.Joules(); err != nil {
							logger.WithError(err).Warning("Unable to read Joules")
							return err
						}
						if r.DailyEnergy, err = inverter.DailyEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read DailyEnergy")
							return err
						}
						if r.WeeklyEnergy, err = inverter.WeeklyEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read WeeklyEnergy")
							return err
						}
						if r.MonthlyEnergy, err = inverter.MonthlyEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read MonthlyEnergy")
							return err
						}
						if r.YearlyEnergy, err = inverter.YearlyEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read YearlyEnergy")
							return err
						}
						if r.TotalEnergy, err = inverter.TotalEnergy(); err != nil {
							logger.WithError(err).Warning("Unable to read TotalEnergy")
							return err
						}
						r.TotalRunTime.Duration, err = inverter.TotalRunTime()
						if err != nil {
							logger.WithError(err).Warning("Unable to read TotalRunTime")
						}
						return err
					})

					if err == nil {
						buffer.Lock()
						buffer.Results[name] = r
						buffer.Unlock()
					}
				}
				now = <-ticker.C
			}
		}(device)
	}

	http.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		logger := log.WithField("remoteaddr", r.RemoteAddr)
		logger.Info("GET /json")
		buffer.RLock()
		defer buffer.RUnlock()
		js, err := json.Marshal(buffer.Results)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Cache-Control", "no-cache")
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Content-Type", "application/json")
		w.Write(js)
	})

	log.Fatal(http.ListenAndServe(config.Listen, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"mime"
	"net/http"
	"strings"

	"github.com/chop-dbhi/origins"
	"github.com/chop-dbhi/origins/storage"
	"github.com/julienschmidt/httprouter"
	"github.com/rs/cors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var httpCmd = &cobra.Command{
	Use: "http",

	Short: "Starts an HTTP peer.",

	Long: "Runs a process exposing an HTTP interface.",

	Run: func(cmd *cobra.Command, args []string) {
		var (
			host = viper.GetString("http_host")
			port = viper.GetInt("http_port")
		)

		engine := initStorage()

		serveHTTP(engine, host, port)
	},
}

func init() {
	flags := httpCmd.Flags()

	flags.String("host", "", "The host the HTTP service will listen on.")
	flags.Int("port", 49110, "The port the HTTP will bind to.")

	viper.BindPFlag("http_host", flags.Lookup("host"))
	viper.BindPFlag("http_port", flags.Lookup("port"))
}

const (
	defaultFormat = "text"

	StatusUnprocessableEntity = 422
)

var (
	mimetypes = map[string]string{
		"application/json": "json",
		"text/csv":         "csv",
		"text/plain":       "text",
	}

	formatMimetypes = map[string]string{
		"csv":  "text/csv",
		"json": "application/json",
		"text": "text/plain",
	}

	queryFormats = map[string]string{
		"json": "json",
		"csv":  "csv",
		"text": "text",
	}
)

// detectFormat applies content negotiation logic to determine the
// appropriate response representation.
func detectFormat(w http.ResponseWriter, r *http.Request) string {
	var (
		ok     bool
		format string
	)

	format = queryFormats[strings.ToLower(r.URL.Query().Get("format"))]

	// Query parameter
	if format == "" {
		// Accept header
		acceptType := r.Header.Get("Accept")
		acceptType, _, _ = mime.ParseMediaType(acceptType)

		// Fallback to default
		if format, ok = mimetypes[acceptType]; !ok {
			format = defaultFormat
		}
	}

	w.Header().Set("content-type", formatMimetypes[format])

	return format
}

func serveHTTP(engine storage.Engine, host string, port int) {
	addr := fmt.Sprintf("%s:%d", host, port)

	// Bind the routes.
	router := httprouter.New()

	router.GET("/", httpRoot)

	// Add CORS middleware
	c := cors.New(cors.Options{
		ExposedHeaders: []string{
			"Link",
			"Link-Template",
		},
	})

	handler := c.Handler(router)

	// Serve it up.
	logrus.Infof("* Listening on %s...", addr)

	logrus.Fatal(http.ListenAndServe(addr, handler))
}

// jsonResponse attempts to encode the passed value as JSON.
func jsonResponse(w http.ResponseWriter, v interface{}) {
	e := json.NewEncoder(w)

	if err := e.Encode(v); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(fmt.Sprint(err)))
	}
}

func httpRoot(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	jsonResponse(w, map[string]interface{}{
		"Title":   "Origins HTTP Service",
		"Version": origins.Version,
	})
}
<commit_msg>Implement resource for a log view<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"mime"
	"net/http"
	"strings"
	"time"

	"github.com/chop-dbhi/origins"
	"github.com/chop-dbhi/origins/chrono"
	"github.com/chop-dbhi/origins/storage"
	"github.com/chop-dbhi/origins/view"
	"github.com/julienschmidt/httprouter"
	"github.com/rs/cors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var httpCmd = &cobra.Command{
	Use: "http",

	Short: "Starts an HTTP peer.",

	Long: "Runs a process exposing an HTTP interface.",

	Run: func(cmd *cobra.Command, args []string) {
		var (
			host = viper.GetString("http_host")
			port = viper.GetInt("http_port")
		)

		engine := initStorage()

		serveHTTP(engine, host, port)
	},
}

func init() {
	flags := httpCmd.Flags()

	flags.String("host", "", "The host the HTTP service will listen on.")
	flags.Int("port", 49110, "The port the HTTP will bind to.")

	viper.BindPFlag("http_host", flags.Lookup("host"))
	viper.BindPFlag("http_port", flags.Lookup("port"))
}

const (
	defaultFormat = "text"

	StatusUnprocessableEntity = 422
)

var (
	mimetypes = map[string]string{
		"application/json": "json",
		"text/csv":         "csv",
		"text/plain":       "text",
	}

	formatMimetypes = map[string]string{
		"csv":  "text/csv",
		"json": "application/json",
		"text": "text/plain",
	}

	queryFormats = map[string]string{
		"json": "json",
		"csv":  "csv",
		"text": "text",
	}
)

// detectFormat applies content negotiation logic to determine the
// appropriate response representation.
func detectFormat(w http.ResponseWriter, r *http.Request) string {
	var (
		ok     bool
		format string
	)

	format = queryFormats[strings.ToLower(r.URL.Query().Get("format"))]

	// Query parameter
	if format == "" {
		// Accept header
		acceptType := r.Header.Get("Accept")
		acceptType, _, _ = mime.ParseMediaType(acceptType)

		// Fallback to default
		if format, ok = mimetypes[acceptType]; !ok {
			format = defaultFormat
		}
	}

	w.Header().Set("content-type", formatMimetypes[format])

	return format
}

func serveHTTP(engine storage.Engine, host string, port int) {
	addr := fmt.Sprintf("%s:%d", host, port)

	// Bind the routes.
	router := httprouter.New()

	router.GET("/", httpRoot)
	router.GET("/log/:domain", httpLogView)

	// Add CORS middleware
	c := cors.New(cors.Options{
		ExposedHeaders: []string{
			"Link",
			"Link-Template",
		},
	})

	handler := c.Handler(router)

	// Serve it up.
	logrus.Infof("* Listening on %s...", addr)

	logrus.Fatal(http.ListenAndServe(addr, handler))
}

// jsonResponse attempts to encode the passed value as JSON.
func jsonResponse(w http.ResponseWriter, v interface{}) {
	e := json.NewEncoder(w)

	if err := e.Encode(v); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(fmt.Sprint(err)))
	}
}

func httpRoot(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	jsonResponse(w, map[string]interface{}{
		"Title":   "Origins HTTP Service",
		"Version": origins.Version,
	})
}

func httpLogView(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	domain := p.ByName("domain")

	var (
		err         error
		since, asof time.Time
	)

	q := r.URL.Query()

	// Parse query parameters
	if q.Get("since") != "" {
		since, err = chrono.Parse(q.Get("since"))

		if err != nil {
			w.WriteHeader(StatusUnprocessableEntity)
			w.Write([]byte(fmt.Sprint(err)))
			return
		}
	}

	if q.Get("asof") != "" {
		asof, err = chrono.Parse(q.Get("asof"))

		if err != nil {
			w.WriteHeader(StatusUnprocessableEntity)
			w.Write([]byte(fmt.Sprint(err)))
			return
		}
	}

	// Open the log.
	engine := initStorage()

	log, err := view.OpenLog(engine, domain, "log.commit")

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(fmt.Sprint(err)))
		return
	}

	// Construct a view of the log for the specified window of time.
	v := log.View(since, asof)

	var fw origins.Writer

	format := detectFormat(w, r)

	switch format {
	case "text", "csv":
		fw = origins.CSVWriter(w)
	default:
		w.WriteHeader(http.StatusNotAcceptable)
		return
	}

	if _, err := origins.ReadWriter(v, fw); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(fmt.Sprint(err)))
		return
	}
}
<|endoftext|>


<commit_before>/*
rqlite -- a replicated SQLite database.

rqlite is a distributed system that provides a replicated SQLite database.
rqlite is written in Go and uses Raft to achieve consensus across all the
instances of the SQLite databases. rqlite ensures that every change made to
the database is made to a majority of underlying SQLite files, or none-at-all.
*/

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"time"

	sql "github.com/otoolep/rqlite/db"
	httpd "github.com/otoolep/rqlite/http"
	"github.com/otoolep/rqlite/store"
)

const sqliteDSN = "db.sqlite"

var httpAddr string
var raftAddr string
var joinAddr string
var dsn string
var cpuprofile string
var disableReporting bool

func init() {
	flag.StringVar(&raftAddr, "raft", "localhost:4001", "Raft communication bind address")
	flag.StringVar(&httpAddr, "http", "localhost:4002", "HTTP query server bind address")
	flag.StringVar(&joinAddr, "join", "", "host:port of leader to join")
	flag.StringVar(&dsn, "dsn", "", "SQLite DSN parameters")
	flag.StringVar(&cpuprofile, "cpuprofile", "", "Write CPU profile to file")
	flag.BoolVar(&disableReporting, "noreport", false, "Disable anonymised launch reporting")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [arguments] <data-path>\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	// Ensure the data path was set.
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	dataPath := flag.Arg(0)

	// Set up profiling, if requested.
	if cpuprofile != "" {
		log.Println("profiling enabled")
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Printf("unable to create path: %s", err.Error())
		}
		defer f.Close()

		err = pprof.StartCPUProfile(f)
		if err != nil {
			log.Printf("unable to start CPU Profile: %s", err.Error())
		}

		defer pprof.StopCPUProfile()
	}

	// Create and open the store.
	dbConf := sql.NewConfig()    // XXX CONSIDER SUPPORTING IN-MEMORY DATABASE! That way is fast, but log is persistence.
	dbConf.DSN = dsn
	store := store.New(dbConf, dataPath, raftAddr)
	if err := store.Open(joinAddr == ""); err != nil {
		log.Fatalf("failed to open store: %s", err.Error())
	}

	// If join was specified, make the join request.
	if joinAddr != "" {
		if err := join(joinAddr, raftAddr); err != nil {
			log.Fatalf("failed to join node at %s: %s", joinAddr, err.Error())
		}
	}

	// Create the HTTP query server.
	s := httpd.New(httpAddr, store)
	if err := s.Start(); err != nil {
		log.Fatalf("failed to start HTTP server: %s", err.Error())

	}

	if !disableReporting {
		reportLaunch()
	}

	terminate := make(chan os.Signal, 1)
	signal.Notify(terminate, os.Interrupt)
	<-terminate
	if err := store.Close(); err != nil {
		log.Printf("failed to close store: %s", err.Error())
	}
	log.Println("rqlite server stopped")
}

func join(joinAddr, raftAddr string) error {
	b, err := json.Marshal(map[string]string{"addr": raftAddr})
	if err != nil {
		return err
	}
	resp, err := http.Post(fmt.Sprintf("http://%s/join", joinAddr), "application-type/json", bytes.NewReader(b))
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

func reportLaunch() {
	json := fmt.Sprintf(`{"os": "%s", "arch": "%s", "app": "rqlite"}`, runtime.GOOS, runtime.GOARCH)
	data := bytes.NewBufferString(json)
	client := http.Client{Timeout: time.Duration(5 * time.Second)}
	go func() {
		_, err := client.Post("https://logs-01.loggly.com/inputs/8a0edd84-92ba-46e4-ada8-c529d0f105af/tag/reporting/",
			"application/json", data)
		if err != nil {
			log.Printf("report launch failed: %s", err.Error())
		}
	}()
}
<commit_msg>Enhance reporting so Hashicorp use is flagged<commit_after>/*
rqlite -- a replicated SQLite database.

rqlite is a distributed system that provides a replicated SQLite database.
rqlite is written in Go and uses Raft to achieve consensus across all the
instances of the SQLite databases. rqlite ensures that every change made to
the database is made to a majority of underlying SQLite files, or none-at-all.
*/

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"time"

	sql "github.com/otoolep/rqlite/db"
	httpd "github.com/otoolep/rqlite/http"
	"github.com/otoolep/rqlite/store"
)

const sqliteDSN = "db.sqlite"

var httpAddr string
var raftAddr string
var joinAddr string
var dsn string
var cpuprofile string
var disableReporting bool

func init() {
	flag.StringVar(&raftAddr, "raft", "localhost:4001", "Raft communication bind address")
	flag.StringVar(&httpAddr, "http", "localhost:4002", "HTTP query server bind address")
	flag.StringVar(&joinAddr, "join", "", "host:port of leader to join")
	flag.StringVar(&dsn, "dsn", "", "SQLite DSN parameters")
	flag.StringVar(&cpuprofile, "cpuprofile", "", "Write CPU profile to file")
	flag.BoolVar(&disableReporting, "noreport", false, "Disable anonymised launch reporting")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [arguments] <data-path>\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	// Ensure the data path was set.
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	dataPath := flag.Arg(0)

	// Set up profiling, if requested.
	if cpuprofile != "" {
		log.Println("profiling enabled")
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Printf("unable to create path: %s", err.Error())
		}
		defer f.Close()

		err = pprof.StartCPUProfile(f)
		if err != nil {
			log.Printf("unable to start CPU Profile: %s", err.Error())
		}

		defer pprof.StopCPUProfile()
	}

	// Create and open the store.
	dbConf := sql.NewConfig()    // XXX CONSIDER SUPPORTING IN-MEMORY DATABASE! That way is fast, but log is persistence.
	dbConf.DSN = dsn
	store := store.New(dbConf, dataPath, raftAddr)
	if err := store.Open(joinAddr == ""); err != nil {
		log.Fatalf("failed to open store: %s", err.Error())
	}

	// If join was specified, make the join request.
	if joinAddr != "" {
		if err := join(joinAddr, raftAddr); err != nil {
			log.Fatalf("failed to join node at %s: %s", joinAddr, err.Error())
		}
	}

	// Create the HTTP query server.
	s := httpd.New(httpAddr, store)
	if err := s.Start(); err != nil {
		log.Fatalf("failed to start HTTP server: %s", err.Error())

	}

	if !disableReporting {
		reportLaunch()
	}

	terminate := make(chan os.Signal, 1)
	signal.Notify(terminate, os.Interrupt)
	<-terminate
	if err := store.Close(); err != nil {
		log.Printf("failed to close store: %s", err.Error())
	}
	log.Println("rqlite server stopped")
}

func join(joinAddr, raftAddr string) error {
	b, err := json.Marshal(map[string]string{"addr": raftAddr})
	if err != nil {
		return err
	}
	resp, err := http.Post(fmt.Sprintf("http://%s/join", joinAddr), "application-type/json", bytes.NewReader(b))
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

func reportLaunch() {
	json := fmt.Sprintf(`{"os": "%s", "arch": "%s", "app": "rqlite", "raft": "hashicorp"}`, runtime.GOOS, runtime.GOARCH)
	data := bytes.NewBufferString(json)
	client := http.Client{Timeout: time.Duration(5 * time.Second)}
	go func() {
		_, err := client.Post("https://logs-01.loggly.com/inputs/8a0edd84-92ba-46e4-ada8-c529d0f105af/tag/reporting/",
			"application/json", data)
		if err != nil {
			log.Printf("report launch failed: %s", err.Error())
		}
	}()
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"os"
	"path/filepath"

	"github.com/codeclimate/test-reporter/formatters"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

type CoverageSummer struct {
	Output string
	Parts  int
}

var summerOptions = CoverageSummer{}

var sumCoverageCmd = &cobra.Command{
	Use:   "sum-coverage",
	Short: "Combine (sum) multiple pre-formatted coverage payloads into one.",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("you must pass in one or more files to be summarized")
		}
		if summerOptions.Parts != 0 && len(args) != summerOptions.Parts {
			return errors.Errorf("expected %d parts, received only %d parts", summerOptions.Parts, len(args))
		}

		rep := formatters.Report{
			SourceFiles: formatters.SourceFiles{},
		}

		f, err := os.Open(args[0])
		if err != nil {
			return errors.WithStack(err)
		}

		err = json.NewDecoder(f).Decode(&rep)
		if err != nil {
			return errors.WithStack(err)
		}

		for _, n := range args[1:] {
			f, err := os.Open(n)
			if err != nil {
				return errors.WithStack(err)
			}

			rr := formatters.Report{
				SourceFiles: formatters.SourceFiles{},
			}

			err = json.NewDecoder(f).Decode(&rr)
			if err != nil {
				return errors.WithStack(err)
			}
			err = rep.Merge(&rr)
			if err != nil {
				return errors.WithStack(err)
			}
		}

		err = os.MkdirAll(filepath.Dir(summerOptions.Output), 0755)
		if err != nil {
			return errors.WithStack(err)
		}
		out, err := os.Create(summerOptions.Output)
		if err != nil {
			return errors.WithStack(err)
		}

		err = rep.Save(out)
		if err != nil {
			return errors.WithStack(err)
		}
		return nil
	},
}

func init() {
	sumCoverageCmd.Flags().IntVarP(&summerOptions.Parts, "parts", "p", 0, "total number of parts to sum")
	sumCoverageCmd.Flags().StringVarP(&summerOptions.Output, "output", "o", ccDefaultCoveragePath, "output path")
	RootCmd.AddCommand(sumCoverageCmd)
}
<commit_msg>Generalize parts mismatch error message (#167)<commit_after>package cmd

import (
	"encoding/json"
	"os"
	"path/filepath"

	"github.com/codeclimate/test-reporter/formatters"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

type CoverageSummer struct {
	Output string
	Parts  int
}

var summerOptions = CoverageSummer{}

var sumCoverageCmd = &cobra.Command{
	Use:   "sum-coverage",
	Short: "Combine (sum) multiple pre-formatted coverage payloads into one.",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("you must pass in one or more files to be summarized")
		}
		if summerOptions.Parts != 0 && len(args) != summerOptions.Parts {
			return errors.Errorf("expected %d parts, received %d parts", summerOptions.Parts, len(args))
		}

		rep := formatters.Report{
			SourceFiles: formatters.SourceFiles{},
		}

		f, err := os.Open(args[0])
		if err != nil {
			return errors.WithStack(err)
		}

		err = json.NewDecoder(f).Decode(&rep)
		if err != nil {
			return errors.WithStack(err)
		}

		for _, n := range args[1:] {
			f, err := os.Open(n)
			if err != nil {
				return errors.WithStack(err)
			}

			rr := formatters.Report{
				SourceFiles: formatters.SourceFiles{},
			}

			err = json.NewDecoder(f).Decode(&rr)
			if err != nil {
				return errors.WithStack(err)
			}
			err = rep.Merge(&rr)
			if err != nil {
				return errors.WithStack(err)
			}
		}

		err = os.MkdirAll(filepath.Dir(summerOptions.Output), 0755)
		if err != nil {
			return errors.WithStack(err)
		}
		out, err := os.Create(summerOptions.Output)
		if err != nil {
			return errors.WithStack(err)
		}

		err = rep.Save(out)
		if err != nil {
			return errors.WithStack(err)
		}
		return nil
	},
}

func init() {
	sumCoverageCmd.Flags().IntVarP(&summerOptions.Parts, "parts", "p", 0, "total number of parts to sum")
	sumCoverageCmd.Flags().StringVarP(&summerOptions.Output, "output", "o", ccDefaultCoveragePath, "output path")
	RootCmd.AddCommand(sumCoverageCmd)
}
<|endoftext|>


<commit_before>package tokenize

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/ikawaha/kagome-dict/dict"
	"github.com/ikawaha/kagome-dict/ipa"
	"github.com/ikawaha/kagome-dict/uni"
	"github.com/ikawaha/kagome/v2/filter"
	"github.com/ikawaha/kagome/v2/tokenizer"
)

// subcommand property
const (
	CommandName  = "tokenize"
	Description  = `command line tokenize`
	usageMessage = "%s [-file input_file] [-dict dic_file] [-userdict user_dic_file]" +
		" [-sysdict (ipa|uni)] [-simple false] [-mode (normal|search|extended)] [-split] [-json]"
)

var (
	// Stdout is the standard writer.
	Stdout io.Writer = os.Stdout
	// Stderr is the standard error writer.
	Stderr io.Writer = os.Stderr
)

// options
type option struct {
	file    string
	dict    string
	udict   string
	sysdict string
	simple  bool
	mode    string
	split   bool
	json    bool
	flagSet *flag.FlagSet
}

// ContinueOnError ErrorHandling // Return a descriptive error.
// ExitOnError                   // Call os.Exit(2).
// PanicOnError                  // Call panic with a descriptive error.flag.ContinueOnError
func newOption(w io.Writer, eh flag.ErrorHandling) (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, eh),
	}
	// option settings
	o.flagSet.SetOutput(w)
	o.flagSet.StringVar(&o.file, "file", "", "input file")
	o.flagSet.StringVar(&o.dict, "dict", "", "dict")
	o.flagSet.StringVar(&o.udict, "udict", "", "user dict")
	o.flagSet.StringVar(&o.sysdict, "sysdict", "ipa", "system dict type (ipa|uni)")
	o.flagSet.BoolVar(&o.simple, "simple", false, "display abbreviated dictionary contents")
	o.flagSet.StringVar(&o.mode, "mode", "normal", "tokenize mode (normal|search|extended)")
	o.flagSet.BoolVar(&o.split, "split", false, "use tiny sentence splitter")
	o.flagSet.BoolVar(&o.json, "json", false, "outputs in JSON format")

	return
}

func (o *option) parse(args []string) error {
	if err := o.flagSet.Parse(args); err != nil {
		return err
	}
	// validations
	if nonFlag := o.flagSet.Args(); len(nonFlag) != 0 {
		return fmt.Errorf("invalid argument: %v", nonFlag)
	}
	if o.mode != "" && o.mode != "normal" && o.mode != "search" && o.mode != "extended" {
		return fmt.Errorf("invalid argument: -mode %v", o.mode)
	}
	if o.sysdict != "" && o.sysdict != "ipa" && o.sysdict != "uni" {
		return fmt.Errorf("invalid argument: -sysdict %v", o.sysdict)
	}
	return nil
}

// OptionCheck receives a slice of args and returns an error if it was not successfully parsed
func OptionCheck(args []string) error {
	opt := newOption(io.Discard, flag.ContinueOnError)
	if err := opt.parse(args); err != nil {
		return fmt.Errorf("%v, %w", CommandName, err)
	}
	return nil
}

func selectDict(path, sysdict string, shurink bool) (*dict.Dict, error) {
	if path != "" {
		if shurink {
			return dict.LoadShrink(path)
		}
		return dict.LoadDictFile(path)
	}
	switch sysdict {
	case "ipa":
		if shurink {
			return ipa.DictShrink(), nil
		}
		return ipa.Dict(), nil
	case "uni":
		if shurink {
			return uni.DictShrink(), nil
		}
		return uni.Dict(), nil
	}
	return nil, fmt.Errorf("unknown dict type, %v", sysdict)
}

func selectMode(mode string) tokenizer.TokenizeMode {
	switch mode {
	case "normal":
		return tokenizer.Normal
	case "search":
		return tokenizer.Search
	case "extended":
		return tokenizer.Extended
	}
	return tokenizer.Normal
}

func command(_ context.Context, opt *option) error {
	d, err := selectDict(opt.dict, opt.sysdict, opt.simple)
	if err != nil {
		return err
	}
	udict := tokenizer.Nop()
	if opt.udict != "" {
		d, err := dict.NewUserDict(opt.udict)
		if err != nil {
			return err
		}
		udict = tokenizer.UserDict(d)
	}
	t, err := tokenizer.New(d, udict)
	if err != nil {
		return err
	}

	fp := os.Stdin
	if opt.file != "" {
		var err error
		fp, err = os.Open(opt.file)
		if err != nil {
			return err
		}
		defer func() {
			_ = fp.Close()
		}()
	}
	mode := selectMode(opt.mode)
	s := bufio.NewScanner(fp)
	if opt.split {
		s.Split(filter.ScanSentences)
	}
	for s.Scan() {
		tokens := t.Analyze(s.Text(), mode)
		if !opt.json {
			printTokens(tokens)
			continue
		}
		if err := printTokensJSON(tokens); err != nil {
			return err
		}
	}
	return s.Err()
}

func printTokens(tokens []tokenizer.Token) {
	w := bufio.NewWriter(Stdout)
	defer w.Flush()
	for _, v := range tokens {
		if v.ID == tokenizer.BosEosID {
			continue
		}
		w.WriteString(v.Surface)
		if v.Class != tokenizer.DUMMY {
			w.WriteString("\t")
			w.WriteString(strings.Join(v.Features(), ","))
		}
		w.WriteString("\n")
	}
	w.WriteString("EOS\n")
}

func printTokensJSON(tokens []tokenizer.Token) error {
	w := bufio.NewWriter(Stdout)
	defer w.Flush()

	if len(tokens) > 0 {
		w.WriteString("[\n")
	}
	var array [][]byte
	for _, v := range tokens {
		if v.Class == tokenizer.DUMMY {
			continue
		}
		r := tokenizer.NewTokenData(v)
		obj, err := json.Marshal(r)
		if err != nil {
			return err
		}
		array = append(array, obj)
	}
	w.Write(bytes.Join(array, []byte(",\n")))
	if len(tokens) > 0 {
		w.WriteString("\n]\n")
	}
	return nil
}

// Run receives the slice of args and executes the tokenize tool
func Run(ctx context.Context, args []string) error {
	opt := newOption(Stderr, flag.ContinueOnError)
	if err := opt.parse(args); err != nil {
		Usage()
		PrintDefaults(flag.ContinueOnError)
		return errors.New("")
	}
	return command(ctx, opt)
}

// Usage provides information on the use of the tokenize tool
func Usage() {
	fmt.Fprintf(Stderr, usageMessage+"\n", CommandName)
}

// PrintDefaults prints out the default flags
func PrintDefaults(eh flag.ErrorHandling) {
	o := newOption(Stderr, eh)
	o.flagSet.PrintDefaults()
}
<commit_msg>fix typo (#262)<commit_after>package tokenize

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/ikawaha/kagome-dict/dict"
	"github.com/ikawaha/kagome-dict/ipa"
	"github.com/ikawaha/kagome-dict/uni"
	"github.com/ikawaha/kagome/v2/filter"
	"github.com/ikawaha/kagome/v2/tokenizer"
)

// subcommand property
const (
	CommandName  = "tokenize"
	Description  = `command line tokenize`
	usageMessage = "%s [-file input_file] [-dict dic_file] [-userdict user_dic_file]" +
		" [-sysdict (ipa|uni)] [-simple false] [-mode (normal|search|extended)] [-split] [-json]"
)

var (
	// Stdout is the standard writer.
	Stdout io.Writer = os.Stdout
	// Stderr is the standard error writer.
	Stderr io.Writer = os.Stderr
)

// options
type option struct {
	file    string
	dict    string
	udict   string
	sysdict string
	simple  bool
	mode    string
	split   bool
	json    bool
	flagSet *flag.FlagSet
}

// ContinueOnError ErrorHandling // Return a descriptive error.
// ExitOnError                   // Call os.Exit(2).
// PanicOnError                  // Call panic with a descriptive error.flag.ContinueOnError
func newOption(w io.Writer, eh flag.ErrorHandling) (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, eh),
	}
	// option settings
	o.flagSet.SetOutput(w)
	o.flagSet.StringVar(&o.file, "file", "", "input file")
	o.flagSet.StringVar(&o.dict, "dict", "", "dict")
	o.flagSet.StringVar(&o.udict, "udict", "", "user dict")
	o.flagSet.StringVar(&o.sysdict, "sysdict", "ipa", "system dict type (ipa|uni)")
	o.flagSet.BoolVar(&o.simple, "simple", false, "display abbreviated dictionary contents")
	o.flagSet.StringVar(&o.mode, "mode", "normal", "tokenize mode (normal|search|extended)")
	o.flagSet.BoolVar(&o.split, "split", false, "use tiny sentence splitter")
	o.flagSet.BoolVar(&o.json, "json", false, "outputs in JSON format")

	return
}

func (o *option) parse(args []string) error {
	if err := o.flagSet.Parse(args); err != nil {
		return err
	}
	// validations
	if nonFlag := o.flagSet.Args(); len(nonFlag) != 0 {
		return fmt.Errorf("invalid argument: %v", nonFlag)
	}
	if o.mode != "" && o.mode != "normal" && o.mode != "search" && o.mode != "extended" {
		return fmt.Errorf("invalid argument: -mode %v", o.mode)
	}
	if o.sysdict != "" && o.sysdict != "ipa" && o.sysdict != "uni" {
		return fmt.Errorf("invalid argument: -sysdict %v", o.sysdict)
	}
	return nil
}

// OptionCheck receives a slice of args and returns an error if it was not successfully parsed
func OptionCheck(args []string) error {
	opt := newOption(io.Discard, flag.ContinueOnError)
	if err := opt.parse(args); err != nil {
		return fmt.Errorf("%v, %w", CommandName, err)
	}
	return nil
}

func selectDict(path, sysdict string, shrink bool) (*dict.Dict, error) {
	if path != "" {
		if shrink {
			return dict.LoadShrink(path)
		}
		return dict.LoadDictFile(path)
	}
	switch sysdict {
	case "ipa":
		if shrink {
			return ipa.DictShrink(), nil
		}
		return ipa.Dict(), nil
	case "uni":
		if shrink {
			return uni.DictShrink(), nil
		}
		return uni.Dict(), nil
	}
	return nil, fmt.Errorf("unknown dict type, %v", sysdict)
}

func selectMode(mode string) tokenizer.TokenizeMode {
	switch mode {
	case "normal":
		return tokenizer.Normal
	case "search":
		return tokenizer.Search
	case "extended":
		return tokenizer.Extended
	}
	return tokenizer.Normal
}

func command(_ context.Context, opt *option) error {
	d, err := selectDict(opt.dict, opt.sysdict, opt.simple)
	if err != nil {
		return err
	}
	udict := tokenizer.Nop()
	if opt.udict != "" {
		d, err := dict.NewUserDict(opt.udict)
		if err != nil {
			return err
		}
		udict = tokenizer.UserDict(d)
	}
	t, err := tokenizer.New(d, udict)
	if err != nil {
		return err
	}

	fp := os.Stdin
	if opt.file != "" {
		var err error
		fp, err = os.Open(opt.file)
		if err != nil {
			return err
		}
		defer func() {
			_ = fp.Close()
		}()
	}
	mode := selectMode(opt.mode)
	s := bufio.NewScanner(fp)
	if opt.split {
		s.Split(filter.ScanSentences)
	}
	for s.Scan() {
		tokens := t.Analyze(s.Text(), mode)
		if !opt.json {
			printTokens(tokens)
			continue
		}
		if err := printTokensJSON(tokens); err != nil {
			return err
		}
	}
	return s.Err()
}

func printTokens(tokens []tokenizer.Token) {
	w := bufio.NewWriter(Stdout)
	defer w.Flush()
	for _, v := range tokens {
		if v.ID == tokenizer.BosEosID {
			continue
		}
		w.WriteString(v.Surface)
		if v.Class != tokenizer.DUMMY {
			w.WriteString("\t")
			w.WriteString(strings.Join(v.Features(), ","))
		}
		w.WriteString("\n")
	}
	w.WriteString("EOS\n")
}

func printTokensJSON(tokens []tokenizer.Token) error {
	w := bufio.NewWriter(Stdout)
	defer w.Flush()

	if len(tokens) > 0 {
		w.WriteString("[\n")
	}
	var array [][]byte
	for _, v := range tokens {
		if v.Class == tokenizer.DUMMY {
			continue
		}
		r := tokenizer.NewTokenData(v)
		obj, err := json.Marshal(r)
		if err != nil {
			return err
		}
		array = append(array, obj)
	}
	w.Write(bytes.Join(array, []byte(",\n")))
	if len(tokens) > 0 {
		w.WriteString("\n]\n")
	}
	return nil
}

// Run receives the slice of args and executes the tokenize tool
func Run(ctx context.Context, args []string) error {
	opt := newOption(Stderr, flag.ContinueOnError)
	if err := opt.parse(args); err != nil {
		Usage()
		PrintDefaults(flag.ContinueOnError)
		return errors.New("")
	}
	return command(ctx, opt)
}

// Usage provides information on the use of the tokenize tool
func Usage() {
	fmt.Fprintf(Stderr, usageMessage+"\n", CommandName)
}

// PrintDefaults prints out the default flags
func PrintDefaults(eh flag.ErrorHandling) {
	o := newOption(Stderr, eh)
	o.flagSet.PrintDefaults()
}
<|endoftext|>


<commit_before>package manager

import (
	"bytes"
	"encoding/json"
	"strings"

	"github.com/sirupsen/logrus"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/cluster"
	"github.com/portworx/kvdb"
)

const (
	// ClusterDBKey is the key at which cluster info is store in kvdb
	ClusterDBKey = "cluster/database"
)

type updateCallbackFn func(db *cluster.ClusterInfo) (bool, error)

func snapAndReadClusterInfo(snapshotPrefixes []string) (*cluster.ClusterInitState, error) {
	kv := kvdb.Instance()

	// To work-around a kvdb issue with watches, try snapshot in a loop
	var (
		collector kvdb.UpdatesCollector
		err       error
		version   uint64
		snap      kvdb.Kvdb
	)
	for i := 0; i < 3; i++ {
		if i > 0 {
			logrus.Infof("Retrying snapshot")
		}
		// Start the watch before the snapshot
		collector, err = kvdb.NewUpdatesCollector(kv, "", 0)
		if err != nil {
			logrus.Errorf("Failed to start collector for cluster db: %v", err)
			collector = nil
			continue
		}
		// Create the snapshot
		snap, version, err = kv.Snapshot(snapshotPrefixes, true)
		if err != nil {
			logrus.Errorf("Snapshot failed for cluster db: %v", err)
			collector.Stop()
			collector = nil
		} else {
			break
		}
	}
	if err != nil {
		return nil, err
	}
	logrus.Infof("Cluster db snapshot at: %v", version)

	clusterDB, err := snap.Get(ClusterDBKey)
	if err != nil && !strings.Contains(err.Error(), "Key not found") {
		logrus.Warnln("Warning, could not read cluster database")
		return nil, err
	}

	db := emptyClusterInfo()
	state := &cluster.ClusterInitState{
		ClusterInfo: &db,
		InitDb:      snap,
		Version:     version,
		Collector:   collector,
	}

	if clusterDB == nil || bytes.Compare(clusterDB.Value, []byte("{}")) == 0 {
		logrus.Infoln("Cluster is uninitialized...")
		return state, nil
	}
	if err := json.Unmarshal(clusterDB.Value, &db); err != nil {
		logrus.Warnln("Fatal, Could not parse cluster database ", kv)
		return state, err
	}

	return state, nil
}

func emptyClusterInfo() cluster.ClusterInfo {
	return cluster.ClusterInfo{
		Status:      api.Status_STATUS_INIT,
		NodeEntries: make(map[string]cluster.NodeEntry),
	}
}

func unmarshalClusterInfo(kv *kvdb.KVPair) (cluster.ClusterInfo, uint64, error) {
	db := emptyClusterInfo()
	version := uint64(0)
	if kv != nil {
		version = kv.ModifiedIndex
	}
	if kv == nil || bytes.Compare(kv.Value, []byte("{}")) == 0 {
		logrus.Infoln("Cluster is uninitialized...")
		return db, version, nil
	}
	if err := json.Unmarshal(kv.Value, &db); err != nil {
		logrus.Warnln("Fatal, Could not parse cluster database ", kv)
		return db, version, err
	}
	return db, version, nil
}

func readClusterInfo() (cluster.ClusterInfo, uint64, error) {
	kv, err := kvdb.Instance().Get(ClusterDBKey)
	if err != nil && !strings.Contains(err.Error(), "Key not found") {
		logrus.Warnln("Warning, could not read cluster database")
		return emptyClusterInfo(), 0, err
	}
	return unmarshalClusterInfo(kv)
}

func lockAndUpdateDB(fn, lockID string, cb updateCallbackFn) (*kvdb.KVPair, error) {
	kv := kvdb.Instance()

	kvlock, err := kv.LockWithID(clusterLockKey, lockID)
	if err != nil {
		logrus.Warnf("Unable to obtain cluster lock for %v op: %v", fn, err)
		return nil, err
	}
	defer kv.Unlock(kvlock)

	db, version, err := readClusterInfo()
	if err != nil {
		return nil, err
	}

	update, err := cb(&db)
	if err != nil {
		return nil, err
	}
	if !update {
		return nil, nil
	}

	if version != 0 {
		b, err := json.Marshal(db)
		if err != nil {
			logrus.Warnf("Fatal, Could not marshal cluster database to JSON: %v", err)
			return nil, err
		}

		kvpInput := &kvdb.KVPair{
			Key:           ClusterDBKey,
			Value:         b,
			ModifiedIndex: version,
		}

		return kv.CompareAndSet(kvpInput, kvdb.KVModifiedIndex, nil)
	} else {
		// key does not exists yet, call create
		kvp, err := kv.Create(ClusterDBKey, db, 0)
		if err == kvdb.ErrExist {
			// key created, we may have lost the lock, retry
			err = kvdb.ErrModified
		}
		return kvp, err
	}
}

func updateDB(fn, lockID string, cb updateCallbackFn) error {
	_, err := updateAndGetDB(fn, lockID, cb, true)
	return err
}

func updateAndGetDB(
	fn string,
	lockID string,
	cb updateCallbackFn,
	retry bool,
) (*kvdb.KVPair, error) {
	for {
		if kvp, err := lockAndUpdateDB(fn, lockID, cb); err == kvdb.ErrModified && retry {
			// compare and set failed, retry
			continue
		} else {
			return kvp, err
		}
	}
}
<commit_msg>PWX-22045: Scale bringup - increasing timeout for clustermap lock<commit_after>package manager

import (
	"bytes"
	"encoding/json"
	"strings"
	"time"

	"github.com/sirupsen/logrus"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/cluster"
	"github.com/portworx/kvdb"
)

const (
	// ClusterDBKey is the key at which cluster info is store in kvdb
	ClusterDBKey = "cluster/database"
)

type updateCallbackFn func(db *cluster.ClusterInfo) (bool, error)

func snapAndReadClusterInfo(snapshotPrefixes []string) (*cluster.ClusterInitState, error) {
	kv := kvdb.Instance()

	// To work-around a kvdb issue with watches, try snapshot in a loop
	var (
		collector kvdb.UpdatesCollector
		err       error
		version   uint64
		snap      kvdb.Kvdb
	)
	for i := 0; i < 3; i++ {
		if i > 0 {
			logrus.Infof("Retrying snapshot")
		}
		// Start the watch before the snapshot
		collector, err = kvdb.NewUpdatesCollector(kv, "", 0)
		if err != nil {
			logrus.Errorf("Failed to start collector for cluster db: %v", err)
			collector = nil
			continue
		}
		// Create the snapshot
		snap, version, err = kv.Snapshot(snapshotPrefixes, true)
		if err != nil {
			logrus.Errorf("Snapshot failed for cluster db: %v", err)
			collector.Stop()
			collector = nil
		} else {
			break
		}
	}
	if err != nil {
		return nil, err
	}
	logrus.Infof("Cluster db snapshot at: %v", version)

	clusterDB, err := snap.Get(ClusterDBKey)
	if err != nil && !strings.Contains(err.Error(), "Key not found") {
		logrus.Warnln("Warning, could not read cluster database")
		return nil, err
	}

	db := emptyClusterInfo()
	state := &cluster.ClusterInitState{
		ClusterInfo: &db,
		InitDb:      snap,
		Version:     version,
		Collector:   collector,
	}

	if clusterDB == nil || bytes.Compare(clusterDB.Value, []byte("{}")) == 0 {
		logrus.Infoln("Cluster is uninitialized...")
		return state, nil
	}
	if err := json.Unmarshal(clusterDB.Value, &db); err != nil {
		logrus.Warnln("Fatal, Could not parse cluster database ", kv)
		return state, err
	}

	return state, nil
}

func emptyClusterInfo() cluster.ClusterInfo {
	return cluster.ClusterInfo{
		Status:      api.Status_STATUS_INIT,
		NodeEntries: make(map[string]cluster.NodeEntry),
	}
}

func unmarshalClusterInfo(kv *kvdb.KVPair) (cluster.ClusterInfo, uint64, error) {
	db := emptyClusterInfo()
	version := uint64(0)
	if kv != nil {
		version = kv.ModifiedIndex
	}
	if kv == nil || bytes.Compare(kv.Value, []byte("{}")) == 0 {
		logrus.Infoln("Cluster is uninitialized...")
		return db, version, nil
	}
	if err := json.Unmarshal(kv.Value, &db); err != nil {
		logrus.Warnln("Fatal, Could not parse cluster database ", kv)
		return db, version, err
	}
	return db, version, nil
}

func readClusterInfo() (cluster.ClusterInfo, uint64, error) {
	kv, err := kvdb.Instance().Get(ClusterDBKey)
	if err != nil && !strings.Contains(err.Error(), "Key not found") {
		logrus.Warnln("Warning, could not read cluster database")
		return emptyClusterInfo(), 0, err
	}
	return unmarshalClusterInfo(kv)
}

func lockAndUpdateDB(fn, lockID string, cb updateCallbackFn) (*kvdb.KVPair, error) {
	kv := kvdb.Instance()

	kvlock, err := kv.LockWithTimeout(clusterLockKey, lockID, 10 * time.Minute, kv.GetLockTimeout())
	if err != nil {
		logrus.Warnf("Unable to obtain cluster lock for %v op: %v", fn, err)
		return nil, err
	}
	defer kv.Unlock(kvlock)

	db, version, err := readClusterInfo()
	if err != nil {
		return nil, err
	}

	update, err := cb(&db)
	if err != nil {
		return nil, err
	}
	if !update {
		return nil, nil
	}

	if version != 0 {
		b, err := json.Marshal(db)
		if err != nil {
			logrus.Warnf("Fatal, Could not marshal cluster database to JSON: %v", err)
			return nil, err
		}

		kvpInput := &kvdb.KVPair{
			Key:           ClusterDBKey,
			Value:         b,
			ModifiedIndex: version,
		}

		return kv.CompareAndSet(kvpInput, kvdb.KVModifiedIndex, nil)
	} else {
		// key does not exists yet, call create
		kvp, err := kv.Create(ClusterDBKey, db, 0)
		if err == kvdb.ErrExist {
			// key created, we may have lost the lock, retry
			err = kvdb.ErrModified
		}
		return kvp, err
	}
}

func updateDB(fn, lockID string, cb updateCallbackFn) error {
	_, err := updateAndGetDB(fn, lockID, cb, true)
	return err
}

func updateAndGetDB(
	fn string,
	lockID string,
	cb updateCallbackFn,
	retry bool,
) (*kvdb.KVPair, error) {
	for {
		if kvp, err := lockAndUpdateDB(fn, lockID, cb); err == kvdb.ErrModified && retry {
			// compare and set failed, retry
			continue
		} else {
			return kvp, err
		}
	}
}
<|endoftext|>