<commit_before>package log

import (
	"encoding/json"
	"fmt"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/fatih/camelcase"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
)

const (
	bucketFactor = 2.0
	bucketCount  = 20 // Which makes the max bucket 2^20 seconds or ~12 days in size
)

// This needs to be a global var, not a field on the logger, because multiple servers
// create new loggers, and the prometheus registration uses a global namespace
var reportMetricGauge prometheus.Gauge
var reportMetricsOnce sync.Once

// Logger is a helper for emitting our grpc API logs
type Logger interface {
	Log(request interface{}, response interface{}, err error, duration time.Duration)
	LogAtLevelFromDepth(request interface{}, response interface{}, err error, duration time.Duration, level logrus.Level, depth int)
}

type logger struct {
	*logrus.Entry
	histogram   map[string]*prometheus.HistogramVec
	counter     map[string]prometheus.Counter
	mutex       *sync.Mutex
	exportStats bool
	service     string
}

// NewLogger creates a new logger
func NewLogger(service string) Logger {
	return newLogger(service, true)
}

// NewLocalLogger creates a new logger for local testing (which does not report prometheus metrics)
func NewLocalLogger(service string) Logger {
	return newLogger(service, false)
}

func newLogger(service string, exportStats bool) Logger {
	l := logrus.New()
	l.Formatter = new(prettyFormatter)
	newLogger := &logger{
		l.WithFields(logrus.Fields{"service": service}),
		make(map[string]*prometheus.HistogramVec),
		make(map[string]prometheus.Counter),
		&sync.Mutex{},
		exportStats,
		service,
	}
	if exportStats {
		reportMetricsOnce.Do(func() {
			newReportMetricGauge := prometheus.NewGauge(
				prometheus.GaugeOpts{
					Namespace: "pachyderm",
					Subsystem: "pachd",
					Name:      "report_metric",
					Help:      "gauge of number of calls to ReportMetric()",
				},
			)
			if err := prometheus.Register(newReportMetricGauge); err != nil {
				entry := newLogger.WithFields(logrus.Fields{"method": "NewLogger"})
				newLogger.LogAtLevel(entry, logrus.WarnLevel, fmt.Sprintf("error registering prometheus metric: %v", newReportMetricGauge), err)
			} else {
				reportMetricGauge = newReportMetricGauge
			}
		})
	}
	return newLogger
}

// Helper function used to log requests and responses from our GRPC method
// implementations
func (l *logger) Log(request interface{}, response interface{}, err error, duration time.Duration) {
	if err != nil {
		l.LogAtLevelFromDepth(request, response, err, duration, logrus.ErrorLevel, 4)
	} else {
		l.LogAtLevelFromDepth(request, response, err, duration, logrus.InfoLevel, 4)
	}
	// We have to grab the method's name here before we
	// enter the goro's stack
	go l.ReportMetric(getMethodName(), duration, err)
}

func getMethodName() string {
	depth := 4
	pc := make([]uintptr, depth)
	runtime.Callers(depth, pc)
	split := strings.Split(runtime.FuncForPC(pc[0]).Name(), ".")
	return split[len(split)-1]
}

func (l *logger) ReportMetric(method string, duration time.Duration, err error) {
	if !l.exportStats {
		return
	}
	// Count the number of ReportMetric() goros in case we start to leak them
	if reportMetricGauge != nil {
		reportMetricGauge.Inc()
	}
	defer func() {
		if reportMetricGauge != nil {
			reportMetricGauge.Dec()
		}
	}()
	l.mutex.Lock() // for conccurent map access (histogram,counter)
	defer l.mutex.Unlock()
	state := "started"
	if err != nil {
		state = "errored"
	} else {
		if duration.Seconds() > 0 {
			state = "finished"
		}
	}
	entry := l.WithFields(logrus.Fields{"method": method})

	var tokens []string
	for _, token := range camelcase.Split(method) {
		tokens = append(tokens, strings.ToLower(token))
	}
	rootStatName := strings.Join(tokens, "_")

	// Recording the distribution of started times is meaningless
	if state != "started" {
		runTimeName := fmt.Sprintf("%v_time", rootStatName)
		runTime, ok := l.histogram[runTimeName]
		if !ok {
			runTime = prometheus.NewHistogramVec(
				prometheus.HistogramOpts{
					Namespace: "pachyderm",
					Subsystem: fmt.Sprintf("pachd_%v", l.service),
					Name:      runTimeName,
					Help:      fmt.Sprintf("Run time of %v", method),
					Buckets:   prometheus.ExponentialBuckets(1.0, bucketFactor, bucketCount),
				},
				[]string{
					"state", // Since both finished and errored API calls can have run times
				},
			)
			if err := prometheus.Register(runTime); err != nil {
				l.LogAtLevel(entry, logrus.WarnLevel, fmt.Sprintf("error registering prometheus metric: %v", runTimeName), err)
			} else {
				l.histogram[runTimeName] = runTime
			}
		}
		if hist, err := runTime.GetMetricWithLabelValues(state); err != nil {
			l.LogAtLevel(entry, logrus.WarnLevel, "failed to get histogram w labels: state (%v) with error %v", state, err)
		} else {
			hist.Observe(duration.Seconds())
		}
	}

	secondsCountName := fmt.Sprintf("%v_seconds_count", rootStatName)
	secondsCount, ok := l.counter[secondsCountName]
	if !ok {
		secondsCount = prometheus.NewCounter(
			prometheus.CounterOpts{
				Namespace: "pachyderm",
				Subsystem: fmt.Sprintf("pachd_%v", l.service),
				Name:      secondsCountName,
				Help:      fmt.Sprintf("cumulative number of seconds spent in %v", method),
			},
		)
		if err := prometheus.Register(secondsCount); err != nil {
			l.LogAtLevel(entry, logrus.WarnLevel, fmt.Sprintf("error registering prometheus metric: %v", secondsCountName), err)
		} else {
			l.counter[secondsCountName] = secondsCount
		}
	}
	secondsCount.Add(duration.Seconds())

}

func (l *logger) LogAtLevel(entry *logrus.Entry, level logrus.Level, args ...interface{}) {
	switch level {
	case logrus.PanicLevel:
		entry.Panic(args)
	case logrus.FatalLevel:
		entry.Fatal(args)
	case logrus.ErrorLevel:
		entry.Error(args)
	case logrus.WarnLevel:
		entry.Warn(args)
	case logrus.InfoLevel:
		entry.Info(args)
	case logrus.DebugLevel:
		entry.Debug(args)
	}
}

func (l *logger) LogAtLevelFromDepth(request interface{}, response interface{}, err error, duration time.Duration, level logrus.Level, depth int) {
	pc := make([]uintptr, depth)
	runtime.Callers(depth, pc)
	split := strings.Split(runtime.FuncForPC(pc[0]).Name(), ".")
	method := split[len(split)-1]

	fields := logrus.Fields{
		"method":  method,
		"request": request,
	}
	if response != nil {
		fields["response"] = response
	}
	if err != nil {
		// "err" itself might be a code or even an empty struct
		fields["error"] = err.Error()
	}
	if duration > 0 {
		fields["duration"] = duration
	}
	l.LogAtLevel(l.WithFields(fields), level)
}

type prettyFormatter struct{}

func (f *prettyFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	serialized := []byte(
		fmt.Sprintf(
			"%v %v ",
			entry.Time.Format(logrus.DefaultTimestampFormat),
			strings.ToUpper(entry.Level.String()),
		),
	)
	if entry.Data["service"] != nil {
		serialized = append(serialized, []byte(fmt.Sprintf("%v.%v ", entry.Data["service"], entry.Data["method"]))...)
	}
	if len(entry.Data) > 2 {
		delete(entry.Data, "service")
		delete(entry.Data, "method")
		if entry.Data["duration"] != nil {
			entry.Data["duration"] = entry.Data["duration"].(time.Duration).Seconds()
		}
		data, err := json.Marshal(entry.Data)
		if err != nil {
			return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
		}
		serialized = append(serialized, []byte(string(data))...)
		serialized = append(serialized, ' ')
	}

	serialized = append(serialized, []byte(entry.Message)...)
	serialized = append(serialized, '\n')
	return serialized, nil
}
<commit_msg>Prometheus subservice must be a single word<commit_after>package log

import (
	"encoding/json"
	"fmt"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/fatih/camelcase"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
)

const (
	bucketFactor = 2.0
	bucketCount  = 20 // Which makes the max bucket 2^20 seconds or ~12 days in size
)

// This needs to be a global var, not a field on the logger, because multiple servers
// create new loggers, and the prometheus registration uses a global namespace
var reportMetricGauge prometheus.Gauge
var reportMetricsOnce sync.Once

// Logger is a helper for emitting our grpc API logs
type Logger interface {
	Log(request interface{}, response interface{}, err error, duration time.Duration)
	LogAtLevelFromDepth(request interface{}, response interface{}, err error, duration time.Duration, level logrus.Level, depth int)
}

type logger struct {
	*logrus.Entry
	histogram   map[string]*prometheus.HistogramVec
	counter     map[string]prometheus.Counter
	mutex       *sync.Mutex
	exportStats bool
	service     string
}

// NewLogger creates a new logger
func NewLogger(service string) Logger {
	return newLogger(service, true)
}

// NewLocalLogger creates a new logger for local testing (which does not report prometheus metrics)
func NewLocalLogger(service string) Logger {
	return newLogger(service, false)
}

func newLogger(service string, exportStats bool) Logger {
	l := logrus.New()
	l.Formatter = new(prettyFormatter)
	newLogger := &logger{
		l.WithFields(logrus.Fields{"service": service}),
		make(map[string]*prometheus.HistogramVec),
		make(map[string]prometheus.Counter),
		&sync.Mutex{},
		exportStats,
		service,
	}
	if exportStats {
		reportMetricsOnce.Do(func() {
			newReportMetricGauge := prometheus.NewGauge(
				prometheus.GaugeOpts{
					Namespace: "pachyderm",
					Subsystem: "pachd",
					Name:      "report_metric",
					Help:      "gauge of number of calls to ReportMetric()",
				},
			)
			if err := prometheus.Register(newReportMetricGauge); err != nil {
				entry := newLogger.WithFields(logrus.Fields{"method": "NewLogger"})
				newLogger.LogAtLevel(entry, logrus.WarnLevel, fmt.Sprintf("error registering prometheus metric: %v", newReportMetricGauge), err)
			} else {
				reportMetricGauge = newReportMetricGauge
			}
		})
	}
	return newLogger
}

// Helper function used to log requests and responses from our GRPC method
// implementations
func (l *logger) Log(request interface{}, response interface{}, err error, duration time.Duration) {
	if err != nil {
		l.LogAtLevelFromDepth(request, response, err, duration, logrus.ErrorLevel, 4)
	} else {
		l.LogAtLevelFromDepth(request, response, err, duration, logrus.InfoLevel, 4)
	}
	// We have to grab the method's name here before we
	// enter the goro's stack
	go l.ReportMetric(getMethodName(), duration, err)
}

func getMethodName() string {
	depth := 4
	pc := make([]uintptr, depth)
	runtime.Callers(depth, pc)
	split := strings.Split(runtime.FuncForPC(pc[0]).Name(), ".")
	return split[len(split)-1]
}

func (l *logger) ReportMetric(method string, duration time.Duration, err error) {
	if !l.exportStats {
		return
	}
	// Count the number of ReportMetric() goros in case we start to leak them
	if reportMetricGauge != nil {
		reportMetricGauge.Inc()
	}
	defer func() {
		if reportMetricGauge != nil {
			reportMetricGauge.Dec()
		}
	}()
	l.mutex.Lock() // for conccurent map access (histogram,counter)
	defer l.mutex.Unlock()
	state := "started"
	if err != nil {
		state = "errored"
	} else {
		if duration.Seconds() > 0 {
			state = "finished"
		}
	}
	entry := l.WithFields(logrus.Fields{"method": method})

	var tokens []string
	for _, token := range camelcase.Split(method) {
		tokens = append(tokens, strings.ToLower(token))
	}
	rootStatName := strings.Join(tokens, "_")

	// Recording the distribution of started times is meaningless
	if state != "started" {
		runTimeName := fmt.Sprintf("%v_time", rootStatName)
		runTime, ok := l.histogram[runTimeName]
		if !ok {
			runTime = prometheus.NewHistogramVec(
				prometheus.HistogramOpts{
					Namespace: "pachyderm",
					Subsystem: l.service,
					Name:      runTimeName,
					Help:      fmt.Sprintf("Run time of %v", method),
					Buckets:   prometheus.ExponentialBuckets(1.0, bucketFactor, bucketCount),
				},
				[]string{
					"state", // Since both finished and errored API calls can have run times
				},
			)
			if err := prometheus.Register(runTime); err != nil {
				l.LogAtLevel(entry, logrus.WarnLevel, fmt.Sprintf("error registering prometheus metric: %v", runTimeName), err)
			} else {
				l.histogram[runTimeName] = runTime
			}
		}
		if hist, err := runTime.GetMetricWithLabelValues(state); err != nil {
			l.LogAtLevel(entry, logrus.WarnLevel, "failed to get histogram w labels: state (%v) with error %v", state, err)
		} else {
			hist.Observe(duration.Seconds())
		}
	}

	secondsCountName := fmt.Sprintf("%v_seconds_count", rootStatName)
	secondsCount, ok := l.counter[secondsCountName]
	if !ok {
		secondsCount = prometheus.NewCounter(
			prometheus.CounterOpts{
				Namespace: "pachyderm",
				Subsystem: l.service,
				Name:      secondsCountName,
				Help:      fmt.Sprintf("cumulative number of seconds spent in %v", method),
			},
		)
		if err := prometheus.Register(secondsCount); err != nil {
			l.LogAtLevel(entry, logrus.WarnLevel, fmt.Sprintf("error registering prometheus metric: %v", secondsCountName), err)
		} else {
			l.counter[secondsCountName] = secondsCount
		}
	}
	secondsCount.Add(duration.Seconds())

}

func (l *logger) LogAtLevel(entry *logrus.Entry, level logrus.Level, args ...interface{}) {
	switch level {
	case logrus.PanicLevel:
		entry.Panic(args)
	case logrus.FatalLevel:
		entry.Fatal(args)
	case logrus.ErrorLevel:
		entry.Error(args)
	case logrus.WarnLevel:
		entry.Warn(args)
	case logrus.InfoLevel:
		entry.Info(args)
	case logrus.DebugLevel:
		entry.Debug(args)
	}
}

func (l *logger) LogAtLevelFromDepth(request interface{}, response interface{}, err error, duration time.Duration, level logrus.Level, depth int) {
	pc := make([]uintptr, depth)
	runtime.Callers(depth, pc)
	split := strings.Split(runtime.FuncForPC(pc[0]).Name(), ".")
	method := split[len(split)-1]

	fields := logrus.Fields{
		"method":  method,
		"request": request,
	}
	if response != nil {
		fields["response"] = response
	}
	if err != nil {
		// "err" itself might be a code or even an empty struct
		fields["error"] = err.Error()
	}
	if duration > 0 {
		fields["duration"] = duration
	}
	l.LogAtLevel(l.WithFields(fields), level)
}

type prettyFormatter struct{}

func (f *prettyFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	serialized := []byte(
		fmt.Sprintf(
			"%v %v ",
			entry.Time.Format(logrus.DefaultTimestampFormat),
			strings.ToUpper(entry.Level.String()),
		),
	)
	if entry.Data["service"] != nil {
		serialized = append(serialized, []byte(fmt.Sprintf("%v.%v ", entry.Data["service"], entry.Data["method"]))...)
	}
	if len(entry.Data) > 2 {
		delete(entry.Data, "service")
		delete(entry.Data, "method")
		if entry.Data["duration"] != nil {
			entry.Data["duration"] = entry.Data["duration"].(time.Duration).Seconds()
		}
		data, err := json.Marshal(entry.Data)
		if err != nil {
			return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
		}
		serialized = append(serialized, []byte(string(data))...)
		serialized = append(serialized, ' ')
	}

	serialized = append(serialized, []byte(entry.Message)...)
	serialized = append(serialized, '\n')
	return serialized, nil
}
<|endoftext|>


<commit_before>// Copyright 2012, The gohg Authors. All rights reserved.
// Use of this source code is governed by a BSD style license
// that can be found in the LICENSE file.

package gohg

import (
	"errors"
	"fmt"
	"path/filepath"
	"strings"
)

func Command(hgcl *HgClient, cmd string, opts []string) (data []byte, err error) {
	// boilerplate code for all commands

	cmdline := PrependStringToSlice(cmd, opts)
	data, hgerr, ret, err := hgcl.run(cmdline)
	if err != nil {
		return nil, fmt.Errorf("from hgcl.run(): %s", err)
	}
	if ret != 0 || hgerr != nil {
		return nil, fmt.Errorf("%s(): returncode=%d\nhgerr:\n%s\n",
			strings.Title(cmd), data, string(hgerr))
	}
	return data, nil
}

// Add provides the 'hg add' command.
func (hgcl *HgClient) Add(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "add", opts)
	return data, err
}

// Identify provides the 'hg identify' command.
func (hgcl *HgClient) Identify(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "identify", opts)
	return data, err
}

// TODO	Implement the flags for hg init.

// Init provides the 'hg init' command.
//
// Be aware of the fact that it cannot be used to initialize the repo you want
// the (current) Hg CS to work on, as the Hg CS requires an existing repo.
// But Init() can be used to create any new repo outside the one the Hg CS is
// running for.
func (hgcl *HgClient) Init(path string, opts []string) error {
	var err1 error
	var fa string
	fa, err1 = filepath.Abs(path)
	if err1 != nil {
		return fmt.Errorf("Init() -> filepath.Abs(): %s", err1)
	}
	if path == "" || path == "." || fa == hgcl.RepoRoot() {
		return errors.New("HgClient.Init: path for new repo must be different" +
			" from the Command Server repo path")
	}

	allopts := PrependStringToSlice(fa, []string{})
	_, err := Command(hgcl, "init", allopts)
	return err
}

// Add provides the 'hg log' command.
func (hgcl *HgClient) Log(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "log", opts)
	return data, err
}

// Status provides the 'hg status' command.
func (hgcl *HgClient) Status(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "status", opts)
	return data, err
}

// Summary provides the 'hg summary' command.
func (hgcl *HgClient) Summary(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "summary", opts)
	return data, err
}

// Version implements the 'hg version -q' command,
// and only returns the version number.
func (hgcl *HgClient) Version() (string, error) {
	var err error
	if hgcl.hgVersion == "" {
		var data []byte
		data, err = Command(hgcl, "version", []string{"-q"})
		if err == nil {
			ver := strings.Split(string(data), "\n")[0]
			ver = ver[strings.LastIndex(ver, " ")+1 : len(ver)-1]
			hgcl.hgVersion = ver
		}
	}
	return hgcl.hgVersion, err
}
<commit_msg>commands: correction in Command()<commit_after>// Copyright 2012, The gohg Authors. All rights reserved.
// Use of this source code is governed by a BSD style license
// that can be found in the LICENSE file.

package gohg

import (
	"errors"
	"fmt"
	"path/filepath"
	"strings"
)

func Command(hgcl *HgClient, cmd string, opts []string) (data []byte, err error) {
	// boilerplate code for all commands

	cmdline := PrependStringToSlice(cmd, opts)
	data, hgerr, ret, err := hgcl.run(cmdline)
	if err != nil {
		return nil, fmt.Errorf("from hgcl.run(): %s", err)
	}
	if ret != 0 || hgerr != nil {
		return nil, fmt.Errorf("%s(): returncode=%d\nhgerr:\n%s\n",
			strings.Title(cmd), ret, string(hgerr))
	}
	return data, nil
}

// Add provides the 'hg add' command.
func (hgcl *HgClient) Add(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "add", opts)
	return data, err
}

// Identify provides the 'hg identify' command.
func (hgcl *HgClient) Identify(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "identify", opts)
	return data, err
}

// TODO	Implement the flags for hg init.

// Init provides the 'hg init' command.
//
// Be aware of the fact that it cannot be used to initialize the repo you want
// the (current) Hg CS to work on, as the Hg CS requires an existing repo.
// But Init() can be used to create any new repo outside the one the Hg CS is
// running for.
func (hgcl *HgClient) Init(path string, opts []string) error {
	var err1 error
	var fa string
	fa, err1 = filepath.Abs(path)
	if err1 != nil {
		return fmt.Errorf("Init() -> filepath.Abs(): %s", err1)
	}
	if path == "" || path == "." || fa == hgcl.RepoRoot() {
		return errors.New("HgClient.Init: path for new repo must be different" +
			" from the Command Server repo path")
	}

	allopts := PrependStringToSlice(fa, []string{})
	_, err := Command(hgcl, "init", allopts)
	return err
}

// Add provides the 'hg log' command.
func (hgcl *HgClient) Log(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "log", opts)
	return data, err
}

// Status provides the 'hg status' command.
func (hgcl *HgClient) Status(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "status", opts)
	return data, err
}

// Summary provides the 'hg summary' command.
func (hgcl *HgClient) Summary(opts []string) ([]byte, error) {
	data, err := Command(hgcl, "summary", opts)
	return data, err
}

// Version implements the 'hg version -q' command,
// and only returns the version number.
func (hgcl *HgClient) Version() (string, error) {
	var err error
	if hgcl.hgVersion == "" {
		var data []byte
		data, err = Command(hgcl, "version", []string{"-q"})
		if err == nil {
			ver := strings.Split(string(data), "\n")[0]
			ver = ver[strings.LastIndex(ver, " ")+1 : len(ver)-1]
			hgcl.hgVersion = ver
		}
	}
	return hgcl.hgVersion, err
}
<|endoftext|>


<commit_before>// Copyright 2017 Javier Arevalo <jare@iguanademos.com>

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Cross platform command execution and some file system operations

package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
)

// RunShell runs an interactive shell.
// Since the current program is still running, so may be
// its coroutines, which will trigger all sorts of weirdness
// For example, Ctrl-C may be caught by our Go runtime, killing us
// but leaving the spawned shell still running, with I/O shared with
// our parent (possibly another shell!). That's pretty ugly.
func RunShell(cwd string) error {
	attr := os.ProcAttr{
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
		Dir:   cwd,
	}
	var args []string
	shell, ok := os.LookupEnv("COMSPEC")
	if !ok {
		args = []string{"-i"}
		shell, ok = os.LookupEnv("SHELL")
		if !ok {
			shell = "/bin/sh"
		}
	}
	process, err := os.StartProcess(shell, args, &attr)
	if err != nil {
		return err
	}
	state, err := process.Wait()
	if err != nil {
		return err
	}
	if state.Success() {
		return nil
	}
	return fmt.Errorf("<< Exited shell: %s", state.String())
}

// RunCommand executes the given command and arguments under the system'
// default shell. Really only tested under CMD and ksh
// If command fails, returns the system error and the output of the command
func RunCommand(command string, args ...string) error {
	var ret []string
	var finalargs []string
	shell, ok := os.LookupEnv("COMSPEC")
	if ok {
		finalargs = append([]string{"/C"}, command)
		finalargs = append(finalargs, args...)
		finalargs = append(finalargs, "2>&1")
	} else {
		shell, ok = os.LookupEnv("SHELL")
		if !ok {
			shell = "/bin/sh"
		}
		for i, v := range args {
			args[i] = strconv.Quote(v)
		}
		args = append([]string{command}, args...)
		finalargs = []string{"-c", strings.Join(args, " ") + " 2>&1"}
	}
	fmt.Fprintf(os.Stderr, "Running command:\n>%s<\n>>%s<<\n", shell, strings.Join(finalargs, "<<\n>>"))
	cmd := exec.Command(shell, finalargs...)

	outp, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("Failed to attach stdout: %v", err)
	}
	cmd.Start()
	scanner := bufio.NewScanner(outp)
	for scanner.Scan() {
		line := scanner.Text()
		ret = append(ret, line)
	}
	err = cmd.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s: %s\n", err, strings.Join(ret, "\n"))
		return fmt.Errorf("%s: %s", err, strings.Join(ret, "\n"))
	}
	return nil
}

// CommandCopy copies a given file or folder into the target folder
// Does not verify that the target folder exists nor if
// it is in fact a folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandCopy(src string, dst string) error {
	dst = filepath.Clean(dst)
	if dst[len(dst)-1] == os.PathSeparator {
		return fmt.Errorf("Copy to root folder %s not allowed for safety", dst)
	}
	dst += string(os.PathSeparator)
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// We end up using xcopy because copy will NOT handle hidden files ever
		stat, err := os.Stat(src)
		if err != nil {
			return err
		}
		if stat.IsDir() {
			fullDest := filepath.Join(dst, filepath.Base(src))
			return RunCommand("xcopy", "/Q", "/I", "/K", "/H", "/Y", "/R", "/S", "/E", src, fullDest)
		}
		return RunCommand("xcopy", "/Q", "/K", "/H", "/Y", "/R", src, dst)
	}
	return RunCommand("cp", "-R", src, dst)
}

// CommandMove moves a given file or folder into the target folder
// Does not verify that the target folder exists nor if
// it is in fact a folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandMove(src string, dst string) error {
	dst = filepath.Clean(dst)
	if dst[len(dst)-1] == os.PathSeparator {
		return fmt.Errorf("Move to root folder %s not allowed for safety", dst)
	}
	dst += string(os.PathSeparator)
	dir := filepath.Dir(src)
	if dir[len(dir)-1] == os.PathSeparator {
		return fmt.Errorf("Moving %s from root folder not allowed for safety", dst)
	}
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// hidden files will wreak havoc with move across devices
		err := RunCommand("move", "/Y", src, dst)
		if err != nil {
			// So if we get any errors we retry via copy & delete
			err = CommandCopy(src, dst)
			if err == nil {
				err = CommandDelete(src)
			}
		}
		return err
	}
	return RunCommand("mv", "-f", src, dst)
}

// CommandDelete deletes a given file or folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandDelete(dst string) error {
	dst = filepath.Clean(dst)
	dir := filepath.Dir(dst)
	if dir[len(dir)-1] == os.PathSeparator {
		return fmt.Errorf("Deleting %s from root folder not allowed for safety", dst)
	}
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// Deleting files in directories and deleting directories are two
		// separate things :(
		err := RunCommand("del", "/Q", "/A", "/F", dst)
		if err == nil {
			// Must ignore the error because dst may have been fully deleted by prev command
			// UGH
			/*err = */
			RunCommand("rd", "/S", "/Q", dst)
		}
		return err
	}
	return RunCommand("rm", "-rf", dst)
}
<commit_msg>Remove traces<commit_after>// Copyright 2017 Javier Arevalo <jare@iguanademos.com>

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Cross platform command execution and some file system operations

package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
)

// RunShell runs an interactive shell.
// Since the current program is still running, so may be
// its coroutines, which will trigger all sorts of weirdness
// For example, Ctrl-C may be caught by our Go runtime, killing us
// but leaving the spawned shell still running, with I/O shared with
// our parent (possibly another shell!). That's pretty ugly.
func RunShell(cwd string) error {
	attr := os.ProcAttr{
		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
		Dir:   cwd,
	}
	var args []string
	shell, ok := os.LookupEnv("COMSPEC")
	if !ok {
		args = []string{"-i"}
		shell, ok = os.LookupEnv("SHELL")
		if !ok {
			shell = "/bin/sh"
		}
	}
	process, err := os.StartProcess(shell, args, &attr)
	if err != nil {
		return err
	}
	state, err := process.Wait()
	if err != nil {
		return err
	}
	if state.Success() {
		return nil
	}
	return fmt.Errorf("<< Exited shell: %s", state.String())
}

// RunCommand executes the given command and arguments under the system'
// default shell. Really only tested under CMD and ksh
// If command fails, returns the system error and the output of the command
func RunCommand(command string, args ...string) error {
	var ret []string
	var finalargs []string
	shell, ok := os.LookupEnv("COMSPEC")
	if ok {
		finalargs = append([]string{"/C"}, command)
		finalargs = append(finalargs, args...)
		finalargs = append(finalargs, "2>&1")
	} else {
		shell, ok = os.LookupEnv("SHELL")
		if !ok {
			shell = "/bin/sh"
		}
		for i, v := range args {
			args[i] = strconv.Quote(v)
		}
		args = append([]string{command}, args...)
		finalargs = []string{"-c", strings.Join(args, " ") + " 2>&1"}
	}
	// fmt.Fprintf(os.Stderr, "Running command:\n>%s<\n>>%s<<\n", shell, strings.Join(finalargs, "<<\n>>"))
	cmd := exec.Command(shell, finalargs...)

	outp, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("Failed to attach stdout: %v", err)
	}
	cmd.Start()
	scanner := bufio.NewScanner(outp)
	for scanner.Scan() {
		line := scanner.Text()
		ret = append(ret, line)
	}
	err = cmd.Wait()
	if err != nil {
		// fmt.Fprintf(os.Stderr, "ERROR: %s: %s\n", err, strings.Join(ret, "\n"))
		return fmt.Errorf("%s: %s", err, strings.Join(ret, "\n"))
	}
	return nil
}

// CommandCopy copies a given file or folder into the target folder
// Does not verify that the target folder exists nor if
// it is in fact a folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandCopy(src string, dst string) error {
	dst = filepath.Clean(dst)
	if dst[len(dst)-1] == os.PathSeparator {
		return fmt.Errorf("Copy to root folder %s not allowed for safety", dst)
	}
	dst += string(os.PathSeparator)
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// We end up using xcopy because copy will NOT handle hidden files ever
		stat, err := os.Stat(src)
		if err != nil {
			return err
		}
		if stat.IsDir() {
			fullDest := filepath.Join(dst, filepath.Base(src))
			return RunCommand("xcopy", "/Q", "/I", "/K", "/H", "/Y", "/R", "/S", "/E", src, fullDest)
		}
		return RunCommand("xcopy", "/Q", "/K", "/H", "/Y", "/R", src, dst)
	}
	return RunCommand("cp", "-R", src, dst)
}

// CommandMove moves a given file or folder into the target folder
// Does not verify that the target folder exists nor if
// it is in fact a folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandMove(src string, dst string) error {
	dst = filepath.Clean(dst)
	if dst[len(dst)-1] == os.PathSeparator {
		return fmt.Errorf("Move to root folder %s not allowed for safety", dst)
	}
	dst += string(os.PathSeparator)
	dir := filepath.Dir(src)
	if dir[len(dir)-1] == os.PathSeparator {
		return fmt.Errorf("Moving %s from root folder not allowed for safety", dst)
	}
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// hidden files will wreak havoc with move across devices
		err := RunCommand("move", "/Y", src, dst)
		if err != nil {
			// So if we get any errors we retry via copy & delete
			err = CommandCopy(src, dst)
			if err == nil {
				err = CommandDelete(src)
			}
		}
		return err
	}
	return RunCommand("mv", "-f", src, dst)
}

// CommandDelete deletes a given file or folder
// Fails if the target is the root folder
// If command fails, returns the system error and the output of the command
func CommandDelete(dst string) error {
	dst = filepath.Clean(dst)
	dir := filepath.Dir(dst)
	if dir[len(dir)-1] == os.PathSeparator {
		return fmt.Errorf("Deleting %s from root folder not allowed for safety", dst)
	}
	// Many safety checks to perform here...
	if runtime.GOOS == "windows" {
		// Deleting files in directories and deleting directories are two
		// separate things :(
		err := RunCommand("del", "/Q", "/A", "/F", dst)
		if err == nil {
			// Must ignore the error because dst may have been fully deleted by prev command
			// UGH
			/*err = */
			RunCommand("rd", "/S", "/Q", dst)
		}
		return err
	}
	return RunCommand("rm", "-rf", dst)
}
<|endoftext|>


<commit_before>package models

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/technoweenie/grohl"
)

type IMagick struct{}

// Process a remote asset url using graphicsmagick with the args supplied
// and write the response to w
func (p *IMagick) Process(w http.ResponseWriter, r *http.Request, args *ProcessArgs) (err error) {
	tempDir, err := createTemporaryWorkspace()
	if err != nil {
		return
	}
	// defer os.RemoveAll(tempDir)

	inFile, err := downloadRemote(tempDir, args.Url)
	if err != nil {
		return
	}

	outFile, err := processImage(tempDir, inFile, args)
	if err != nil {
		return
	}

	// serve response
	http.ServeFile(w, r, outFile)
	return
}

func createTemporaryWorkspace() (string, error) {
	return ioutil.TempDir("", "_firesize")
}

func downloadRemote(tempDir string, url string) (string, error) {
	inFile := filepath.Join(tempDir, "in")

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"download":  url,
		"local":     inFile,
	})

	out, err := os.Create(inFile)
	if err != nil {
		return inFile, err
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		return inFile, err
	}
	defer resp.Body.Close()

	_, err = io.Copy(out, resp.Body)

	return inFile, err
}

func processImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	outFile := filepath.Join(tempDir, "out")
	cmdArgs, outFileWithFormat := args.CommandArgs(inFile, outFile)

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"args":      cmdArgs,
	})

	executable := "convert"
	cmd := exec.Command(executable, cmdArgs...)
	outErr, err := runWithTimeout(cmd, 60*time.Second)
	if err != nil {
		grohl.Log(grohl.Data{
			"processor": "imagick",
			"failure":   err,
			"args":      cmdArgs,
			"output":    string(outErr),
		})
	}

	return outFileWithFormat, err
}

func runWithTimeout(cmd *exec.Cmd, timeout time.Duration) ([]byte, error) {
	// Capture the output
	var b bytes.Buffer
	cmd.Stdout, cmd.Stderr = &b, &b

	// Start the process
	err := cmd.Start()
	if err != nil {
		return nil, err
	}

	// Kill the process if it doesn't exit in time
	defer time.AfterFunc(timeout, func() {
		fmt.Println("command timed out")
		cmd.Process.Kill()
	}).Stop()

	// Wait for the process to finish
	if err := cmd.Wait(); err != nil {
		return b.Bytes(), err
	}

	return b.Bytes(), nil
}
<commit_msg>put in a preProcessImage()<commit_after>package models

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/technoweenie/grohl"
)

type IMagick struct{}

// Process a remote asset url using graphicsmagick with the args supplied
// and write the response to w
func (p *IMagick) Process(w http.ResponseWriter, r *http.Request, args *ProcessArgs) (err error) {
	tempDir, err := createTemporaryWorkspace()
	if err != nil {
		return
	}
	// defer os.RemoveAll(tempDir)

	inFile, err := downloadRemote(tempDir, args.Url)
	if err != nil {
		return
	}

	preProcessedInFile, err := preProcessImage(tempDir, inFile)
	if err != nil {
		return
	}

	outFile, err := processImage(tempDir, preProcessedInFile, args)
	if err != nil {
		return
	}

	// serve response
	http.ServeFile(w, r, outFile)
	return
}

func createTemporaryWorkspace() (string, error) {
	return ioutil.TempDir("", "_firesize")
}

func downloadRemote(tempDir string, url string) (string, error) {
	inFile := filepath.Join(tempDir, "in")

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"download":  url,
		"local":     inFile,
	})

	out, err := os.Create(inFile)
	if err != nil {
		return inFile, err
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		return inFile, err
	}
	defer resp.Body.Close()

	_, err = io.Copy(out, resp.Body)

	return inFile, err
}

func preProcessImage(tempDir string, inFile string) (string, error) {
	return inFile, nil
}

func processImage(tempDir string, inFile string, args *ProcessArgs) (string, error) {
	outFile := filepath.Join(tempDir, "out")
	cmdArgs, outFileWithFormat := args.CommandArgs(inFile, outFile)

	grohl.Log(grohl.Data{
		"processor": "imagick",
		"args":      cmdArgs,
	})

	executable := "convert"
	cmd := exec.Command(executable, cmdArgs...)
	outErr, err := runWithTimeout(cmd, 60*time.Second)
	if err != nil {
		grohl.Log(grohl.Data{
			"processor": "imagick",
			"failure":   err,
			"args":      cmdArgs,
			"output":    string(outErr),
		})
	}

	return outFileWithFormat, err
}

func runWithTimeout(cmd *exec.Cmd, timeout time.Duration) ([]byte, error) {
	// Capture the output
	var b bytes.Buffer
	cmd.Stdout, cmd.Stderr = &b, &b

	// Start the process
	err := cmd.Start()
	if err != nil {
		return nil, err
	}

	// Kill the process if it doesn't exit in time
	defer time.AfterFunc(timeout, func() {
		fmt.Println("command timed out")
		cmd.Process.Kill()
	}).Stop()

	// Wait for the process to finish
	if err := cmd.Wait(); err != nil {
		return b.Bytes(), err
	}

	return b.Bytes(), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gcp implements upspin.StoreServer using Google Cloud Platform as its storage.
package gcp

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"strings"
	"sync"

	"upspin.io/cloud/storage"
	"upspin.io/errors"
	"upspin.io/key/sha256key"
	"upspin.io/log"
	"upspin.io/metric"
	"upspin.io/store/gcp/cache"
	"upspin.io/upspin"

	// We use GCS as the backing for our data.
	_ "upspin.io/cloud/storage/gcs"
)

// Configuration options for this package.
const (
	// ConfigTemporaryDir specifies which temporary directory to write files to before they're
	// uploaded to the destination bucket. If not present, one will be created in the
	// system's default location.
	ConfigTemporaryDir = "gcpTemporaryDir"
)

// server implements upspin.StoreServer.
type server struct {
	mu       sync.RWMutex // Protects fields below.
	refCount uint64       // How many clones of us exist.
	storage  storage.Storage
	cache    *cache.FileCache
}

var _ upspin.StoreServer = (*server)(nil)

// New returns a StoreServer that serves the given endpoint with the provided options.
func New(options ...string) (upspin.StoreServer, error) {
	const op = "store/gcp.New"

	var dialOpts []storage.DialOpts
	var tempDir string
	for _, option := range options {
		// Parse all options we understand.
		// What we don't understand we pass it down to the storage.
		switch {
		case strings.HasPrefix(option, ConfigTemporaryDir):
			tempDir = option[len(ConfigTemporaryDir)+1:] // skip 'ConfigTemporaryDir='
		default:
			dialOpts = append(dialOpts, storage.WithOptions(option))
		}
	}

	s, err := storage.Dial("GCS", dialOpts...)
	if err != nil {
		return nil, errors.E(op, err)
	}
	c := cache.NewFileCache(tempDir)
	if c == nil {
		return nil, errors.E(op, errors.Str("filecache failed to create temp directory"))
	}
	log.Debug.Printf("Configured GCP store: %v", options)

	return &server{
		storage: s,
		cache:   c,
	}, nil
}

// Put implements upspin.StoreServer.
func (s *server) Put(data []byte) (*upspin.Refdata, error) {
	const op = "store/gcp.Put"

	m, sp := metric.NewSpan(op)
	sp.SetAnnotation(fmt.Sprintf("size=%d", len(data)))
	s2 := sp.StartSpan("cacheData")

	ref := sha256key.Of(data).String()
	s.mu.RLock()
	err := s.cache.Put(ref, bytes.NewReader(data))
	s2.End()
	if err != nil {
		s.mu.RUnlock()
		m.Done()
		return nil, errors.E(op, err)
	}

	// Now go store it in the cloud.
	go func() {
		sp = sp.StartSpan("gcsUpload")
		if _, err := s.storage.PutLocalFile(s.cache.GetFileLocation(ref), ref); err == nil {
			// Remove the locally-cached entry so we never
			// keep files locally, as we're a tiny server
			// compared with our much better-provisioned
			// storage backend.  This is safe to do
			// because FileCache is thread safe.
			s.cache.Purge(ref)
		}
		sp.End()
		s.mu.RUnlock()
		m.Done()
	}()
	refdata := &upspin.Refdata{
		Reference: upspin.Reference(ref),
		Volatile:  false,
		Duration:  0,
	}
	return refdata, nil
}

// Get implements upspin.StoreServer.
func (s *server) Get(ref upspin.Reference) ([]byte, *upspin.Refdata, []upspin.Location, error) {
	const op = "store/gcp.Get"

	m, sp := metric.NewSpan(op)
	defer m.Done()

	file, loc, err := s.innerGet(ref, sp.StartSpan("innerGet"))
	if err != nil {
		return nil, nil, nil, err
	}
	if file != nil {
		sp = sp.StartSpan("readAll")
		defer sp.End()
		defer file.Close()
		bytes, err := ioutil.ReadAll(file)
		if err != nil {
			err = errors.E(op, err)
		}
		sp.SetAnnotation(fmt.Sprintf("size=%d", len(bytes)))
		return bytes, nil, nil, err
	}
	refdata := &upspin.Refdata{
		Reference: ref,
		Volatile:  false,
		Duration:  0,
	}
	sp.SetAnnotation(fmt.Sprintf("refsize=%d", len(ref)))
	return nil, refdata, []upspin.Location{loc}, nil
}

// innerGet gets a local file descriptor or a new location for the reference. It returns only one of the two return
// values or an error. file is non-nil when the ref is found locally; the file is open for read and the
// caller should close it. If location is non-zero ref is in the backend at that location.
func (s *server) innerGet(ref upspin.Reference, span *metric.Span) (file *os.File, location upspin.Location, err error) {
	const op = "store/gcp.Get"
	s.mu.RLock()
	defer s.mu.RUnlock()

	s1 := span.StartSpan("localLookup")
	file, err = s.cache.OpenRefForRead(string(ref))
	s1.End()
	if err == nil {
		// Ref is in the local cache. Send the file and be done.
		log.Debug.Printf("ref %s is in local cache. Returning it as file: %s", ref, file.Name())
		return
	}

	// File is not local, try to get it from our storage.
	sp := span.StartSpan("gcsLookup")
	defer sp.End()
	var link string
	link, err = s.storage.Get(string(ref))
	if err != nil {
		err = errors.E(op, err)
		return
	}
	// GCP should return an http link
	if !strings.HasPrefix(link, "http") {
		err = errors.E(op, errors.Errorf("invalid link returned from GCP: %s", link))
		log.Error.Println(err)
		return
	}

	url, err := url.Parse(link)
	if err != nil {
		err = errors.E(op, errors.Errorf("can't parse url: %s: %s", link, err))
		log.Error.Print(err)
		return
	}
	location.Reference = upspin.Reference(link)
	// Go fetch using the provided link. NetAddr is important so we can both ping the server and also cache the
	// HTTPS transport client efficiently.
	location.Endpoint.Transport = upspin.HTTPS
	location.Endpoint.NetAddr = upspin.NetAddr(fmt.Sprintf("%s://%s", url.Scheme, url.Host))
	log.Debug.Printf("Ref %s returned as link: %s", ref, link)
	return
}

// Delete implements upspin.StoreServer.
func (s *server) Delete(ref upspin.Reference) error {
	const op = "store/gcp.Delete"
	s.mu.RLock()
	defer s.mu.RUnlock()

	m, _ := metric.NewSpan(op)
	defer m.Done()

	err := s.storage.Delete(string(ref))
	if err != nil {
		return errors.E(op, errors.Errorf("%s: %s", ref, err))
	}
	return nil
}

// Dial implements upspin.Service.
func (s *server) Dial(context upspin.Context, e upspin.Endpoint) (upspin.Service, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.refCount++
	return s, nil
}

// Ping implements upspin.Service.
func (s *server) Ping() bool {
	return true
}

// Close implements upspin.Service.
func (s *server) Close() {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.refCount == 0 {
		log.Error.Printf("Closing non-dialed gcp store")
		return
	}
	s.refCount--

	if s.refCount == 0 {
		if s.storage != nil {
			s.storage.Close()
		}
		s.storage = nil
		if s.cache != nil {
			s.cache.Delete()
		}
		s.cache = nil
	}
}

// Endpoint implements upspin.Service.
func (s *server) Endpoint() upspin.Endpoint {
	return upspin.Endpoint{} // No endpoint.
}
<commit_msg>store/gcp: tidy up log messages<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gcp implements upspin.StoreServer using Google Cloud Platform as its storage.
package gcp

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"strings"
	"sync"

	"upspin.io/cloud/storage"
	"upspin.io/errors"
	"upspin.io/key/sha256key"
	"upspin.io/log"
	"upspin.io/metric"
	"upspin.io/store/gcp/cache"
	"upspin.io/upspin"

	// We use GCS as the backing for our data.
	_ "upspin.io/cloud/storage/gcs"
)

// Configuration options for this package.
const (
	// ConfigTemporaryDir specifies which temporary directory to write files to before they're
	// uploaded to the destination bucket. If not present, one will be created in the
	// system's default location.
	ConfigTemporaryDir = "gcpTemporaryDir"
)

// server implements upspin.StoreServer.
type server struct {
	mu       sync.RWMutex // Protects fields below.
	refCount uint64       // How many clones of us exist.
	storage  storage.Storage
	cache    *cache.FileCache
}

var _ upspin.StoreServer = (*server)(nil)

// New returns a StoreServer that serves the given endpoint with the provided options.
func New(options ...string) (upspin.StoreServer, error) {
	const op = "store/gcp.New"

	var dialOpts []storage.DialOpts
	var tempDir string
	for _, option := range options {
		// Parse all options we understand.
		// What we don't understand we pass it down to the storage.
		switch {
		case strings.HasPrefix(option, ConfigTemporaryDir):
			tempDir = option[len(ConfigTemporaryDir)+1:] // skip 'ConfigTemporaryDir='
		default:
			dialOpts = append(dialOpts, storage.WithOptions(option))
		}
	}

	s, err := storage.Dial("GCS", dialOpts...)
	if err != nil {
		return nil, errors.E(op, err)
	}
	c := cache.NewFileCache(tempDir)
	if c == nil {
		return nil, errors.E(op, errors.Str("filecache failed to create temp directory"))
	}

	return &server{
		storage: s,
		cache:   c,
	}, nil
}

// Put implements upspin.StoreServer.
func (s *server) Put(data []byte) (*upspin.Refdata, error) {
	const op = "store/gcp.Put"

	m, sp := metric.NewSpan(op)
	sp.SetAnnotation(fmt.Sprintf("size=%d", len(data)))
	s2 := sp.StartSpan("cacheData")

	ref := sha256key.Of(data).String()
	s.mu.RLock()
	err := s.cache.Put(ref, bytes.NewReader(data))
	s2.End()
	if err != nil {
		s.mu.RUnlock()
		m.Done()
		return nil, errors.E(op, err)
	}

	// Now go store it in the cloud.
	go func() {
		sp = sp.StartSpan("gcsUpload")
		if _, err := s.storage.PutLocalFile(s.cache.GetFileLocation(ref), ref); err == nil {
			// Remove the locally-cached entry so we never
			// keep files locally, as we're a tiny server
			// compared with our much better-provisioned
			// storage backend.  This is safe to do
			// because FileCache is thread safe.
			s.cache.Purge(ref)
		}
		sp.End()
		s.mu.RUnlock()
		m.Done()
	}()
	refdata := &upspin.Refdata{
		Reference: upspin.Reference(ref),
		Volatile:  false,
		Duration:  0,
	}
	return refdata, nil
}

// Get implements upspin.StoreServer.
func (s *server) Get(ref upspin.Reference) ([]byte, *upspin.Refdata, []upspin.Location, error) {
	const op = "store/gcp.Get"

	m, sp := metric.NewSpan(op)
	defer m.Done()

	file, loc, err := s.innerGet(ref, sp.StartSpan("innerGet"))
	if err != nil {
		return nil, nil, nil, err
	}
	if file != nil {
		sp = sp.StartSpan("readAll")
		defer sp.End()
		defer file.Close()
		bytes, err := ioutil.ReadAll(file)
		if err != nil {
			err = errors.E(op, err)
		}
		sp.SetAnnotation(fmt.Sprintf("size=%d", len(bytes)))
		return bytes, nil, nil, err
	}
	refdata := &upspin.Refdata{
		Reference: ref,
		Volatile:  false,
		Duration:  0,
	}
	sp.SetAnnotation(fmt.Sprintf("refsize=%d", len(ref)))
	return nil, refdata, []upspin.Location{loc}, nil
}

// innerGet gets a local file descriptor or a new location for the reference. It returns only one of the two return
// values or an error. file is non-nil when the ref is found locally; the file is open for read and the
// caller should close it. If location is non-zero ref is in the backend at that location.
func (s *server) innerGet(ref upspin.Reference, span *metric.Span) (file *os.File, location upspin.Location, err error) {
	const op = "store/gcp.Get"
	s.mu.RLock()
	defer s.mu.RUnlock()

	s1 := span.StartSpan("localLookup")
	file, err = s.cache.OpenRefForRead(string(ref))
	s1.End()
	if err == nil {
		// Ref is in the local cache. Send the file and be done.
		return
	}

	// File is not local, try to get it from our storage.
	sp := span.StartSpan("gcsLookup")
	defer sp.End()
	var link string
	link, err = s.storage.Get(string(ref))
	if err != nil {
		err = errors.E(op, err)
		return
	}
	// GCP should return an http link
	if !strings.HasPrefix(link, "http") {
		err = errors.E(op, errors.Errorf("invalid link returned from GCP: %s", link))
		return
	}

	url, err := url.Parse(link)
	if err != nil {
		err = errors.E(op, errors.Errorf("can't parse url: %s: %s", link, err))
		return
	}
	location.Reference = upspin.Reference(link)
	// Go fetch using the provided link. NetAddr is important so we can both ping the server and also cache the
	// HTTPS transport client efficiently.
	location.Endpoint.Transport = upspin.HTTPS
	location.Endpoint.NetAddr = upspin.NetAddr(fmt.Sprintf("%s://%s", url.Scheme, url.Host))
	return
}

// Delete implements upspin.StoreServer.
func (s *server) Delete(ref upspin.Reference) error {
	const op = "store/gcp.Delete"
	s.mu.RLock()
	defer s.mu.RUnlock()

	m, _ := metric.NewSpan(op)
	defer m.Done()

	err := s.storage.Delete(string(ref))
	if err != nil {
		return errors.E(op, errors.Errorf("%s: %s", ref, err))
	}
	return nil
}

// Dial implements upspin.Service.
func (s *server) Dial(context upspin.Context, e upspin.Endpoint) (upspin.Service, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.refCount++
	return s, nil
}

// Ping implements upspin.Service.
func (s *server) Ping() bool {
	return true
}

// Close implements upspin.Service.
func (s *server) Close() {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.refCount == 0 {
		log.Error.Printf("store/gcp: closing store that was not dialed")
		return
	}
	s.refCount--

	if s.refCount == 0 {
		if s.storage != nil {
			s.storage.Close()
		}
		s.storage = nil
		if s.cache != nil {
			s.cache.Delete()
		}
		s.cache = nil
	}
}

// Endpoint implements upspin.Service.
func (s *server) Endpoint() upspin.Endpoint {
	return upspin.Endpoint{} // No endpoint.
}
<|endoftext|>


<commit_before>package forward

import (
	"encoding/json"
	"errors"
	"fmt"
	"net"

	"github.com/service-exposer/exposer"
)

const (
	CMD_AUTH          = "auth"
	CMD_AUTH_REPLY    = "auth:reply"
	CMD_FORWARD       = "forward"
	CMD_FORWARD_REPLY = "forward:reply"
)

type Reply struct {
	OK  bool
	Err string
}

type Auth struct {
	Key string
}

type Forward struct {
	Network string
	Address string
}

func ServerSide(authFn func(string) bool) exposer.HandshakeHandleFunc {
	return func(proto *exposer.Protocal, cmd string, details []byte) error {
		switch cmd {
		case CMD_AUTH:
			var auth Auth
			err := json.Unmarshal(details, &auth)
			if err != nil {
				return err
			}

			if authFn(auth.Key) == true {
				return proto.Reply(CMD_AUTH_REPLY, &Reply{
					OK: true,
				})
			}

			err = proto.Reply(CMD_AUTH_REPLY, &Reply{
				OK:  false,
				Err: "auth failure",
			})
			if err != nil {
				return err
			}

			return errors.New("auth failure")
		case CMD_FORWARD:
			var forward Forward
			err := json.Unmarshal(details, &forward)
			if err != nil {
				return err
			}

			conn, err := net.Dial(forward.Network, forward.Address)
			if err != nil {
				proto.Reply(CMD_FORWARD_REPLY, &Reply{
					OK:  false,
					Err: err.Error(),
				})
				return err
			}
			conn.Close()

			err = proto.Reply(CMD_FORWARD_REPLY, &Reply{
				OK: true,
			})
			if err != nil {
				return err
			}

			/*
				ln := proto.Multiplex(false)
				defer ln.Close()

				for {
					local_conn, err := ln.Accept()
					if err != nil {
						return err
					}
					defer local_conn.Close()

					remote_conn, err := net.Dial(forward.Network, forward.Address)
					if err != nil {
						return err
					}
					defer remote_conn.Close()

					go func() {
						wg := &sync.WaitGroup{}
						wg.Add(2)
						go func() {
							defer wg.Done()
							io.Copy(local_conn, remote_conn)
						}()

						go func() {
							defer wg.Done()
							io.Copy(remote_conn, local_conn)
						}()
						wg.Wait()
					}()
				}
			*/
			exposer.Serve(proto.Multiplex(false), func(conn net.Conn) exposer.ProtocalHandler {
				proto := exposer.NewProtocal(conn)
				proto.On = func(proto *exposer.Protocal, cmd string, details []byte) error {
					err := proto.Reply("", nil)
					if err != nil {
						return err
					}

					conn, err := net.Dial(forward.Network, forward.Address)
					if err != nil {
						return err
					}

					fmt.Println("forward to", forward.Address)
					proto.Forward(conn)
					return nil
				}
				return proto
			})

		}
		return errors.New("unknow cmd")
	}

}

func ClientSide(forward Forward, ln net.Listener) exposer.HandshakeHandleFunc {
	return func(proto *exposer.Protocal, cmd string, details []byte) error {
		switch cmd {
		case CMD_AUTH_REPLY:
			var reply Reply
			err := json.Unmarshal(details, &reply)
			if err != nil {
				return err
			}

			if !reply.OK {
				return errors.New(reply.Err)
			}

			return proto.Reply(CMD_FORWARD, &forward)
		case CMD_FORWARD_REPLY:

			var reply Reply
			err := json.Unmarshal(details, &reply)
			if err != nil {
				return err
			}

			if !reply.OK {
				return errors.New(reply.Err)
			}

			session := proto.Multiplex(true)

			for {
				local_conn, err := ln.Accept()
				if err != nil {
					return err
				}

				remote_conn, err := session.Open()
				if err != nil {
					return err
				}

				/*
					go func() {
						wg := &sync.WaitGroup{}
						wg.Add(2)
						go func() {
							defer wg.Done()
							io.Copy(local_conn, remote_conn)
						}()

						go func() {
							defer wg.Done()
							io.Copy(remote_conn, local_conn)
						}()
						wg.Wait()
					}()
				*/

				proto_forward := exposer.NewProtocal(remote_conn)
				proto_forward.On = func(proto *exposer.Protocal, cmd string, details []byte) error {
					proto.Forward(local_conn)
					return nil
				}

				go proto_forward.Request("", nil)
			}
		default:
			return errors.New("unknow cmd")
		}
	}

}
<commit_msg>remove fmt.Println for debug<commit_after>package forward

import (
	"encoding/json"
	"errors"
	"net"

	"github.com/service-exposer/exposer"
)

const (
	CMD_AUTH          = "auth"
	CMD_AUTH_REPLY    = "auth:reply"
	CMD_FORWARD       = "forward"
	CMD_FORWARD_REPLY = "forward:reply"
)

type Reply struct {
	OK  bool
	Err string
}

type Auth struct {
	Key string
}

type Forward struct {
	Network string
	Address string
}

func ServerSide(authFn func(string) bool) exposer.HandshakeHandleFunc {
	return func(proto *exposer.Protocal, cmd string, details []byte) error {
		switch cmd {
		case CMD_AUTH:
			var auth Auth
			err := json.Unmarshal(details, &auth)
			if err != nil {
				return err
			}

			if authFn(auth.Key) == true {
				return proto.Reply(CMD_AUTH_REPLY, &Reply{
					OK: true,
				})
			}

			err = proto.Reply(CMD_AUTH_REPLY, &Reply{
				OK:  false,
				Err: "auth failure",
			})
			if err != nil {
				return err
			}

			return errors.New("auth failure")
		case CMD_FORWARD:
			var forward Forward
			err := json.Unmarshal(details, &forward)
			if err != nil {
				return err
			}

			conn, err := net.Dial(forward.Network, forward.Address)
			if err != nil {
				proto.Reply(CMD_FORWARD_REPLY, &Reply{
					OK:  false,
					Err: err.Error(),
				})
				return err
			}
			conn.Close()

			err = proto.Reply(CMD_FORWARD_REPLY, &Reply{
				OK: true,
			})
			if err != nil {
				return err
			}

			/*
				ln := proto.Multiplex(false)
				defer ln.Close()

				for {
					local_conn, err := ln.Accept()
					if err != nil {
						return err
					}
					defer local_conn.Close()

					remote_conn, err := net.Dial(forward.Network, forward.Address)
					if err != nil {
						return err
					}
					defer remote_conn.Close()

					go func() {
						wg := &sync.WaitGroup{}
						wg.Add(2)
						go func() {
							defer wg.Done()
							io.Copy(local_conn, remote_conn)
						}()

						go func() {
							defer wg.Done()
							io.Copy(remote_conn, local_conn)
						}()
						wg.Wait()
					}()
				}
			*/
			exposer.Serve(proto.Multiplex(false), func(conn net.Conn) exposer.ProtocalHandler {
				proto := exposer.NewProtocal(conn)
				proto.On = func(proto *exposer.Protocal, cmd string, details []byte) error {
					err := proto.Reply("", nil)
					if err != nil {
						return err
					}

					conn, err := net.Dial(forward.Network, forward.Address)
					if err != nil {
						return err
					}

					proto.Forward(conn)
					return nil
				}
				return proto
			})

		}
		return errors.New("unknow cmd")
	}

}

func ClientSide(forward Forward, ln net.Listener) exposer.HandshakeHandleFunc {
	return func(proto *exposer.Protocal, cmd string, details []byte) error {
		switch cmd {
		case CMD_AUTH_REPLY:
			var reply Reply
			err := json.Unmarshal(details, &reply)
			if err != nil {
				return err
			}

			if !reply.OK {
				return errors.New(reply.Err)
			}

			return proto.Reply(CMD_FORWARD, &forward)
		case CMD_FORWARD_REPLY:

			var reply Reply
			err := json.Unmarshal(details, &reply)
			if err != nil {
				return err
			}

			if !reply.OK {
				return errors.New(reply.Err)
			}

			session := proto.Multiplex(true)

			for {
				local_conn, err := ln.Accept()
				if err != nil {
					return err
				}

				remote_conn, err := session.Open()
				if err != nil {
					return err
				}

				/*
					go func() {
						wg := &sync.WaitGroup{}
						wg.Add(2)
						go func() {
							defer wg.Done()
							io.Copy(local_conn, remote_conn)
						}()

						go func() {
							defer wg.Done()
							io.Copy(remote_conn, local_conn)
						}()
						wg.Wait()
					}()
				*/

				proto_forward := exposer.NewProtocal(remote_conn)
				proto_forward.On = func(proto *exposer.Protocal, cmd string, details []byte) error {
					proto.Forward(local_conn)
					return nil
				}

				go proto_forward.Request("", nil)
			}
		default:
			return errors.New("unknow cmd")
		}
	}

}
<|endoftext|>


<commit_before>package protocol

import (
	"github.com/SmartPool/smartpool-client"
)

type testClaimRepo struct {
	c  []smartpool.Share
	oc []smartpool.Claim
}

func newClaimRepo() *testClaimRepo {
	return &testClaimRepo{[]smartpool.Share{}, []smartpool.Claim{}}
}

func (cr *testClaimRepo) GetCurrentClaim(threshold int) smartpool.Claim {
	if len(cr.c) < threshold {
		return nil
	}
	claim := &testClaim{cr.c}
	cr.c = []smartpool.Share{}
	cr.oc = []smartpool.Claim{claim}
	return claim
}

func (cr *testClaimRepo) AddShare(s smartpool.Share) error {
	cr.c = append(cr.c, s)
	return nil
}

func (cr *testClaimRepo) NoActiveShares() uint64 {
	return 0
}

func (cr *testClaimRepo) Persist(storage smartpool.PersistentStorage) error {
	return nil
}

func (cr *testClaimRepo) PutOpenClaim(claim smartpool.Claim) {
}

func (cr *testClaimRepo) GetOpenClaim(claimIndex int) smartpool.Claim {
	return cr.oc[0]
}

func (cr *testClaimRepo) ResetOpenClaims() {
}

func (cr *testClaimRepo) NumOpenClaims() uint64 {
	return 0
}

func (cr *testClaimRepo) SealClaimBatch() {
}
<commit_msg>fix test<commit_after>package protocol

import (
	"github.com/SmartPool/smartpool-client"
)

type testClaimRepo struct {
	c  []smartpool.Share
	oc []smartpool.Claim
}

func newClaimRepo() *testClaimRepo {
	return &testClaimRepo{[]smartpool.Share{}, []smartpool.Claim{}}
}

func (cr *testClaimRepo) GetCurrentClaim(threshold int) smartpool.Claim {
	if len(cr.c) < threshold {
		return nil
	}
	claim := &testClaim{cr.c}
	cr.c = []smartpool.Share{}
	cr.oc = []smartpool.Claim{claim}
	return claim
}

func (cr *testClaimRepo) AddShare(s smartpool.Share) error {
	cr.c = append(cr.c, s)
	return nil
}

func (cr *testClaimRepo) NoActiveShares() uint64 {
	return 0
}

func (cr *testClaimRepo) Persist(storage smartpool.PersistentStorage) error {
	return nil
}

func (cr *testClaimRepo) PutOpenClaim(claim smartpool.Claim) {
}

func (cr *testClaimRepo) GetOpenClaim(claimIndex int) smartpool.Claim {
	return cr.oc[0]
}

func (cr *testClaimRepo) ResetOpenClaims() {
}

func (cr *testClaimRepo) RemoveOpenClaim(claim smartpool.Claim) {
}

func (cr *testClaimRepo) NumOpenClaims() uint64 {
	return 0
}

func (cr *testClaimRepo) SealClaimBatch() {
}
<|endoftext|>


<commit_before>package shardmaster

import "raft"
import "labrpc"
import "sync"
import (
	"encoding/gob"
	"fmt"
	"time"
	"log"
	"sort"
)

const Debug = 0

func DPrintf(format string, a ...interface{}) (n int, err error) {
	if Debug > 0 {
		log.Printf(format, a...)
	}
	return
}

type ShardMaster struct {
	mu      sync.Mutex
	me      int
	rf      *raft.Raft
	applyCh chan raft.ApplyMsg

	// Your data here.
	oldRequests map[int64]int64 // preserve client request
	res map[int]chan OpReply // per client request channel

	configs []Config // indexed by config num
}

type Op struct {
	Cid int64
	Seq int64
	Action string
	Servers map[int][]string
	GIDs []int
	Shard int
	GID   int
	Num int
}

type OpReply struct {
	Cid int64
	Seq int64
	Conf Config
	WrongLeader bool
	Err         Err
}

func (sm *ShardMaster) lastConf() *Config{
	return &sm.configs[len(sm.configs)-1]
}

func (sm *ShardMaster) newConfig() *Config{
	lastConfig := sm.lastConf()

	var shards [NShards]int

	var config Config

	config.Num = lastConfig.Num + 1
	config.Shards = shards
	config.Groups = make(map[int][]string)

	for k,v := range lastConfig.Groups{
		config.Groups[k] = v
	}

	for i := range config.Shards{
		config.Shards[i] = lastConfig.Shards[i]
	}

	return &config
}

func (sm *ShardMaster) logToRaft(arg *Op, reply *OpReply){
	op := Op{Action:arg.Action,
		Cid:arg.Cid,
		Seq:arg.Seq,
		Servers:arg.Servers,
		GIDs:arg.GIDs,
		Shard:arg.Shard,
		GID:arg.GID,
		Num:arg.Num}

	reply.WrongLeader = true
	reply.Err = ""
	reply.Seq = op.Seq
	reply.Cid = op.Cid

	index,_,isLeader := sm.rf.Start(op)
	if isLeader {
		sm.mu.Lock()
		_,ok := sm.res[index]
		if !ok{
			sm.res[index] = make(chan OpReply, 1)
		}
		sm.mu.Unlock()

		select{
		case rep := <- sm.res[index]:
			if rep.Cid == op.Cid && rep.Seq == op.Seq{
				reply.WrongLeader = false
				reply.Conf = rep.Conf
			}else{
				reply.Err = Error
			}
		case <-time.After(time.Duration(100)*time.Millisecond):
			reply.Err = TimeOut
		}
	}

	sm.mu.Lock()
	delete(sm.res, index)
	sm.mu.Unlock()
}

func (sm *ShardMaster) Join(args *JoinArgs, reply *JoinReply) {
	op := Op{Action:"Join",Cid:args.Cid,Seq:args.Seq,Servers:args.Servers}

	var opReply OpReply
	sm.logToRaft(&op, &opReply)

	reply.WrongLeader = opReply.WrongLeader
	reply.Err = ""

	DPrintf(fmt.Sprintf("Server %d Join %v with reply %v",sm.me, *args, *reply))
}

func (sm *ShardMaster) Leave(args *LeaveArgs, reply *LeaveReply) {
	op := Op{Action:"Leave",Cid:args.Cid,Seq:args.Seq,GIDs:args.GIDs}

	var opReply OpReply
	sm.logToRaft(&op, &opReply)

	reply.WrongLeader = opReply.WrongLeader
	reply.Err = ""

	DPrintf(fmt.Sprintf("Server %d Leave %v with reply %v",sm.me, *args, *reply))
}

func (sm *ShardMaster) Move(args *MoveArgs, reply *MoveReply) {
	op := Op{Action:"Move",Cid:args.Cid,Seq:args.Seq,Shard:args.Shard,GID:args.GID}

	var opReply OpReply
	sm.logToRaft(&op, &opReply)

	reply.WrongLeader = opReply.WrongLeader
	reply.Err = ""

	DPrintf(fmt.Sprintf("Server %d Move %v with reply %v",sm.me, *args, *reply))
}

func (sm *ShardMaster) Query(args *QueryArgs, reply *QueryReply) {
	op := Op{Action:"Query",Cid:args.Cid,Seq:args.Seq,Num:args.Num}

	var opReply OpReply
	sm.logToRaft(&op, &opReply)

	reply.WrongLeader = opReply.WrongLeader
	reply.Err = ""
	reply.Config = opReply.Conf

	DPrintf(fmt.Sprintf("Server %d Query %v with reply %v",sm.me, *args, *reply))
}

//
// the tester calls Kill() when a ShardMaster instance won't
// be needed again. you are not required to do anything
// in Kill(), but it might be convenient to (for example)
// turn off debug output from this instance.
//
func (sm *ShardMaster) Kill() {
	sm.rf.Kill()
	// Your code here, if desired.
}

// needed by shardkv tester
func (sm *ShardMaster) Raft() *raft.Raft {
	return sm.rf
}

//
// servers[] contains the ports of the set of
// servers that will cooperate via Paxos to
// form the fault-tolerant shardmaster service.
// me is the index of the current server in servers[].
//
func StartServer(servers []*labrpc.ClientEnd, me int, persister *raft.Persister) *ShardMaster {
	sm := new(ShardMaster)
	sm.me = me

	sm.configs = make([]Config, 1)
	sm.configs[0].Groups = map[int][]string{}

	gob.Register(Op{})
	sm.applyCh = make(chan raft.ApplyMsg)
	sm.rf = raft.Make(servers, me, persister, sm.applyCh)

	// Your code here.
	sm.res = make(map[int]chan OpReply)
	sm.oldRequests = make(map[int64]int64)
	go func(){
		for {
			select {
			case rep := <-sm.applyCh:
				msg, ok:= rep.Command.(Op)
				if ok{
					sm.execute(&msg, rep.Index)
				}
			}
		}
	}()

	return sm
}

func (sm *ShardMaster) execute(msg *Op, index int){
	sm.mu.Lock()

	opReply := &OpReply{Cid:msg.Cid,Seq:msg.Seq}

	// execute command
	if msg.Seq > sm.oldRequests[msg.Cid]{
		switch msg.Action{
		case "Join":
			sm.doJoin(msg)
		case "Leave":
			sm.doLeave(msg)
		case "Move":
			sm.doMove(msg)
		case "Query":
			sm.doQuery(msg, opReply)
		default:
			break;
		}

		sm.oldRequests[msg.Cid] = msg.Seq
	}

	// send msg to wake up client wait
	channel, ok := sm.res[index]
	if !ok {
		channel = make(chan OpReply, 1)
		sm.res[index] = channel
	}else{
		channel <- *opReply
	}

	sm.mu.Unlock()
}

func (sm *ShardMaster) doJoin(msg *Op){
	config := sm.newConfig()

	for k,v := range msg.Servers{
		config.Groups[k] = v
	}

	var gids []int
	for k := range config.Groups{
		gids = append(gids,k)
	}
	sort.Ints(gids)

	numGroups := len(gids)
	for i := range config.Shards{
		m := i % numGroups
		config.Shards[i] = gids[m]
	}

	sm.configs = append(sm.configs, *config)
}

func (sm *ShardMaster) doLeave(msg *Op){
	config := sm.newConfig()

	for _,g:= range msg.GIDs {
		delete(config.Groups, g)
	}

	var gids []int
	for g := range config.Groups{
		gids = append(gids, g)
	}
	sort.Ints(gids)

	numGroups := len(gids)

	for i := range config.Shards{
		m := i % numGroups
		config.Shards[i] = gids[m]
	}

	sm.configs = append(sm.configs, *config)
}

func (sm *ShardMaster) doMove(msg *Op){
	config := sm.newConfig()

	config.Shards[msg.Shard] = msg.GID

	sm.configs = append(sm.configs, *config)
}

func (sm *ShardMaster) doQuery(msg *Op, reply *OpReply){
	if msg.Num == -1 || msg.Num >= len(sm.configs){
		reply.Conf = sm.configs[len(sm.configs)-1]
	}else{
		reply.Conf = sm.configs[msg.Num]
	}
}<commit_msg>fix bug: enable query when sequence less or equal than previous sequence<commit_after>package shardmaster

import "raft"
import "labrpc"
import "sync"
import (
	"encoding/gob"
	"fmt"
	"time"
	"log"
	"sort"
)

const Debug = 0

func DPrintf(format string, a ...interface{}) (n int, err error) {
	if Debug > 0 {
		log.Printf(format, a...)
	}
	return
}

type ShardMaster struct {
	mu      sync.Mutex
	me      int
	rf      *raft.Raft
	applyCh chan raft.ApplyMsg

	// Your data here.
	oldRequests map[int64]int64 // preserve client request
	res map[int]chan OpReply // per client request channel

	configs []Config // indexed by config num
}

type Op struct {
	Cid int64
	Seq int64
	Action string
	Servers map[int][]string
	GIDs []int
	Shard int
	GID   int
	Num int
}

type OpReply struct {
	Cid int64
	Seq int64
	Conf Config
	WrongLeader bool
	Err         Err
}

func (sm *ShardMaster) lastConf() *Config{
	return &sm.configs[len(sm.configs)-1]
}

func (sm *ShardMaster) newConfig() *Config{
	lastConfig := sm.lastConf()

	var config Config

	config.Num = len(sm.configs)
	config.Shards = [NShards]int{}
	config.Groups = make(map[int][]string)

	for k,v := range lastConfig.Groups{
		config.Groups[k] = v
	}

	for i := range config.Shards{
		config.Shards[i] = lastConfig.Shards[i]
	}

	return &config
}

func (sm *ShardMaster) logToRaft(arg *Op, reply *OpReply){
	op := Op{Action:arg.Action,
		Cid:arg.Cid,
		Seq:arg.Seq,
		Servers:arg.Servers,
		GIDs:arg.GIDs,
		Shard:arg.Shard,
		GID:arg.GID,
		Num:arg.Num}

	reply.WrongLeader = true
	reply.Err = ""
	reply.Seq = op.Seq
	reply.Cid = op.Cid

	index,_,isLeader := sm.rf.Start(op)
	if isLeader {
		sm.mu.Lock()
		_,ok := sm.res[index]
		if !ok{
			sm.res[index] = make(chan OpReply, 1)
		}
		sm.mu.Unlock()

		select{
		case rep := <- sm.res[index]:
			if rep.Cid == op.Cid && rep.Seq == op.Seq{
				reply.WrongLeader = false
				reply.Conf = rep.Conf
			}else{
				reply.Err = Error
			}
		case <-time.After(time.Duration(100)*time.Millisecond):
			reply.Err = TimeOut
		}
	}

	sm.mu.Lock()
	delete(sm.res, index)
	sm.mu.Unlock()
}

func (sm *ShardMaster) Join(args *JoinArgs, reply *JoinReply) {
	op := Op{Action:"Join",Cid:args.Cid,Seq:args.Seq,Servers:args.Servers}

	var opReply OpReply
	sm.logToRaft(&op, &opReply)

	reply.WrongLeader = opReply.WrongLeader
	reply.Err = ""

	DPrintf(fmt.Sprintf("Server %d Join %v with reply %v",sm.me, *args, *reply))
}

func (sm *ShardMaster) Leave(args *LeaveArgs, reply *LeaveReply) {
	op := Op{Action:"Leave",Cid:args.Cid,Seq:args.Seq,GIDs:args.GIDs}

	var opReply OpReply
	sm.logToRaft(&op, &opReply)

	reply.WrongLeader = opReply.WrongLeader
	reply.Err = ""

	DPrintf(fmt.Sprintf("Server %d Leave %v with reply %v",sm.me, *args, *reply))
}

func (sm *ShardMaster) Move(args *MoveArgs, reply *MoveReply) {
	op := Op{Action:"Move",Cid:args.Cid,Seq:args.Seq,Shard:args.Shard,GID:args.GID}

	var opReply OpReply
	sm.logToRaft(&op, &opReply)

	reply.WrongLeader = opReply.WrongLeader
	reply.Err = ""

	DPrintf(fmt.Sprintf("Server %d Move %v with reply %v",sm.me, *args, *reply))
}

func (sm *ShardMaster) Query(args *QueryArgs, reply *QueryReply) {
	op := Op{Action:"Query",Cid:args.Cid,Seq:args.Seq,Num:args.Num}

	var opReply OpReply
	sm.logToRaft(&op, &opReply)

	reply.WrongLeader = opReply.WrongLeader
	reply.Err = ""
	reply.Config = opReply.Conf

	DPrintf(fmt.Sprintf("Server %d Query %v with reply %v",sm.me, *args, *reply))
}

//
// the tester calls Kill() when a ShardMaster instance won't
// be needed again. you are not required to do anything
// in Kill(), but it might be convenient to (for example)
// turn off debug output from this instance.
//
func (sm *ShardMaster) Kill() {
	sm.rf.Kill()
	// Your code here, if desired.
}

// needed by shardkv tester
func (sm *ShardMaster) Raft() *raft.Raft {
	return sm.rf
}

//
// servers[] contains the ports of the set of
// servers that will cooperate via Paxos to
// form the fault-tolerant shardmaster service.
// me is the index of the current server in servers[].
//
func StartServer(servers []*labrpc.ClientEnd, me int, persister *raft.Persister) *ShardMaster {
	sm := new(ShardMaster)
	sm.me = me

	sm.configs = make([]Config, 1)
	sm.configs[0].Groups = map[int][]string{}
	sm.configs[0].Num = 0
	sm.configs[0].Shards = [NShards]int{}

	gob.Register(Op{})
	sm.applyCh = make(chan raft.ApplyMsg)
	sm.rf = raft.Make(servers, me, persister, sm.applyCh)

	// Your code here.
	sm.res = make(map[int]chan OpReply)
	sm.oldRequests = make(map[int64]int64)
	go func(){
		for {
			select {
			case rep := <-sm.applyCh:
				msg, ok:= rep.Command.(Op)
				if ok{
					sm.execute(&msg, rep.Index)
				}
			}
		}
	}()

	return sm
}

func (sm *ShardMaster) execute(msg *Op, index int){
	sm.mu.Lock()

	opReply := &OpReply{Cid:msg.Cid,Seq:msg.Seq,WrongLeader:true}

	// execute command
	if msg.Seq > sm.oldRequests[msg.Cid]{
		switch msg.Action{
		case "Join":
			opReply.WrongLeader = false
			sm.doJoin(msg)
		case "Leave":
			opReply.WrongLeader = false
			sm.doLeave(msg)
		case "Move":
			opReply.WrongLeader = false
			sm.doMove(msg)
		case "Query":
			opReply.WrongLeader = false
			sm.doQuery(msg, opReply)
		default:
			break;
		}

		sm.oldRequests[msg.Cid] = msg.Seq
	}else{
		if msg.Action == "Query"{
			if msg.Num == -1 || msg.Num >= len(sm.configs){
				opReply.Conf = sm.configs[len(sm.configs)-1]
			}else{
				opReply.Conf = sm.configs[msg.Num]
			}
		}
	}

	// send msg to wake up client wait
	channel, ok := sm.res[index]
	if !ok {
		channel = make(chan OpReply, 1)
		sm.res[index] = channel
	}else{
		channel <- *opReply
	}

	sm.mu.Unlock()
}

func (sm *ShardMaster) doJoin(msg *Op){
	config := sm.newConfig()

	for k,v := range msg.Servers{
		config.Groups[k] = v
	}

	var gids []int
	for k := range config.Groups{
		gids = append(gids,k)
	}
	sort.Ints(gids)

	numGroups := len(gids)
	for i := range config.Shards{
		m := i % numGroups
		config.Shards[i] = gids[m]
	}

	sm.configs = append(sm.configs, *config)
}

func (sm *ShardMaster) doLeave(msg *Op){
	config := sm.newConfig()

	for _,g:= range msg.GIDs {
		delete(config.Groups, g)
	}

	var gids []int
	for g := range config.Groups{
		gids = append(gids, g)
	}
	sort.Ints(gids)

	numGroups := len(gids)

	for i := range config.Shards{
		m := i % numGroups
		config.Shards[i] = gids[m]
	}

	sm.configs = append(sm.configs, *config)
}

func (sm *ShardMaster) doMove(msg *Op){
	config := sm.newConfig()

	config.Shards[msg.Shard] = msg.GID

	sm.configs = append(sm.configs, *config)
}

func (sm *ShardMaster) doQuery(msg *Op, reply *OpReply){
	if msg.Num == -1 || msg.Num >= len(sm.configs){
		reply.Conf = sm.configs[len(sm.configs)-1]
	}else{
		reply.Conf = sm.configs[msg.Num]
	}
}<|endoftext|>


<commit_before>// Copyright 2016 Red Hat, Inc. and/or its affiliates
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package main
package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/yaacov/mohawk/backends"
)

// validRegex regexp for validating sql variables
var validRegex = regexp.MustCompile(`^[A-Za-z0-9_/\[\]\(\)\.-]*$`)

// Handler common variables to be used by all Handler functions
// 	version the version of the Hawkular server we are mocking
// 	backend the backend to be used by the Handler functions
type Handler struct {
	version string
	backend backend.Backend
}

// parseTags takes a comma separeted key:value list string and returns a map[string]string
// 	e.g.
// 	"warm:kitty,soft:kitty" => {"warm": "kitty", "soft": "kitty"}
func parseTags(tags string) map[string]string {
	vsf := make(map[string]string)

	tagsList := strings.Split(tags, ",")
	for _, tag := range tagsList {
		t := strings.Split(tag, ":")
		if len(t) == 2 {
			vsf[t[0]] = t[1]
		}
	}
	return vsf
}

func validStr(s string) bool {
	valid := validRegex.MatchString(s)
	if !valid {
		log.Printf("Valid string fail: %s\n", s)
	}
	return valid
}

func validTags(tags map[string]string) bool {
	for k, v := range tags {
		if !validStr(k) || !validStr(v) {
			return false
		}
	}

	return true
}

// GetStatus return a json status struct
func (h Handler) GetStatus(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	resTemplate := `{
	"MetricsService":"STARTED",
	"Implementation-Version":"%s",
	"MohawkVersion":"%s",
	"MohawkBackend":"%s"
}`
	res := fmt.Sprintf(resTemplate, h.version, VER, h.backend.Name())

	w.WriteHeader(200)
	fmt.Fprintln(w, res)
}

// GetAPIVersions return a json apiVersion struct
func (h Handler) GetAPIVersions(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	resTemplate := `{
	"kind": "APIVersions",
	"apiVersion": "%s",
	"versions": [
		"%s"
	],
	"serverAddressByClientCIDRs": null
}`
	res := fmt.Sprintf(resTemplate, "v1", "v1")

	w.WriteHeader(200)
	fmt.Fprintln(w, res)
}

// GetMetrics return a list of metrics definitions
func (h Handler) GetMetrics(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	var res []backend.Item

	r.ParseForm()
	if tagsStr, ok := r.Form["tags"]; ok && len(tagsStr) > 0 {
		tags := parseTags(tagsStr[0])
		if !validTags(tags) {
			w.WriteHeader(504)
			return
		}
		res = h.backend.GetItemList(tags)
	} else {
		res = h.backend.GetItemList(map[string]string{})
	}
	resJSON, _ := json.Marshal(res)

	w.WriteHeader(200)
	fmt.Fprintln(w, string(resJSON))
}

// GetData return a list of metrics raw / stat data
func (h Handler) GetData(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	var resStr string
	var id string
	var end int64
	var start int64
	var limit int64
	var bucketDuration int64
	var order string

	// use the id from the argv list
	id = argv["id"]
	if !validStr(id) {
		w.WriteHeader(504)
		return
	}

	// get data from the form arguments
	r.ParseForm()
	if v, ok := r.Form["end"]; ok && len(v) > 0 {
		i, _ := strconv.Atoi(v[0])
		end = int64(i)
	} else {
		end = int64(time.Now().Unix() * 1000)
	}
	if v, ok := r.Form["start"]; ok && len(v) > 0 {
		i, _ := strconv.Atoi(v[0])
		start = int64(i)
	} else {
		start = end - int64(8*60*60*1000)
	}
	if v, ok := r.Form["limit"]; ok && len(v) > 0 {
		i, _ := strconv.Atoi(v[0])
		limit = int64(i)
	} else {
		limit = int64(100)
	}
	if v, ok := r.Form["order"]; ok && len(v) > 0 {
		order = v[0]
		// do sanity check
		if order != "ASC" || order != "DESC" {
			order = "DESC"
		}
	} else {
		order = "DESC"
	}
	if v, ok := r.Form["bucketDuration"]; ok && len(v) > 0 {
		i, _ := strconv.Atoi(v[0][:len(v[0])-1])
		bucketDuration = int64(i)
	} else {
		bucketDuration = int64(0)
	}

	// call backend for data
	if bucketDuration == 0 {
		res := h.backend.GetRawData(id, end, start, limit, order)
		resJSON, _ := json.Marshal(res)
		resStr = string(resJSON)
	} else {
		res := h.backend.GetStatData(id, end, start, limit, order, bucketDuration)
		resJSON, _ := json.Marshal(res)
		resStr = string(resJSON)
	}

	// output to client
	w.WriteHeader(200)
	fmt.Fprintln(w, resStr)
}

// PostData send timestamp, value to the backend
func (h Handler) PostData(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	var u []map[string]interface{}
	json.NewDecoder(r.Body).Decode(&u)

	id := u[0]["id"].(string)
	if !validStr(id) {
		w.WriteHeader(504)
		return
	}

	t := u[0]["data"].([]interface{})[0].(map[string]interface{})["timestamp"].(float64)
	vStr := u[0]["data"].([]interface{})[0].(map[string]interface{})["value"].(string)
	v, _ := strconv.ParseFloat(vStr, 64)

	h.backend.PostRawData(id, int64(t), v)
	w.WriteHeader(200)
	fmt.Fprintln(w, "{}")
}

// PutTags send tag, value pairs to the backend
func (h Handler) PutTags(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	var tags map[string]string
	json.NewDecoder(r.Body).Decode(&tags)

	// use the id from the argv list
	id := argv["id"]
	if !validStr(id) || !validTags(tags) {
		w.WriteHeader(504)
		return
	}

	h.backend.PutTags(id, tags)
	w.WriteHeader(200)
	fmt.Fprintln(w, "{}")
}
<commit_msg>allow spaces in keys<commit_after>// Copyright 2016 Red Hat, Inc. and/or its affiliates
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package main
package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/yaacov/mohawk/backends"
)

// validRegex regexp for validating sql variables
var validRegex = regexp.MustCompile(`^[ A-Za-z0-9_/\[\]\(\)\.-]*$`)

// Handler common variables to be used by all Handler functions
// 	version the version of the Hawkular server we are mocking
// 	backend the backend to be used by the Handler functions
type Handler struct {
	version string
	backend backend.Backend
}

// parseTags takes a comma separeted key:value list string and returns a map[string]string
// 	e.g.
// 	"warm:kitty,soft:kitty" => {"warm": "kitty", "soft": "kitty"}
func parseTags(tags string) map[string]string {
	vsf := make(map[string]string)

	tagsList := strings.Split(tags, ",")
	for _, tag := range tagsList {
		t := strings.Split(tag, ":")
		if len(t) == 2 {
			vsf[t[0]] = t[1]
		}
	}
	return vsf
}

func validStr(s string) bool {
	valid := validRegex.MatchString(s)
	if !valid {
		log.Printf("Valid string fail: %s\n", s)
	}
	return valid
}

func validTags(tags map[string]string) bool {
	for k, v := range tags {
		if !validStr(k) || !validStr(v) {
			return false
		}
	}

	return true
}

// GetStatus return a json status struct
func (h Handler) GetStatus(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	resTemplate := `{
	"MetricsService":"STARTED",
	"Implementation-Version":"%s",
	"MohawkVersion":"%s",
	"MohawkBackend":"%s"
}`
	res := fmt.Sprintf(resTemplate, h.version, VER, h.backend.Name())

	w.WriteHeader(200)
	fmt.Fprintln(w, res)
}

// GetAPIVersions return a json apiVersion struct
func (h Handler) GetAPIVersions(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	resTemplate := `{
	"kind": "APIVersions",
	"apiVersion": "%s",
	"versions": [
		"%s"
	],
	"serverAddressByClientCIDRs": null
}`
	res := fmt.Sprintf(resTemplate, "v1", "v1")

	w.WriteHeader(200)
	fmt.Fprintln(w, res)
}

// GetMetrics return a list of metrics definitions
func (h Handler) GetMetrics(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	var res []backend.Item

	r.ParseForm()
	if tagsStr, ok := r.Form["tags"]; ok && len(tagsStr) > 0 {
		tags := parseTags(tagsStr[0])
		if !validTags(tags) {
			w.WriteHeader(504)
			return
		}
		res = h.backend.GetItemList(tags)
	} else {
		res = h.backend.GetItemList(map[string]string{})
	}
	resJSON, _ := json.Marshal(res)

	w.WriteHeader(200)
	fmt.Fprintln(w, string(resJSON))
}

// GetData return a list of metrics raw / stat data
func (h Handler) GetData(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	var resStr string
	var id string
	var end int64
	var start int64
	var limit int64
	var bucketDuration int64
	var order string

	// use the id from the argv list
	id = argv["id"]
	if !validStr(id) {
		w.WriteHeader(504)
		return
	}

	// get data from the form arguments
	r.ParseForm()
	if v, ok := r.Form["end"]; ok && len(v) > 0 {
		i, _ := strconv.Atoi(v[0])
		end = int64(i)
	} else {
		end = int64(time.Now().Unix() * 1000)
	}
	if v, ok := r.Form["start"]; ok && len(v) > 0 {
		i, _ := strconv.Atoi(v[0])
		start = int64(i)
	} else {
		start = end - int64(8*60*60*1000)
	}
	if v, ok := r.Form["limit"]; ok && len(v) > 0 {
		i, _ := strconv.Atoi(v[0])
		limit = int64(i)
	} else {
		limit = int64(100)
	}
	if v, ok := r.Form["order"]; ok && len(v) > 0 {
		order = v[0]
		// do sanity check
		if order != "ASC" || order != "DESC" {
			order = "DESC"
		}
	} else {
		order = "DESC"
	}
	if v, ok := r.Form["bucketDuration"]; ok && len(v) > 0 {
		i, _ := strconv.Atoi(v[0][:len(v[0])-1])
		bucketDuration = int64(i)
	} else {
		bucketDuration = int64(0)
	}

	// call backend for data
	if bucketDuration == 0 {
		res := h.backend.GetRawData(id, end, start, limit, order)
		resJSON, _ := json.Marshal(res)
		resStr = string(resJSON)
	} else {
		res := h.backend.GetStatData(id, end, start, limit, order, bucketDuration)
		resJSON, _ := json.Marshal(res)
		resStr = string(resJSON)
	}

	// output to client
	w.WriteHeader(200)
	fmt.Fprintln(w, resStr)
}

// PostData send timestamp, value to the backend
func (h Handler) PostData(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	var u []map[string]interface{}
	json.NewDecoder(r.Body).Decode(&u)

	id := u[0]["id"].(string)
	if !validStr(id) {
		w.WriteHeader(504)
		return
	}

	t := u[0]["data"].([]interface{})[0].(map[string]interface{})["timestamp"].(float64)
	vStr := u[0]["data"].([]interface{})[0].(map[string]interface{})["value"].(string)
	v, _ := strconv.ParseFloat(vStr, 64)

	h.backend.PostRawData(id, int64(t), v)
	w.WriteHeader(200)
	fmt.Fprintln(w, "{}")
}

// PutTags send tag, value pairs to the backend
func (h Handler) PutTags(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	var tags map[string]string
	json.NewDecoder(r.Body).Decode(&tags)

	// use the id from the argv list
	id := argv["id"]
	if !validStr(id) || !validTags(tags) {
		w.WriteHeader(504)
		return
	}

	h.backend.PutTags(id, tags)
	w.WriteHeader(200)
	fmt.Fprintln(w, "{}")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

func songControl(millisecondWait int64, is_playing bool, text string, song string, start_next bool) {
	time.Sleep(time.Duration(millisecondWait) * time.Millisecond)
	if song == statevar.CurrentSong {
		log.Printf(song + " " + text)
		statevar.IsPlaying = is_playing
		if start_next == true {
			skipTrack(-1)
		}
	}
}

func getPlaylistHTML() (playlist_html string) {
	playlist_html = ""
	for i, k := range statevar.SongList {
		name := statevar.SongMap[k].Title
		names := strings.Split(name, "/")
		showName := names[len(names)-1]
		if statevar.CurrentSong != statevar.SongMap[k].Fullname {
			playlist_html += "" + showName + "
\n"
		} else {
			playlist_html += "" + showName + "
\n"

		}
	}
	return
}

func getPlaybackPositionInSeconds() float64 {
	position := float64(getTime()-statevar.SongStartTime) / 1000.0
	if statevar.IsPlaying == true && position > 0 {
		return position
	} else {
		return 0.0
	}
}

func SyncRequest(rw http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		// defer timeTrack(time.Now(), r.RemoteAddr+" /sync")
		//current_song := r.FormValue("current_song")
		client_timestamp_str := r.FormValue("client_timestamp")
		client_timestamp, _ := strconv.ParseUint(client_timestamp_str, 10, 64)
		is_muted, _ := strconv.ParseBool(r.FormValue("is_muted"))
		mute_button_clicked, _ := strconv.ParseBool(r.FormValue("mute_button_clicked"))
		if mute_button_clicked == true {
			statevar.LastMuted = getTime()
			statevar.IsMuted = is_muted
		}

		if getTime()-statevar.LastMuted < 3000 {
			mute_button_clicked = true
			is_muted = statevar.IsMuted
		}
		name := statevar.CurrentSong
		names := strings.Split(name, "/")
		showName := names[len(names)-1]
		data := SyncJSON{
			Current_song:        showName,
			Client_timestamp:    int64(client_timestamp),
			Server_timestamp:    getTime(),
			Is_playing:          statevar.IsPlaying,
			Song_time:           getPlaybackPositionInSeconds(),
			Song_start_time:     statevar.SongStartTime,
			Mute_button_clicked: mute_button_clicked,
			Is_muted:            statevar.IsMuted,
		}
		b, err := json.Marshal(data)
		if err != nil {
			panic(err)
		}

		rw.Header().Set("Content-Type", "application/json")
		rw.Write([]byte(b))
	}
}

func NextSongRequest(rw http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		defer timeTrack(time.Now(), r.RemoteAddr+" /nextsong")
		skip, _ := strconv.Atoi(r.FormValue("skip"))
		skipTrack(skip)
		data := SyncJSON{
			Current_song:     "None",
			Client_timestamp: 0,
			Server_timestamp: 0,
			Is_playing:       false,
			Song_time:        0,
			Song_start_time:  0,
		}

		b, err := json.Marshal(data)
		if err != nil {
			panic(err)
		}
		rw.Header().Set("Content-Type", "application/json")
		rw.Write([]byte(b))
	}
}

func skipTrack(song_index int) {
	if song_index < 0 {
		statevar.CurrentSongIndex += song_index + 2
	} else {
		statevar.CurrentSongIndex = song_index
	}
	song := statevar.SongList[statevar.CurrentSongIndex]

	err := os.Remove("./static/sound.mp3")
	if err != nil {
		fmt.Println(err)
	}

	// To be served by Caddy
	CopyFile(statevar.SongMap[song].Path, "./static/sound.mp3")
	statevar.MusicExtension = "mp3"
	err = os.Remove("./static/sound.webm")
	if err != nil {
		fmt.Println(err)
	}
	err = os.Remove("./static/sound.wav")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println("Converting to wav..")
	cmd := "ffmpeg"
	args := []string{"-i", "./static/sound.mp3", "-acodec", "pcm_u8", "-ar", "44100", "./static/sound.wav"}
	if err := exec.Command(cmd, args...).Run(); err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("Converting to webm..")
		cmd = "ffmpeg"
		args = []string{"-i", "./static/sound.wav", "-dash", "1", "./static/sound.webm"}
		if err := exec.Command(cmd, args...).Run(); err != nil {
			// If unsuccessful, will defualt to sound.mp3
			fmt.Println(err)
		} else {
			// If successful get rid of sound.mp3 and use sound.webm
			statevar.MusicExtension = "webm"
			err := os.Remove("./static/sound.mp3")
			if err != nil {
				fmt.Println(err)
			}
		}
	}

	rawSongData, _ = ioutil.ReadFile(statevar.SongMap[song].Path)

	statevar.CurrentSong = statevar.SongMap[song].Fullname
	statevar.SongStartTime = getTime() + 9000
	statevar.IsPlaying = false
	b, _ := json.Marshal(statevar)
	ioutil.WriteFile("state.json", b, 0644)
	go songControl(statevar.SongStartTime-getTime()-3000, false, "3", statevar.SongMap[song].Fullname, false)
	go songControl(statevar.SongStartTime-getTime()-2000, false, "2", statevar.SongMap[song].Fullname, false)
	go songControl(statevar.SongStartTime-getTime()-1000, false, "1", statevar.SongMap[song].Fullname, false)
	go songControl(statevar.SongStartTime-getTime(), true, "Playing "+statevar.SongMap[song].Fullname, statevar.SongMap[song].Fullname, false)
	go songControl(statevar.SongStartTime-getTime()+statevar.SongMap[song].Length, false, "Stopping "+statevar.SongMap[song].Fullname, statevar.SongMap[song].Fullname, true)
}
<commit_msg>Problem with skipping to beginning fixed<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

func songControl(millisecondWait int64, is_playing bool, text string, song string, start_next bool) {
	time.Sleep(time.Duration(millisecondWait) * time.Millisecond)
	if song == statevar.CurrentSong {
		log.Printf(song + " " + text)
		statevar.IsPlaying = is_playing
		if start_next == true {
			skipTrack(-1)
		}
	}
}

func getPlaylistHTML() (playlist_html string) {
	playlist_html = ""
	for i, k := range statevar.SongList {
		name := statevar.SongMap[k].Title
		names := strings.Split(name, "/")
		showName := names[len(names)-1]
		if statevar.CurrentSong != statevar.SongMap[k].Fullname {
			playlist_html += "" + showName + "
\n"
		} else {
			playlist_html += "" + showName + "
\n"

		}
	}
	return
}

func getPlaybackPositionInSeconds() float64 {
	position := float64(getTime()-statevar.SongStartTime) / 1000.0
	if statevar.IsPlaying == true && position > 0 {
		return position
	} else {
		return 0.0
	}
}

func SyncRequest(rw http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		// defer timeTrack(time.Now(), r.RemoteAddr+" /sync")
		//current_song := r.FormValue("current_song")
		client_timestamp_str := r.FormValue("client_timestamp")
		client_timestamp, _ := strconv.ParseUint(client_timestamp_str, 10, 64)
		is_muted, _ := strconv.ParseBool(r.FormValue("is_muted"))
		mute_button_clicked, _ := strconv.ParseBool(r.FormValue("mute_button_clicked"))
		if mute_button_clicked == true {
			statevar.LastMuted = getTime()
			statevar.IsMuted = is_muted
		}

		if getTime()-statevar.LastMuted < 3000 {
			mute_button_clicked = true
			is_muted = statevar.IsMuted
		}
		name := statevar.CurrentSong
		names := strings.Split(name, "/")
		showName := names[len(names)-1]
		data := SyncJSON{
			Current_song:        showName,
			Client_timestamp:    int64(client_timestamp),
			Server_timestamp:    getTime(),
			Is_playing:          statevar.IsPlaying,
			Song_time:           getPlaybackPositionInSeconds(),
			Song_start_time:     statevar.SongStartTime,
			Mute_button_clicked: mute_button_clicked,
			Is_muted:            statevar.IsMuted,
		}
		b, err := json.Marshal(data)
		if err != nil {
			panic(err)
		}

		rw.Header().Set("Content-Type", "application/json")
		rw.Write([]byte(b))
	}
}

func NextSongRequest(rw http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		defer timeTrack(time.Now(), r.RemoteAddr+" /nextsong")
		skip, _ := strconv.Atoi(r.FormValue("skip"))
		skipTrack(skip)
		data := SyncJSON{
			Current_song:     "None",
			Client_timestamp: 0,
			Server_timestamp: 0,
			Is_playing:       false,
			Song_time:        0,
			Song_start_time:  0,
		}

		b, err := json.Marshal(data)
		if err != nil {
			panic(err)
		}
		rw.Header().Set("Content-Type", "application/json")
		rw.Write([]byte(b))
	}
}

func skipTrack(song_index int) {
	if song_index < 0 {
		statevar.CurrentSongIndex += song_index + 2
	} else {
		statevar.CurrentSongIndex = song_index
	}
	if statevar.CurrentSongIndex >= len(statevar.SongList) {
		statevar.CurrentSongIndex = 0
	}
	fmt.Println(statevar.CurrentSongIndex, len(statevar.SongList))
	song := statevar.SongList[statevar.CurrentSongIndex]

	err := os.Remove("./static/sound.mp3")
	if err != nil {
		fmt.Println(err)
	}

	// To be served by Caddy
	CopyFile(statevar.SongMap[song].Path, "./static/sound.mp3")
	statevar.MusicExtension = "mp3"
	err = os.Remove("./static/sound.webm")
	if err != nil {
		fmt.Println(err)
	}
	err = os.Remove("./static/sound.wav")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println("Converting to wav..")
	cmd := "ffmpeg"
	args := []string{"-i", "./static/sound.mp3", "-acodec", "pcm_u8", "-ar", "44100", "./static/sound.wav"}
	if err := exec.Command(cmd, args...).Run(); err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("Converting to webm..")
		cmd = "ffmpeg"
		args = []string{"-i", "./static/sound.wav", "-dash", "1", "./static/sound.webm"}
		if err := exec.Command(cmd, args...).Run(); err != nil {
			// If unsuccessful, will defualt to sound.mp3
			fmt.Println(err)
		} else {
			// If successful get rid of sound.mp3 and use sound.webm
			statevar.MusicExtension = "webm"
			err := os.Remove("./static/sound.mp3")
			if err != nil {
				fmt.Println(err)
			}
		}
	}

	rawSongData, _ = ioutil.ReadFile(statevar.SongMap[song].Path)

	statevar.CurrentSong = statevar.SongMap[song].Fullname
	statevar.SongStartTime = getTime() + 9000
	statevar.IsPlaying = false
	b, _ := json.Marshal(statevar)
	ioutil.WriteFile("state.json", b, 0644)
	go songControl(statevar.SongStartTime-getTime()-3000, false, "3", statevar.SongMap[song].Fullname, false)
	go songControl(statevar.SongStartTime-getTime()-2000, false, "2", statevar.SongMap[song].Fullname, false)
	go songControl(statevar.SongStartTime-getTime()-1000, false, "1", statevar.SongMap[song].Fullname, false)
	go songControl(statevar.SongStartTime-getTime(), true, "Playing "+statevar.SongMap[song].Fullname, statevar.SongMap[song].Fullname, false)
	go songControl(statevar.SongStartTime-getTime()+statevar.SongMap[song].Length, false, "Stopping "+statevar.SongMap[song].Fullname, statevar.SongMap[song].Fullname, true)
}
<|endoftext|>