<commit_before>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/montanaflynn/stats"
)

// global command line parameters
var runs *int
var cpus [2]string
var threads *string
var hermitcore *bool

var resctrlPath *string
var cat *bool
var catBitChunk *uint64

func main() {
	commandFile := parseArgs()

	commands, err := readCommands(*commandFile)
	if err != nil {
		log.Fatalf("Error reading command file %v: %v", *commandFile, err)
	}
	if len(commands) < 2 {
		log.Fatal("You must provide at least 2 commands")
	}

	minBits := uint64(0)
	numBits := uint64(0)

	if *cat {
		var err error
		minBits, numBits, err = setupCAT()
		if err != nil {
			log.Fatalf("%v\n", err)
		}
		defer resetCAT()
	}

	commandPairs := generateCommandPairs(commands)
	catPairs := generateCatConfigs(minBits, numBits)

	fmt.Println("Executing the following command pairs:")
	for _, c := range commandPairs {
		fmt.Println(c)
	}

	for i, c := range commandPairs {
		fmt.Printf("Running pair %v\n", i)
		fmt.Println(c)

		for _, catConfig := range catPairs {
			runtimes, err := runPair(c, i, catConfig)
			if err != nil {
				log.Fatalf("Error while running pair %v (%v): %v", i, c, err)
			}

			err = processRuntime(i, c, catConfig, runtimes)
			if err != nil {
				log.Fatalf("Error processing runtime: %v", err)
			}
		}
	}
}

func processRuntime(id int, cPair [2]string, catMasks []uint64, runtimes [][]time.Duration) error {

	statsFile, err := os.Create("stats")
	if err == nil {
		defer statsFile.Close()

		// write header
		statsFile.WriteString("cmd \t avg. runtime (s) \t std. dev. \t variance \t runs")
		if *cat {
			statsFile.WriteString("\t CAT")
		}
		statsFile.WriteString("\t nom. perf\n")
	} else {
		if os.IsExist(err) {
			statsFile, err = os.OpenFile("stats", os.O_WRONLY|os.O_APPEND, 0777)
			if err != nil {
				return fmt.Errorf("Error while opening file: %v", err)
			}
			defer statsFile.Close()
		} else {
			return fmt.Errorf("Error while creating file: %v", err)
		}
	}

	for i, runtime := range runtimes {
		var runtimeSeconds []float64
		for _, r := range runtime {
			runtimeSeconds = append(runtimeSeconds, r.Seconds())
		}

		mean, _ := stats.Mean(runtimeSeconds)
		stddev, _ := stats.StandardDeviation(runtimeSeconds)
		vari, _ := stats.Variance(runtimeSeconds)
		runtimeSum, _ := stats.Sum(runtimeSeconds)

		s := fmt.Sprintf("%v \t %9.2fs avg. runtime \t %1.6f std. dev. \t %1.6f variance \t %3d runs", cPair[i], mean, stddev, vari, len(runtime))
		if *cat {
			s += fmt.Sprintf("\t %6x CAT", catMasks[i])
		} else {
			s += "\t           "
		}
		s += fmt.Sprintf("\t %1.6f nom. perf", (float64)(len(runtime))/runtimeSum)
		fmt.Println(s)

		statsFile.WriteString(fmt.Sprintf("%v \t %v \t %v \t %v \t %v", cPair[i], mean, stddev, vari, len(runtime)))
		if *cat {
			statsFile.WriteString(fmt.Sprintf("\t %6x", catMasks[i]))
		} else {
			statsFile.WriteString("\t       ")
		}
		statsFile.WriteString(fmt.Sprintf("\t %v\n", (float64)(len(runtime))/runtimeSum))
	}
	fmt.Print("\n")

	for i, runtime := range runtimes {
		filename := fmt.Sprintf("%v-%v", id, i)
		if *cat {
			filename += fmt.Sprintf("-%x", catMasks[i])
		}
		filename += ".time"
		measurementsFile, err := os.Create(filename)
		if err != nil {
			return fmt.Errorf("Error while creating file: %v", err)
		}
		defer measurementsFile.Close()

		out := "# runtime in nanoseconds of \"" + cPair[i] + "\" on CPUs " + cpus[i] + "while \"" + cPair[(i+1)%2] + "\" was running on cores " + cpus[(i+1)%len(cpus)]
		if *cat {
			out += fmt.Sprintf(" with CAT %6x ", catMasks[i])
		}
		out += "\n"
		for _, r := range runtime {
			out += strconv.FormatInt(r.Nanoseconds(), 10)
			out += "\n"
		}

		_, err = measurementsFile.WriteString(out)
		if err != nil {
			return fmt.Errorf("Error while writing measurements file: %v", err)
		}
	}

	return nil
}

func generateCommandPairs(commands []string) [][2]string {
	var pairs [][2]string
	for i, c0 := range commands {
		for j, c1 := range commands {
			if i >= j {
				continue
			}
			pairs = append(pairs, [2]string{c0, c1})
		}
	}
	return pairs
}

func readCommands(filename string) ([]string, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, errors.New("Error opening file " + filename + ": " + err.Error())
	}
	defer file.Close()

	var commands []string

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		commands = append(commands, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, errors.New("Error scanning commands: " + err.Error())
	}

	return commands, nil
}

func parseArgs() *string {
	runs = flag.Int("runs", 2, "Number of times the applications are executed")
	commandFile := flag.String("cmd", "cmd.txt", "Text file containing the commands to execute")

	cpus0 := flag.String("cpus0", "0-4", "List of CPUs to be used for the 1st command")
	cpus1 := flag.String("cpus1", "5-9", "List of CPUs to be used for the 2nd command")
	threads = flag.String("threads", "5", "Number of threads to be used")

	cat = flag.Bool("cat", false, "Measure with all CAT settings")
	catBitChunk = flag.Uint64("catChunk", 2, "Bits changed from one run to the next")
	resctrlPath = flag.String("resctrl", "/sys/fs/resctrl/", "Root path of the resctrl file system")

	hermitcore = flag.Bool("hermitcore", false, "Use if you are executing hermitcore binaries")

	flag.Parse()

	if *runs < 1 {
		fmt.Println("runs must be > 0")
		os.Exit(0)
	}
	if *catBitChunk < 1 {
		fmt.Println("catChunk must be > 0")
		os.Exit(0)
	}

	cpus[0] = *cpus0
	cpus[1] = *cpus1

	return commandFile
}
<commit_msg>Stats file is no longer overwritten.<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/montanaflynn/stats"
)

// global command line parameters
var runs *int
var cpus [2]string
var threads *string
var hermitcore *bool

var resctrlPath *string
var cat *bool
var catBitChunk *uint64

func main() {
	commandFile := parseArgs()

	commands, err := readCommands(*commandFile)
	if err != nil {
		log.Fatalf("Error reading command file %v: %v", *commandFile, err)
	}
	if len(commands) < 2 {
		log.Fatal("You must provide at least 2 commands")
	}

	minBits := uint64(0)
	numBits := uint64(0)

	if *cat {
		var err error
		minBits, numBits, err = setupCAT()
		if err != nil {
			log.Fatalf("%v\n", err)
		}
		defer resetCAT()
	}

	commandPairs := generateCommandPairs(commands)
	catPairs := generateCatConfigs(minBits, numBits)

	fmt.Println("Executing the following command pairs:")
	for _, c := range commandPairs {
		fmt.Println(c)
	}

	for i, c := range commandPairs {
		fmt.Printf("Running pair %v\n", i)
		fmt.Println(c)

		for _, catConfig := range catPairs {
			runtimes, err := runPair(c, i, catConfig)
			if err != nil {
				log.Fatalf("Error while running pair %v (%v): %v", i, c, err)
			}

			err = processRuntime(i, c, catConfig, runtimes)
			if err != nil {
				log.Fatalf("Error processing runtime: %v", err)
			}
		}
	}
}

func processRuntime(id int, cPair [2]string, catMasks []uint64, runtimes [][]time.Duration) error {

	var statsFile *os.File
	if _, err := os.Stat("stats"); os.IsNotExist(err) {
		// stats does not exist
		statsFile, err = os.Create("stats")
		if err != nil {
			return fmt.Errorf("Error while creating file: %v", err)
		}
		defer statsFile.Close()

		// write header
		statsFile.WriteString("cmd \t avg. runtime (s) \t std. dev. \t variance \t runs")
		if *cat {
			statsFile.WriteString("\t CAT")
		}
		statsFile.WriteString("\t nom. perf\n")
	} else {
		statsFile, err = os.OpenFile("stats", os.O_WRONLY|os.O_APPEND, 0777)
		if err != nil {
			return fmt.Errorf("Error while opening file: %v", err)
		}
		defer statsFile.Close()
	}

	for i, runtime := range runtimes {
		var runtimeSeconds []float64
		for _, r := range runtime {
			runtimeSeconds = append(runtimeSeconds, r.Seconds())
		}

		mean, _ := stats.Mean(runtimeSeconds)
		stddev, _ := stats.StandardDeviation(runtimeSeconds)
		vari, _ := stats.Variance(runtimeSeconds)
		runtimeSum, _ := stats.Sum(runtimeSeconds)

		s := fmt.Sprintf("%v \t %9.2fs avg. runtime \t %1.6f std. dev. \t %1.6f variance \t %3d runs", cPair[i], mean, stddev, vari, len(runtime))
		if *cat {
			s += fmt.Sprintf("\t %6x CAT", catMasks[i])
		} else {
			s += "\t           "
		}
		s += fmt.Sprintf("\t %1.6f nom. perf", (float64)(len(runtime))/runtimeSum)
		fmt.Println(s)

		statsFile.WriteString(fmt.Sprintf("%v \t %v \t %v \t %v \t %v", cPair[i], mean, stddev, vari, len(runtime)))
		if *cat {
			statsFile.WriteString(fmt.Sprintf("\t %6x", catMasks[i]))
		} else {
			statsFile.WriteString("\t       ")
		}
		statsFile.WriteString(fmt.Sprintf("\t %v\n", (float64)(len(runtime))/runtimeSum))
	}
	fmt.Print("\n")

	for i, runtime := range runtimes {
		filename := fmt.Sprintf("%v-%v", id, i)
		if *cat {
			filename += fmt.Sprintf("-%x", catMasks[i])
		}
		filename += ".time"
		measurementsFile, err := os.Create(filename)
		if err != nil {
			return fmt.Errorf("Error while creating file: %v", err)
		}
		defer measurementsFile.Close()

		out := "# runtime in nanoseconds of \"" + cPair[i] + "\" on CPUs " + cpus[i] + "while \"" + cPair[(i+1)%2] + "\" was running on cores " + cpus[(i+1)%len(cpus)]
		if *cat {
			out += fmt.Sprintf(" with CAT %6x ", catMasks[i])
		}
		out += "\n"
		for _, r := range runtime {
			out += strconv.FormatInt(r.Nanoseconds(), 10)
			out += "\n"
		}

		_, err = measurementsFile.WriteString(out)
		if err != nil {
			return fmt.Errorf("Error while writing measurements file: %v", err)
		}
	}

	return nil
}

func generateCommandPairs(commands []string) [][2]string {
	var pairs [][2]string
	for i, c0 := range commands {
		for j, c1 := range commands {
			if i >= j {
				continue
			}
			pairs = append(pairs, [2]string{c0, c1})
		}
	}
	return pairs
}

func readCommands(filename string) ([]string, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, errors.New("Error opening file " + filename + ": " + err.Error())
	}
	defer file.Close()

	var commands []string

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		commands = append(commands, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, errors.New("Error scanning commands: " + err.Error())
	}

	return commands, nil
}

func parseArgs() *string {
	runs = flag.Int("runs", 2, "Number of times the applications are executed")
	commandFile := flag.String("cmd", "cmd.txt", "Text file containing the commands to execute")

	cpus0 := flag.String("cpus0", "0-4", "List of CPUs to be used for the 1st command")
	cpus1 := flag.String("cpus1", "5-9", "List of CPUs to be used for the 2nd command")
	threads = flag.String("threads", "5", "Number of threads to be used")

	cat = flag.Bool("cat", false, "Measure with all CAT settings")
	catBitChunk = flag.Uint64("catChunk", 2, "Bits changed from one run to the next")
	resctrlPath = flag.String("resctrl", "/sys/fs/resctrl/", "Root path of the resctrl file system")

	hermitcore = flag.Bool("hermitcore", false, "Use if you are executing hermitcore binaries")

	flag.Parse()

	if *runs < 1 {
		fmt.Println("runs must be > 0")
		os.Exit(0)
	}
	if *catBitChunk < 1 {
		fmt.Println("catChunk must be > 0")
		os.Exit(0)
	}

	cpus[0] = *cpus0
	cpus[1] = *cpus1

	return commandFile
}
<|endoftext|>


<commit_before>package cmd

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/mithrandie/csvq/lib/file"
)

type writeTest struct {
	Name     string
	Filename string
	Content  string
	Result   string
	Error    string
}

var createFileTests = []writeTest{
	{
		Name:     "Create",
		Filename: "create.txt",
		Content:  "write",
		Result:   "write",
	},
	{
		Name:     "Output to Stdout",
		Filename: "",
		Content:  "write",
		Result:   "write",
	},
	{
		Name:     "File Exists Error",
		Filename: "create.txt",
		Error:    "environment-dependent",
	},
	{
		Name:     "File Open Error",
		Filename: filepath.Join("notexistdir", "create.txt"),
		Error:    "environment-dependent",
	},
}

func TestCreateFile(t *testing.T) {
	file.LockFiles = make(file.LockFileContainer)

	for _, v := range createFileTests {
		if len(v.Filename) < 1 {
			oldStdout := os.Stdout
			r, w, _ := os.Pipe()
			os.Stdout = w

			ToStdout(v.Content)

			w.Close()
			os.Stdout = oldStdout

			buf, _ := ioutil.ReadAll(r)
			if string(buf) != v.Result {
				t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
			}
		} else {
			filename := GetTestFilePath(v.Filename)
			err := CreateFile(filename, v.Content)
			if err != nil {
				if len(v.Error) < 1 {
					t.Errorf("%s: unexpected error %q", v.Name, err)
				} else if v.Error != "environment-dependent" && err.Error() != v.Error {
					t.Errorf("%s: error %q, want error %q", v.Name, err.Error(), v.Error)
				}
				continue
			}
			if 0 < len(v.Error) {
				t.Errorf("%s: no error, want error %q", v.Name, v.Error)
				continue
			}

			fp, _ := os.Open(filename)
			buf, _ := ioutil.ReadAll(fp)
			if string(buf) != v.Result {
				t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
			}
		}
	}

	file.UnlockAll()
}

var updateFileTests = []writeTest{
	{
		Name:     "Update",
		Filename: "create.txt",
		Content:  "truncate and write",
		Result:   "truncate and write",
	},
}

func TestUpdateFile(t *testing.T) {
	file.LockFiles = make(file.LockFileContainer)

	for _, v := range updateFileTests {
		filename := GetTestFilePath(v.Filename)
		fp, _ := file.OpenToUpdate(filename)
		err := UpdateFile(fp, v.Content)
		if err != nil {
			if len(v.Error) < 1 {
				t.Errorf("%s: unexpected error %q", v.Name, err)
			} else if v.Error != "environment-dependent" && err.Error() != v.Error {
				t.Errorf("%s: error %q, want error %q", v.Name, err.Error(), v.Error)
			}
			continue
		}
		if 0 < len(v.Error) {
			t.Errorf("%s: no error, want error %q", v.Name, v.Error)
			continue
		}

		fp, _ = os.Open(filename)
		buf, _ := ioutil.ReadAll(fp)
		if string(buf) != v.Result {
			t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
		}
	}

	file.UnlockAll()
}

func TestTryCreateFile(t *testing.T) {
	err := TryCreateFile(GetTestFilePath("table1.csv"))
	if err == nil {
		t.Error("Create table1.csv: no error, want error")
	}

	err = TryCreateFile(GetTestFilePath("notexist.csv"))
	if err != nil {
		t.Errorf("Create notexist.csv: unexpected error %q", err)
	} else {
		if _, err := os.Stat(GetTestFilePath("notexist.csv")); err == nil {
			t.Errorf("Create notexist.csv: temporary file does not removed")
		}
	}
}
<commit_msg>Fix a bug of file operation test.<commit_after>package cmd

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/mithrandie/csvq/lib/file"
)

type writeTest struct {
	Name     string
	Filename string
	Content  string
	Result   string
	Error    string
}

var createFileTests = []writeTest{
	{
		Name:     "Create",
		Filename: "create.txt",
		Content:  "write",
		Result:   "write",
	},
	{
		Name:     "Output to Stdout",
		Filename: "",
		Content:  "write",
		Result:   "write",
	},
	{
		Name:     "File Exists Error",
		Filename: "create.txt",
		Error:    "environment-dependent",
	},
	{
		Name:     "File Open Error",
		Filename: filepath.Join("notexistdir", "create.txt"),
		Error:    "environment-dependent",
	},
}

func TestCreateFile(t *testing.T) {
	file.LockFiles = make(file.LockFileContainer)

	for _, v := range createFileTests {
		if len(v.Filename) < 1 {
			oldStdout := os.Stdout
			r, w, _ := os.Pipe()
			os.Stdout = w

			ToStdout(v.Content)

			w.Close()
			os.Stdout = oldStdout

			buf, _ := ioutil.ReadAll(r)
			if string(buf) != v.Result {
				t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
			}
		} else {
			filename := GetTestFilePath(v.Filename)
			err := CreateFile(filename, v.Content)
			if err != nil {
				if len(v.Error) < 1 {
					t.Errorf("%s: unexpected error %q", v.Name, err)
				} else if v.Error != "environment-dependent" && err.Error() != v.Error {
					t.Errorf("%s: error %q, want error %q", v.Name, err.Error(), v.Error)
				}
				continue
			}
			if 0 < len(v.Error) {
				t.Errorf("%s: no error, want error %q", v.Name, v.Error)
				continue
			}

			fp, _ := os.Open(filename)
			buf, _ := ioutil.ReadAll(fp)
			if string(buf) != v.Result {
				t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
			}
		}
	}

	file.UnlockAll()
}

var updateFileTests = []writeTest{
	{
		Name:     "Update",
		Filename: "create.txt",
		Content:  "truncate and write",
		Result:   "truncate and write",
	},
}

func TestUpdateFile(t *testing.T) {
	file.LockFiles = make(file.LockFileContainer)

	for _, v := range updateFileTests {
		filename := GetTestFilePath(v.Filename)
		fp, _ := file.OpenToUpdate(filename)
		err := UpdateFile(fp, v.Content)
		file.Close(fp)
		if err != nil {
			if len(v.Error) < 1 {
				t.Errorf("%s: unexpected error %q", v.Name, err)
			} else if v.Error != "environment-dependent" && err.Error() != v.Error {
				t.Errorf("%s: error %q, want error %q", v.Name, err.Error(), v.Error)
			}
			continue
		}
		if 0 < len(v.Error) {
			t.Errorf("%s: no error, want error %q", v.Name, v.Error)
			continue
		}

		fp, _ = os.Open(filename)
		buf, _ := ioutil.ReadAll(fp)
		if string(buf) != v.Result {
			t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
		}
	}

	file.UnlockAll()
}

func TestTryCreateFile(t *testing.T) {
	err := TryCreateFile(GetTestFilePath("table1.csv"))
	if err == nil {
		t.Error("Create table1.csv: no error, want error")
	}

	err = TryCreateFile(GetTestFilePath("notexist.csv"))
	if err != nil {
		t.Errorf("Create notexist.csv: unexpected error %q", err)
	} else {
		if _, err := os.Stat(GetTestFilePath("notexist.csv")); err == nil {
			t.Errorf("Create notexist.csv: temporary file does not removed")
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package services

import (
	"context"
	"crypto/subtle"
	"time"

	ds "go.chromium.org/gae/service/datastore"

	"go.chromium.org/luci/common/clock"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	"go.chromium.org/luci/grpc/grpcutil"
	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/api/logpb"
	"go.chromium.org/luci/logdog/appengine/coordinator"
	"go.chromium.org/luci/logdog/appengine/coordinator/endpoints"
	"go.chromium.org/luci/logdog/appengine/coordinator/mutations"
	"go.chromium.org/luci/logdog/common/types"
	"go.chromium.org/luci/tumble"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc/codes"
)

var (
	registerStreamMetric = metric.NewCounter(
		"logdog/endpoints/register_stream",
		"Requests to register stream",
		nil,
		field.String("project"))
)

func buildLogStreamState(ls *coordinator.LogStream, lst *coordinator.LogStreamState) *logdog.LogStreamState {
	st := logdog.LogStreamState{
		ProtoVersion:  ls.ProtoVersion,
		Secret:        lst.Secret,
		TerminalIndex: lst.TerminalIndex,
		Archived:      lst.ArchivalState().Archived(),
		Purged:        ls.Purged,
	}
	if !lst.Terminated() {
		st.TerminalIndex = -1
	}
	return &st
}

// RegisterStream is an idempotent stream state register operation.
//
// Successive operations will succeed if they have the correct secret for their
// registered stream, regardless of whether the contents of their request match
// the currently registered state.
func (s *server) RegisterStream(c context.Context, req *logdog.RegisterStreamRequest) (*logdog.RegisterStreamResponse, error) {
	var path types.StreamPath

	// Unmarshal the serialized protobuf.
	var desc logpb.LogStreamDescriptor
	switch req.ProtoVersion {
	case logpb.Version:
		if err := proto.Unmarshal(req.Desc, &desc); err != nil {
			log.Fields{
				log.ErrorKey:   err,
				"protoVersion": req.ProtoVersion,
			}.Errorf(c, "Failed to unmarshal descriptor protobuf.")
			return nil, grpcutil.Errf(codes.InvalidArgument, "Failed to unmarshal protobuf.")
		}

	default:
		log.Fields{
			"protoVersion": req.ProtoVersion,
		}.Errorf(c, "Unrecognized protobuf version.")
		return nil, grpcutil.Errf(codes.InvalidArgument, "Unrecognized protobuf version: %q", req.ProtoVersion)
	}

	path = desc.Path()
	logStreamID := coordinator.LogStreamID(path)
	log.Fields{
		"project":       req.Project,
		"path":          path,
		"prospectiveID": logStreamID,
		"terminalIndex": req.TerminalIndex,
	}.Infof(c, "Registration request for log stream.")

	if err := desc.Validate(true); err != nil {
		return nil, grpcutil.Errf(codes.InvalidArgument, "Invalid log stream descriptor: %s", err)
	}
	prefix, _ := path.Split()

	// Load our service and project configs.
	cfg, err := endpoints.GetServices(c).Config(c)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to load configuration.")
		return nil, grpcutil.Internal
	}

	pcfg, err := coordinator.CurrentProjectConfig(c)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to load current project configuration.")
		return nil, grpcutil.Internal
	}

	// Load our Prefix. It must be registered.
	pfx := &coordinator.LogPrefix{ID: coordinator.LogPrefixID(prefix)}
	if err := ds.Get(c, pfx); err != nil {
		log.Fields{
			log.ErrorKey: err,
			"id":         pfx.ID,
			"prefix":     prefix,
		}.Errorf(c, "Failed to load log stream prefix.")
		if err == ds.ErrNoSuchEntity {
			return nil, grpcutil.Errf(codes.FailedPrecondition, "prefix is not registered")
		}
		return nil, grpcutil.Internal
	}

	// If we're past prefix's expiration, reject this stream.
	//
	// If the prefix doesn't have an expiration, use its creation time and apply
	// the maximum expiration.
	expirationTime := pfx.Expiration
	if expirationTime.IsZero() {
		expiration := endpoints.MinDuration(cfg.Coordinator.PrefixExpiration, pcfg.PrefixExpiration)
		if expiration > 0 {
			expirationTime = pfx.Created.Add(expiration)
		}
	}
	if now := clock.Now(c); expirationTime.IsZero() || !now.Before(expirationTime) {
		log.Fields{
			"prefix":     pfx.Prefix,
			"expiration": expirationTime,
		}.Errorf(c, "The log stream Prefix has expired.")
		return nil, grpcutil.Errf(codes.FailedPrecondition, "prefix has expired")
	}

	// The prefix secret must match the request secret. If it does, we know this
	// is a legitimate registration attempt.
	if subtle.ConstantTimeCompare(pfx.Secret, req.Secret) != 1 {
		log.Errorf(c, "Request secret does not match prefix secret.")
		return nil, grpcutil.Errf(codes.InvalidArgument, "invalid secret")
	}

	// Check for registration, and that the prefix did not expire
	// (non-transactional).
	ls := &coordinator.LogStream{ID: logStreamID}
	lst := ls.State(c)

	if err := ds.Get(c, ls, lst); err != nil {
		if !anyNoSuchEntity(err) {
			log.WithError(err).Errorf(c, "Failed to check for log stream.")
			return nil, err
		}

		// The stream does not exist. Proceed with transactional registration.
		lstKey := ds.KeyForObj(c, lst)
		err = tumble.RunUnbuffered(c, lstKey, func(c context.Context) ([]tumble.Mutation, error) {
			// Load our state and stream (transactional).
			switch err := ds.Get(c, ls, lst); {
			case err == nil:
				// The stream is already registered.
				return nil, nil

			case !anyNoSuchEntity(err):
				log.WithError(err).Errorf(c, "Failed to check for stream registration (transactional).")
				return nil, err
			}

			// The stream is not yet registered.
			log.Infof(c, "Registering new log stream.")

			// Construct our LogStreamState.
			now := clock.Now(c).UTC()
			lst.Created = now
			lst.Updated = now
			lst.Secret = pfx.Secret // Copy Prefix Secret to reduce datastore Gets.

			// Construct our LogStream.
			ls.Created = now
			ls.ProtoVersion = req.ProtoVersion

			if err := ls.LoadDescriptor(&desc); err != nil {
				log.Fields{
					log.ErrorKey: err,
				}.Errorf(c, "Failed to load descriptor into LogStream.")
				return nil, grpcutil.Errf(codes.InvalidArgument, "Failed to load descriptor.")
			}

			// If our registration request included a terminal index, terminate the
			// log stream state as well.
			if req.TerminalIndex >= 0 {
				log.Fields{
					"terminalIndex": req.TerminalIndex,
				}.Debugf(c, "Registration request included terminal index.")

				lst.TerminalIndex = req.TerminalIndex
				lst.TerminatedTime = now
			} else {
				lst.TerminalIndex = -1
			}

			if err := ds.Put(c, ls, lst); err != nil {
				log.Fields{
					log.ErrorKey: err,
				}.Errorf(c, "Failed to Put LogStream.")
				return nil, grpcutil.Internal
			}

			// Legacy pipline
			// Add a named delayed mutation to archive this stream if it's not archived
			// yet.
			//
			// If the registration did not include a terminal index, this will be our
			// pessimistic archival request, scheduled on registration to catch streams
			// that don't expire. This mutation will be replaced by the optimistic
			// archival mutation when/if the stream is terminated via TerminateStream.
			//
			// If the registration included a terminal index, apply our standard
			// parameters to the archival. Since TerminateStream will not be called,
			// this will be our formal optimistic archival task.
			optimistic := false
			params := standardArchivalParams(cfg, pcfg)
			cat := mutations.CreateArchiveTask{
				ID: ls.ID,
			}
			if req.TerminalIndex < 0 {
				// No terminal index, schedule pessimistic cleanup archival.
				cat.Expiration = now.Add(params.CompletePeriod)

				log.Fields{
					"deadline": cat.Expiration,
				}.Debugf(c, "Scheduling cleanup archival mutation.")
			} else {
				// Terminal index, schedule optimistic archival (mirrors TerminateStream).
				optimistic = true
				cat.SettleDelay = params.SettleDelay
				cat.CompletePeriod = params.CompletePeriod

				// Schedule this mutation to execute after our settle delay.
				cat.Expiration = now.Add(params.SettleDelay)

				log.Fields{
					"settleDelay":    cat.SettleDelay,
					"completePeriod": cat.CompletePeriod,
					"scheduledAt":    cat.Expiration,
				}.Debugf(c, "Scheduling archival mutation.")
			}

			aeName := cat.TaskName(c)
			if err := tumble.PutNamedMutations(c, lstKey, map[string]tumble.Mutation{aeName: &cat}); err != nil {
				log.WithError(err).Errorf(c, "Failed to write named mutations.")
				return nil, grpcutil.Internal
			}

			set := coordinator.GetSettings(c)
			percent := set.PessimisticArchivalPercent
			// TODO(hinoka): This should be set to 48hr / params.CompletePeriod, once the pipeline is migrated.
			// In all honesty this should just be a hardcoded value instead of a luci-config value.
			// No sane person is going to muck with this setting.
			delay := 47 * time.Hour
			if optimistic {
				percent = set.OptimisticalArchivalPercent
				delay = set.OptimisticArchivalDelay
			}
			return nil, TaskArchival(c, lst, delay, percent)
		})
		if err != nil {
			log.Fields{
				log.ErrorKey: err,
			}.Errorf(c, "Failed to register LogStream.")
			return nil, err
		}
	}

	registerStreamMetric.Add(c, 1, req.Project)
	return &logdog.RegisterStreamResponse{
		Id:    string(ls.ID),
		State: buildLogStreamState(ls, lst),
	}, nil
}
<commit_msg>[logdog] Also record if the stream was scheduled for an optimistic archival<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package services

import (
	"context"
	"crypto/subtle"
	"time"

	ds "go.chromium.org/gae/service/datastore"

	"go.chromium.org/luci/common/clock"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	"go.chromium.org/luci/grpc/grpcutil"
	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/api/logpb"
	"go.chromium.org/luci/logdog/appengine/coordinator"
	"go.chromium.org/luci/logdog/appengine/coordinator/endpoints"
	"go.chromium.org/luci/logdog/appengine/coordinator/mutations"
	"go.chromium.org/luci/logdog/common/types"
	"go.chromium.org/luci/tumble"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc/codes"
)

var (
	registerStreamMetric = metric.NewCounter(
		"logdog/endpoints/register_stream",
		"Requests to register stream",
		nil,
		field.String("project"),
		field.Bool("terminate"))
)

func buildLogStreamState(ls *coordinator.LogStream, lst *coordinator.LogStreamState) *logdog.LogStreamState {
	st := logdog.LogStreamState{
		ProtoVersion:  ls.ProtoVersion,
		Secret:        lst.Secret,
		TerminalIndex: lst.TerminalIndex,
		Archived:      lst.ArchivalState().Archived(),
		Purged:        ls.Purged,
	}
	if !lst.Terminated() {
		st.TerminalIndex = -1
	}
	return &st
}

// RegisterStream is an idempotent stream state register operation.
//
// Successive operations will succeed if they have the correct secret for their
// registered stream, regardless of whether the contents of their request match
// the currently registered state.
func (s *server) RegisterStream(c context.Context, req *logdog.RegisterStreamRequest) (*logdog.RegisterStreamResponse, error) {
	var path types.StreamPath

	// Unmarshal the serialized protobuf.
	var desc logpb.LogStreamDescriptor
	switch req.ProtoVersion {
	case logpb.Version:
		if err := proto.Unmarshal(req.Desc, &desc); err != nil {
			log.Fields{
				log.ErrorKey:   err,
				"protoVersion": req.ProtoVersion,
			}.Errorf(c, "Failed to unmarshal descriptor protobuf.")
			return nil, grpcutil.Errf(codes.InvalidArgument, "Failed to unmarshal protobuf.")
		}

	default:
		log.Fields{
			"protoVersion": req.ProtoVersion,
		}.Errorf(c, "Unrecognized protobuf version.")
		return nil, grpcutil.Errf(codes.InvalidArgument, "Unrecognized protobuf version: %q", req.ProtoVersion)
	}

	path = desc.Path()
	logStreamID := coordinator.LogStreamID(path)
	log.Fields{
		"project":       req.Project,
		"path":          path,
		"prospectiveID": logStreamID,
		"terminalIndex": req.TerminalIndex,
	}.Infof(c, "Registration request for log stream.")

	if err := desc.Validate(true); err != nil {
		return nil, grpcutil.Errf(codes.InvalidArgument, "Invalid log stream descriptor: %s", err)
	}
	prefix, _ := path.Split()

	// Load our service and project configs.
	cfg, err := endpoints.GetServices(c).Config(c)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to load configuration.")
		return nil, grpcutil.Internal
	}

	pcfg, err := coordinator.CurrentProjectConfig(c)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to load current project configuration.")
		return nil, grpcutil.Internal
	}

	// Load our Prefix. It must be registered.
	pfx := &coordinator.LogPrefix{ID: coordinator.LogPrefixID(prefix)}
	if err := ds.Get(c, pfx); err != nil {
		log.Fields{
			log.ErrorKey: err,
			"id":         pfx.ID,
			"prefix":     prefix,
		}.Errorf(c, "Failed to load log stream prefix.")
		if err == ds.ErrNoSuchEntity {
			return nil, grpcutil.Errf(codes.FailedPrecondition, "prefix is not registered")
		}
		return nil, grpcutil.Internal
	}

	// If we're past prefix's expiration, reject this stream.
	//
	// If the prefix doesn't have an expiration, use its creation time and apply
	// the maximum expiration.
	expirationTime := pfx.Expiration
	if expirationTime.IsZero() {
		expiration := endpoints.MinDuration(cfg.Coordinator.PrefixExpiration, pcfg.PrefixExpiration)
		if expiration > 0 {
			expirationTime = pfx.Created.Add(expiration)
		}
	}
	if now := clock.Now(c); expirationTime.IsZero() || !now.Before(expirationTime) {
		log.Fields{
			"prefix":     pfx.Prefix,
			"expiration": expirationTime,
		}.Errorf(c, "The log stream Prefix has expired.")
		return nil, grpcutil.Errf(codes.FailedPrecondition, "prefix has expired")
	}

	// The prefix secret must match the request secret. If it does, we know this
	// is a legitimate registration attempt.
	if subtle.ConstantTimeCompare(pfx.Secret, req.Secret) != 1 {
		log.Errorf(c, "Request secret does not match prefix secret.")
		return nil, grpcutil.Errf(codes.InvalidArgument, "invalid secret")
	}

	// Check for registration, and that the prefix did not expire
	// (non-transactional).
	ls := &coordinator.LogStream{ID: logStreamID}
	lst := ls.State(c)
	// If true, the stream was terminated and is scheduled for an optimistic archival.
	// Semantically, it's as if "TerminateStream" was called on this stream.
	preTerminated := false

	if err := ds.Get(c, ls, lst); err != nil {
		if !anyNoSuchEntity(err) {
			log.WithError(err).Errorf(c, "Failed to check for log stream.")
			return nil, err
		}

		// The stream does not exist. Proceed with transactional registration.
		lstKey := ds.KeyForObj(c, lst)
		err = tumble.RunUnbuffered(c, lstKey, func(c context.Context) ([]tumble.Mutation, error) {
			// Load our state and stream (transactional).
			switch err := ds.Get(c, ls, lst); {
			case err == nil:
				// The stream is already registered.
				return nil, nil

			case !anyNoSuchEntity(err):
				log.WithError(err).Errorf(c, "Failed to check for stream registration (transactional).")
				return nil, err
			}

			// The stream is not yet registered.
			log.Infof(c, "Registering new log stream.")

			// Construct our LogStreamState.
			now := clock.Now(c).UTC()
			lst.Created = now
			lst.Updated = now
			lst.Secret = pfx.Secret // Copy Prefix Secret to reduce datastore Gets.

			// Construct our LogStream.
			ls.Created = now
			ls.ProtoVersion = req.ProtoVersion

			if err := ls.LoadDescriptor(&desc); err != nil {
				log.Fields{
					log.ErrorKey: err,
				}.Errorf(c, "Failed to load descriptor into LogStream.")
				return nil, grpcutil.Errf(codes.InvalidArgument, "Failed to load descriptor.")
			}

			// If our registration request included a terminal index, terminate the
			// log stream state as well.
			if req.TerminalIndex >= 0 {
				log.Fields{
					"terminalIndex": req.TerminalIndex,
				}.Debugf(c, "Registration request included terminal index.")

				lst.TerminalIndex = req.TerminalIndex
				lst.TerminatedTime = now
			} else {
				lst.TerminalIndex = -1
			}

			if err := ds.Put(c, ls, lst); err != nil {
				log.Fields{
					log.ErrorKey: err,
				}.Errorf(c, "Failed to Put LogStream.")
				return nil, grpcutil.Internal
			}

			// Legacy pipline
			// Add a named delayed mutation to archive this stream if it's not archived
			// yet.
			//
			// If the registration did not include a terminal index, this will be our
			// pessimistic archival request, scheduled on registration to catch streams
			// that don't expire. This mutation will be replaced by the optimistic
			// archival mutation when/if the stream is terminated via TerminateStream.
			//
			// If the registration included a terminal index, apply our standard
			// parameters to the archival. Since TerminateStream will not be called,
			// this will be our formal optimistic archival task.
			params := standardArchivalParams(cfg, pcfg)
			cat := mutations.CreateArchiveTask{
				ID: ls.ID,
			}
			if req.TerminalIndex < 0 {
				// No terminal index, schedule pessimistic cleanup archival.
				cat.Expiration = now.Add(params.CompletePeriod)

				log.Fields{
					"deadline": cat.Expiration,
				}.Debugf(c, "Scheduling cleanup archival mutation.")
			} else {
				// Terminal index, schedule optimistic archival (mirrors TerminateStream).
				preTerminated = true
				cat.SettleDelay = params.SettleDelay
				cat.CompletePeriod = params.CompletePeriod

				// Schedule this mutation to execute after our settle delay.
				cat.Expiration = now.Add(params.SettleDelay)

				log.Fields{
					"settleDelay":    cat.SettleDelay,
					"completePeriod": cat.CompletePeriod,
					"scheduledAt":    cat.Expiration,
				}.Debugf(c, "Scheduling archival mutation.")
			}

			aeName := cat.TaskName(c)
			if err := tumble.PutNamedMutations(c, lstKey, map[string]tumble.Mutation{aeName: &cat}); err != nil {
				log.WithError(err).Errorf(c, "Failed to write named mutations.")
				return nil, grpcutil.Internal
			}

			set := coordinator.GetSettings(c)
			percent := set.PessimisticArchivalPercent
			// TODO(hinoka): This should be set to 48hr / params.CompletePeriod, once the pipeline is migrated.
			// In all honesty this should just be a hardcoded value instead of a luci-config value.
			// No sane person is going to muck with this setting.
			delay := 47 * time.Hour
			if preTerminated {
				percent = set.OptimisticalArchivalPercent
				delay = set.OptimisticArchivalDelay
			}
			return nil, TaskArchival(c, lst, delay, percent)
		})
		if err != nil {
			log.Fields{
				log.ErrorKey: err,
			}.Errorf(c, "Failed to register LogStream.")
			return nil, err
		}
	}

	registerStreamMetric.Add(c, 1, req.Project, preTerminated)
	return &logdog.RegisterStreamResponse{
		Id:    string(ls.ID),
		State: buildLogStreamState(ls, lst),
	}, nil
}
<|endoftext|>


<commit_before>package handler

import (
	"os"
	"path/filepath"

	"github.com/dinever/dingo/app/model"
	"github.com/dinever/dingo/app/utils"
	"github.com/dinever/golf"
)

var app *golf.Application

func Initialize() *golf.Application {
	app = golf.New()

	app.Config.Set("app/static_dir", "static")
	app.Config.Set("app.log_dir", "tmp/log")
	app.Config.Set("app/upload_dir", "upload")
	upload_dir, _ := app.Config.GetString("app/upload_dir", "upload")
	registerMiddlewares()
	registerFuncMap()
	RegisterFunctions(app)
	theme := model.GetSettingValue("theme")
	app.View.SetTemplateLoader("base", "view")
	app.View.SetTemplateLoader("admin", filepath.Join("view", "admin"))
	app.View.SetTemplateLoader("theme", filepath.Join("view", theme))
	//      static_dir, _ := app.Config.GetString("app/static_dir", "static")
	app.Static("/upload/", upload_dir)
	app.Static("/", filepath.Join("view", "admin", "assets"))
	app.Static("/", filepath.Join("view", theme, "assets"))

	app.SessionManager = golf.NewMemorySessionManager()
	app.Error(404, NotFoundHandler)

	registerAdminURLHandlers()
	registerHomeHandler()
	registerAPIHandler()

	return app
}

func registerFuncMap() {
	app.View.FuncMap["DateFormat"] = utils.DateFormat
	app.View.FuncMap["DateInt64"] = utils.DateInt64
	app.View.FuncMap["DateString"] = utils.DateString
	app.View.FuncMap["DateTime"] = utils.DateTime
	app.View.FuncMap["Now"] = utils.Now
	app.View.FuncMap["Html2Str"] = utils.Html2Str
	app.View.FuncMap["FileSize"] = utils.FileSize
	app.View.FuncMap["Setting"] = model.GetSettingValue
	app.View.FuncMap["Navigator"] = model.GetNavigators
	app.View.FuncMap["Md2html"] = utils.Markdown2HtmlTemplate
}

func registerMiddlewares() {
	app.Use(
		golf.LoggingMiddleware(os.Stdout),
		golf.RecoverMiddleware,
		golf.SessionMiddleware,
	)
}

func registerAdminURLHandlers() {
	authChain := golf.NewChain(AuthMiddleware)
	app.Get("/login/", AuthLoginPageHandler)
	app.Post("/login/", AuthLoginHandler)

	app.Get("/signup/", AuthSignUpPageHandler)
	app.Post("/signup/", AuthSignUpHandler)

	app.Get("/logout/", AuthLogoutHandler)

	app.Get("/admin/", authChain.Final(AdminHandler))

	app.Get("/admin/profile/", authChain.Final(ProfileHandler))
	app.Post("/admin/profile/", authChain.Final(ProfileChangeHandler))

	app.Get("/admin/editor/post/", authChain.Final(PostCreateHandler))
	app.Post("/admin/editor/post/", authChain.Final(PostSaveHandler))

	app.Get("/admin/editor/page/", authChain.Final(PageCreateHandler))
	app.Post("/admin/editor/page/", authChain.Final(PageSaveHandler))

	app.Get("/admin/posts/", authChain.Final(AdminPostHandler))
	app.Get("/admin/editor/:id/", authChain.Final(PostEditHandler))
	app.Post("/admin/editor/:id/", authChain.Final(PostSaveHandler))
	app.Delete("/admin/editor/:id/", authChain.Final(PostRemoveHandler))

	app.Get("/admin/pages/", authChain.Final(AdminPageHandler))

	app.Get("/admin/comments/", authChain.Final(CommentViewHandler))
	app.Post("/admin/comments/", authChain.Final(CommentAddHandler))
	app.Put("/admin/comments/", authChain.Final(CommentUpdateHandler))
	app.Delete("/admin/comments/", authChain.Final(CommentRemoveHandler))

	app.Get("/admin/setting/", authChain.Final(SettingViewHandler))
	app.Post("/admin/setting/", authChain.Final(SettingUpdateHandler))
	app.Post("/admin/setting/custom/", authChain.Final(SettingCustomHandler))
	app.Post("/admin/setting/nav/", authChain.Final(SettingNavHandler))
	//
	app.Get("/admin/files/", authChain.Final(FileViewHandler))
	app.Delete("/admin/files/", authChain.Final(FileRemoveHandler))
	app.Post("/admin/files/upload/", authChain.Final(FileUploadHandler))

	app.Get("/admin/password/", authChain.Final(AdminPasswordPage))
	app.Post("/admin/password/", authChain.Final(AdminPasswordChange))

	app.Get("/admin/monitor/", authChain.Final(AdminMonitorPage))
}

func registerHomeHandler() {
	statsChain := golf.NewChain()
	app.Get("/", statsChain.Final(HomeHandler))
	app.Get("/page/:page/", HomeHandler)
	app.Post("/comment/:id/", CommentHandler)
	app.Get("/tag/:tag/", TagHandler)
	app.Get("/tag/:tag/page/:page/", TagHandler)
	app.Get("/feed/", RssHandler)
	app.Get("/sitemap.xml", SiteMapHandler)
	app.Get("/:slug/", statsChain.Final(ContentHandler))
}

func registerAPIHandler() {
	// Auth
	app.Post("/auth", JWTAuthLoginHandler)
	app.Get("/auth", JWTAuthValidateHandler)

	// register the API handler
	app.Get("/api", APIDocumentationHandler)

	// Posts
	app.Get("/api/posts", APIPostsHandler)
	app.Get("/api/posts/:id", APIPostHandler)
	app.Get("/api/posts/slug/:slug", APIPostSlugHandler)

	// Tags
	app.Get("/api/tags", APITagsHandler)
	app.Get("/api/tags/:id", APITagHandler)
	app.Get("/api/tags/slug/:slug", APITagSlugHandler)

	// Users
	app.Get("/api/users", APIUsersHandler)
	app.Get("/api/users/:id", APIUserHandler)
	app.Get("/api/users/slug/:slug", APIUserSlugHandler)
	app.Get("/api/users/email/:email", APIUserEmailHandler)
}
<commit_msg>[fix] Fix admin assets static path<commit_after>package handler

import (
	"os"
	"path/filepath"

	"github.com/dinever/dingo/app/model"
	"github.com/dinever/dingo/app/utils"
	"github.com/dinever/golf"
)

var app *golf.Application

func Initialize() *golf.Application {
	app = golf.New()

	app.Config.Set("app/static_dir", "static")
	app.Config.Set("app.log_dir", "tmp/log")
	app.Config.Set("app/upload_dir", "upload")
	upload_dir, _ := app.Config.GetString("app/upload_dir", "upload")
	registerMiddlewares()
	registerFuncMap()
	RegisterFunctions(app)
	theme := model.GetSettingValue("theme")
	app.View.SetTemplateLoader("base", "view")
	app.View.SetTemplateLoader("admin", filepath.Join("view", "admin"))
	app.View.SetTemplateLoader("theme", filepath.Join("view", theme))
	//      static_dir, _ := app.Config.GetString("app/static_dir", "static")
	app.Static("/upload/", upload_dir)
	app.Static("/", filepath.Join("view", "admin", "assets", "dist"))
	app.Static("/", filepath.Join("view", theme, "assets"))

	app.SessionManager = golf.NewMemorySessionManager()
	app.Error(404, NotFoundHandler)

	registerAdminURLHandlers()
	registerHomeHandler()
	registerAPIHandler()

	return app
}

func registerFuncMap() {
	app.View.FuncMap["DateFormat"] = utils.DateFormat
	app.View.FuncMap["DateInt64"] = utils.DateInt64
	app.View.FuncMap["DateString"] = utils.DateString
	app.View.FuncMap["DateTime"] = utils.DateTime
	app.View.FuncMap["Now"] = utils.Now
	app.View.FuncMap["Html2Str"] = utils.Html2Str
	app.View.FuncMap["FileSize"] = utils.FileSize
	app.View.FuncMap["Setting"] = model.GetSettingValue
	app.View.FuncMap["Navigator"] = model.GetNavigators
	app.View.FuncMap["Md2html"] = utils.Markdown2HtmlTemplate
}

func registerMiddlewares() {
	app.Use(
		golf.LoggingMiddleware(os.Stdout),
		golf.RecoverMiddleware,
		golf.SessionMiddleware,
	)
}

func registerAdminURLHandlers() {
	authChain := golf.NewChain(AuthMiddleware)
	app.Get("/login/", AuthLoginPageHandler)
	app.Post("/login/", AuthLoginHandler)

	app.Get("/signup/", AuthSignUpPageHandler)
	app.Post("/signup/", AuthSignUpHandler)

	app.Get("/logout/", AuthLogoutHandler)

	app.Get("/admin/", authChain.Final(AdminHandler))

	app.Get("/admin/profile/", authChain.Final(ProfileHandler))
	app.Post("/admin/profile/", authChain.Final(ProfileChangeHandler))

	app.Get("/admin/editor/post/", authChain.Final(PostCreateHandler))
	app.Post("/admin/editor/post/", authChain.Final(PostSaveHandler))

	app.Get("/admin/editor/page/", authChain.Final(PageCreateHandler))
	app.Post("/admin/editor/page/", authChain.Final(PageSaveHandler))

	app.Get("/admin/posts/", authChain.Final(AdminPostHandler))
	app.Get("/admin/editor/:id/", authChain.Final(PostEditHandler))
	app.Post("/admin/editor/:id/", authChain.Final(PostSaveHandler))
	app.Delete("/admin/editor/:id/", authChain.Final(PostRemoveHandler))

	app.Get("/admin/pages/", authChain.Final(AdminPageHandler))

	app.Get("/admin/comments/", authChain.Final(CommentViewHandler))
	app.Post("/admin/comments/", authChain.Final(CommentAddHandler))
	app.Put("/admin/comments/", authChain.Final(CommentUpdateHandler))
	app.Delete("/admin/comments/", authChain.Final(CommentRemoveHandler))

	app.Get("/admin/setting/", authChain.Final(SettingViewHandler))
	app.Post("/admin/setting/", authChain.Final(SettingUpdateHandler))
	app.Post("/admin/setting/custom/", authChain.Final(SettingCustomHandler))
	app.Post("/admin/setting/nav/", authChain.Final(SettingNavHandler))
	//
	app.Get("/admin/files/", authChain.Final(FileViewHandler))
	app.Delete("/admin/files/", authChain.Final(FileRemoveHandler))
	app.Post("/admin/files/upload/", authChain.Final(FileUploadHandler))

	app.Get("/admin/password/", authChain.Final(AdminPasswordPage))
	app.Post("/admin/password/", authChain.Final(AdminPasswordChange))

	app.Get("/admin/monitor/", authChain.Final(AdminMonitorPage))
}

func registerHomeHandler() {
	statsChain := golf.NewChain()
	app.Get("/", statsChain.Final(HomeHandler))
	app.Get("/page/:page/", HomeHandler)
	app.Post("/comment/:id/", CommentHandler)
	app.Get("/tag/:tag/", TagHandler)
	app.Get("/tag/:tag/page/:page/", TagHandler)
	app.Get("/feed/", RssHandler)
	app.Get("/sitemap.xml", SiteMapHandler)
	app.Get("/:slug/", statsChain.Final(ContentHandler))
}

func registerAPIHandler() {
	// Auth
	app.Post("/auth", JWTAuthLoginHandler)
	app.Get("/auth", JWTAuthValidateHandler)

	// register the API handler
	app.Get("/api", APIDocumentationHandler)

	// Posts
	app.Get("/api/posts", APIPostsHandler)
	app.Get("/api/posts/:id", APIPostHandler)
	app.Get("/api/posts/slug/:slug", APIPostSlugHandler)

	// Tags
	app.Get("/api/tags", APITagsHandler)
	app.Get("/api/tags/:id", APITagHandler)
	app.Get("/api/tags/slug/:slug", APITagSlugHandler)

	// Users
	app.Get("/api/users", APIUsersHandler)
	app.Get("/api/users/:id", APIUserHandler)
	app.Get("/api/users/slug/:slug", APIUserSlugHandler)
	app.Get("/api/users/email/:email", APIUserEmailHandler)
}
<|endoftext|>


<commit_before>package image_ecosystem

import (
	"fmt"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	exutil "github.com/openshift/origin/test/extended/util"
	dbutil "github.com/openshift/origin/test/extended/util/db"
)

var _ = g.Describe("[image_ecosystem][mongodb][Slow] openshift mongodb replication (with statefulset)", func() {
	defer g.GinkgoRecover()

	const templatePath = "https://raw.githubusercontent.com/sclorg/mongodb-container/master/examples/petset/mongodb-petset-persistent.yaml"

	oc := exutil.NewCLI("mongodb-petset-replica", exutil.KubeConfigPath()).Verbose()

	g.Describe("creating from a template", func() {
		g.AfterEach(func() {
			for i := 0; i < 3; i++ {
				pod := fmt.Sprintf("mongodb-replicaset-%d", i)
				podLogs, err := oc.Run("logs").Args(pod, "--timestamps").Output()
				if err != nil {
					ginkgolog("error retrieving pod logs for %s: %v", pod, err)
					continue
				}
				ginkgolog("pod logs for %s:\n%s", podLogs, err)
			}
		})
		g.It(fmt.Sprintf("should process and create the %q template", templatePath), func() {
			oc.SetOutputDir(exutil.TestContext.OutputDir)

			g.By("creating persistent volumes")
			_, err := exutil.SetupHostPathVolumes(
				oc.AdminKubeClient().Core().PersistentVolumes(),
				oc.Namespace(),
				"256Mi",
				3,
			)
			o.Expect(err).NotTo(o.HaveOccurred())

			defer func() {
				// We're removing only PVs because all other things will be removed
				// together with namespace.
				err := exutil.CleanupHostPathVolumes(oc.AdminKubeClient().Core().PersistentVolumes(), oc.Namespace())
				if err != nil {
					ginkgolog("WARNING: couldn't cleanup persistent volumes: %v", err)
				}
			}()

			g.By("creating a new app")
			o.Expect(
				oc.Run("new-app").Args(
					"-f", templatePath,
					"-p", "VOLUME_CAPACITY=256Mi",
					"-p", "MEMORY_LIMIT=512Mi",
					"-p", "MONGODB_IMAGE=centos/mongodb-32-centos7",
					"-p", "MONGODB_SERVICE_NAME=mongodb-replicaset",
				).Execute(),
			).Should(o.Succeed())

			g.By("waiting for all pods to reach running status")
			podNames, err := exutil.WaitForPods(
				oc.KubeClient().Core().Pods(oc.Namespace()),
				exutil.ParseLabelsOrDie("name=mongodb-replicaset"),
				exutil.CheckPodIsRunningFn,
				3,
				4*time.Minute,
			)
			if err != nil {
				desc, _ := oc.Run("describe").Args("statefulset").Output()
				ginkgolog("\n\nStatefulset at failure:\n%s\n\n", desc)
				desc, _ = oc.Run("describe").Args("pods").Output()
				ginkgolog("\n\nPods at statefulset failure:\n%s\n\n", desc)
			}
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("expecting that we can insert a new record on primary node")
			mongo := dbutil.NewMongoDB(podNames[0])
			replicaSet := mongo.(exutil.ReplicaSet)
			out, err := replicaSet.QueryPrimary(oc, `db.test.save({ "status" : "passed" })`)
			ginkgolog("save result: %s\n", out)
			o.Expect(err).ShouldNot(o.HaveOccurred())

			g.By("expecting that we can read a record from all members")
			for _, podName := range podNames {
				o.Expect(readRecordFromPod(oc, podName)).To(o.Succeed())
			}

			g.By("restarting replica set")
			err = oc.Run("delete").Args("pods", "--all", "-n", oc.Namespace()).Execute()
			o.Expect(err).ShouldNot(o.HaveOccurred())

			g.By("waiting for all pods to reach running status")
			podNames, err = exutil.WaitForPods(
				oc.KubeClient().Core().Pods(oc.Namespace()),
				exutil.ParseLabelsOrDie("name=mongodb-replicaset"),
				exutil.CheckPodIsRunningFn,
				3,
				2*time.Minute,
			)
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("expecting that we can read a record from all members after its restart")
			for _, podName := range podNames {
				o.Expect(readRecordFromPod(oc, podName)).To(o.Succeed())
			}
		})
	})

})

func readRecordFromPod(oc *exutil.CLI, podName string) error {
	// don't include _id field to output because it changes every time
	findCmd := "rs.slaveOk(); printjson(db.test.find({}, {_id: 0}).toArray())"

	fmt.Fprintf(g.GinkgoWriter, "DEBUG: reading record from the pod %v\n", podName)

	mongoPod := dbutil.NewMongoDB(podName)
	// pod is running but we need to wait when it will be really ready
	// (will become a member of replica set and will finish data sync)
	return exutil.WaitForQueryOutputContains(oc, mongoPod, 1*time.Minute, false, findCmd, `{ "status" : "passed" }`)
}
<commit_msg>Take more care in handling statefulset pods in mongodb test<commit_after>package image_ecosystem

import (
	"fmt"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	exutil "github.com/openshift/origin/test/extended/util"
	dbutil "github.com/openshift/origin/test/extended/util/db"
	kapi "k8s.io/kubernetes/pkg/api"
)

var _ = g.Describe("[image_ecosystem][mongodb][Slow] openshift mongodb replication (with statefulset)", func() {
	defer g.GinkgoRecover()

	const templatePath = "https://raw.githubusercontent.com/sclorg/mongodb-container/master/examples/petset/mongodb-petset-persistent.yaml"

	oc := exutil.NewCLI("mongodb-petset-replica", exutil.KubeConfigPath()).Verbose()

	g.Describe("creating from a template", func() {
		g.AfterEach(func() {
			for i := 0; i < 3; i++ {
				pod := fmt.Sprintf("mongodb-replicaset-%d", i)
				podLogs, err := oc.Run("logs").Args(pod, "--timestamps").Output()
				if err != nil {
					ginkgolog("error retrieving pod logs for %s: %v", pod, err)
					continue
				}
				ginkgolog("pod logs for %s:\n%s", podLogs, err)
			}
		})
		g.It(fmt.Sprintf("should process and create the %q template", templatePath), func() {
			oc.SetOutputDir(exutil.TestContext.OutputDir)

			g.By("creating persistent volumes")
			_, err := exutil.SetupHostPathVolumes(
				oc.AdminKubeClient().Core().PersistentVolumes(),
				oc.Namespace(),
				"256Mi",
				3,
			)
			o.Expect(err).NotTo(o.HaveOccurred())

			defer func() {
				// We're removing only PVs because all other things will be removed
				// together with namespace.
				err := exutil.CleanupHostPathVolumes(oc.AdminKubeClient().Core().PersistentVolumes(), oc.Namespace())
				if err != nil {
					ginkgolog("WARNING: couldn't cleanup persistent volumes: %v", err)
				}
			}()

			g.By("creating a new app")
			o.Expect(
				oc.Run("new-app").Args(
					"-f", templatePath,
					"-p", "VOLUME_CAPACITY=256Mi",
					"-p", "MEMORY_LIMIT=512Mi",
					"-p", "MONGODB_IMAGE=centos/mongodb-32-centos7",
					"-p", "MONGODB_SERVICE_NAME=mongodb-replicaset",
				).Execute(),
			).Should(o.Succeed())

			g.By("waiting for all pods to reach ready status")
			podNames, err := exutil.WaitForPods(
				oc.KubeClient().Core().Pods(oc.Namespace()),
				exutil.ParseLabelsOrDie("name=mongodb-replicaset"),
				exutil.CheckPodIsReadyFn,
				3,
				4*time.Minute,
			)
			if err != nil {
				desc, _ := oc.Run("describe").Args("statefulset").Output()
				ginkgolog("\n\nStatefulset at failure:\n%s\n\n", desc)
				desc, _ = oc.Run("describe").Args("pods").Output()
				ginkgolog("\n\nPods at statefulset failure:\n%s\n\n", desc)
			}
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("expecting that we can insert a new record on primary node")
			mongo := dbutil.NewMongoDB(podNames[0])
			replicaSet := mongo.(exutil.ReplicaSet)
			out, err := replicaSet.QueryPrimary(oc, `db.test.save({ "status" : "passed" })`)
			ginkgolog("save result: %s\n", out)
			o.Expect(err).ShouldNot(o.HaveOccurred())

			g.By("expecting that we can read a record from all members")
			for _, podName := range podNames {
				o.Expect(readRecordFromPod(oc, podName)).To(o.Succeed())
			}

			g.By("restarting replica set")
			err = oc.Run("delete").Args("pods", "--all", "-n", oc.Namespace()).Execute()
			o.Expect(err).ShouldNot(o.HaveOccurred())

			g.By("waiting for all pods to be gracefully deleted")
			podNames, err = exutil.WaitForPods(
				oc.KubeClient().Core().Pods(oc.Namespace()),
				exutil.ParseLabelsOrDie("name=mongodb-replicaset"),
				func(pod kapi.Pod) bool { return pod.DeletionTimestamp != nil },
				0,
				2*time.Minute,
			)
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("waiting for all pods to reach ready status")
			podNames, err = exutil.WaitForPods(
				oc.KubeClient().Core().Pods(oc.Namespace()),
				exutil.ParseLabelsOrDie("name=mongodb-replicaset"),
				exutil.CheckPodIsReadyFn,
				3,
				2*time.Minute,
			)
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("expecting that we can read a record from all members after its restart")
			for _, podName := range podNames {
				o.Expect(readRecordFromPod(oc, podName)).To(o.Succeed())
			}
		})
	})

})

func readRecordFromPod(oc *exutil.CLI, podName string) error {
	// don't include _id field to output because it changes every time
	findCmd := "rs.slaveOk(); printjson(db.test.find({}, {_id: 0}).toArray())"

	fmt.Fprintf(g.GinkgoWriter, "DEBUG: reading record from the pod %v\n", podName)

	mongoPod := dbutil.NewMongoDB(podName)
	// pod is running but we need to wait when it will be really ready
	// (will become a member of replica set and will finish data sync)
	return exutil.WaitForQueryOutputContains(oc, mongoPod, 1*time.Minute, false, findCmd, `{ "status" : "passed" }`)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/fatih/color"
	"os"
)

// Command is a representation of an executable command
type Command struct {
	ID      string
	Summary string `yaml:"summary"`
	Command string `yaml:"command"`
	// TODO(thiderman): Extend to contain Summary
	Hosts map[string]string `yaml:"hosts"`
}

// Execute will execute the command specified by the item.
//
// If the `host` attribute is set, the command will be executed on the host(s)
// specified.
func (c *Command) Execute(r *Repo, cl cli.Args) {
	blue := color.New(color.FgBlue, color.Bold).SprintfFunc()
	magenta := color.New(color.FgMagenta, color.Bold).SprintfFunc()
	yellow := color.New(color.FgYellow, color.Bold).SprintfFunc()
	green := color.New(color.FgGreen, color.Bold).SprintfFunc()

	if len(cl) == 0 {
		fmt.Println("Specify host targets:")
		for key, def := range c.Hosts {
			fmt.Println(
				fmt.Sprintf(
					"  %s: %s",
					green(key),
					yellow(def),
				),
			)
		}
		return
	}

	hostdef := c.Hosts[cl[0]]

	fmt.Println(
		fmt.Sprintf("%s: %s\nRuns %s on hosts matching %s\n",
			blue(c.ID),
			magenta(c.Summary),
			yellow(c.Command),
			green(hostdef),
		),
	)

	if !ask("Do you want to continue? [y/N] ") {
		fmt.Println("Doing nothing.")

		os.Exit(1)
	}

	repo := r.ParentRepo()
	host := repo.GetHost(hostdef)
	host.Execute(c.Command)
	return

	// sh, _ := exec.LookPath("sh")
	// args := []string{sh, "-c", c.Command}

	// cmd := exec.Cmd{
	// 	Path:   sh,
	// 	Args:   args,
	// 	Stdout: os.Stdout,
	// 	Stderr: os.Stderr,
	// }
	// err := cmd.Run()
	// if err != nil {
	// 	log.Fatal("oh noes :(")
	// }

}

func (c *Command) getHosts(args cli.Args) (names []string) {
	return
}
<commit_msg>Remove commented code<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/fatih/color"
	"os"
)

// Command is a representation of an executable command
type Command struct {
	ID      string
	Summary string `yaml:"summary"`
	Command string `yaml:"command"`
	// TODO(thiderman): Extend to contain Summary
	Hosts map[string]string `yaml:"hosts"`
}

// Execute will execute the command specified by the item.
//
// If the `host` attribute is set, the command will be executed on the host(s)
// specified.
func (c *Command) Execute(r *Repo, cl cli.Args) {
	blue := color.New(color.FgBlue, color.Bold).SprintfFunc()
	magenta := color.New(color.FgMagenta, color.Bold).SprintfFunc()
	yellow := color.New(color.FgYellow, color.Bold).SprintfFunc()
	green := color.New(color.FgGreen, color.Bold).SprintfFunc()

	if len(cl) == 0 {
		fmt.Println("Specify host targets:")
		for key, def := range c.Hosts {
			fmt.Println(
				fmt.Sprintf(
					"  %s: %s",
					green(key),
					yellow(def),
				),
			)
		}
		return
	}

	hostdef := c.Hosts[cl[0]]

	fmt.Println(
		fmt.Sprintf("%s: %s\nRuns %s on hosts matching %s\n",
			blue(c.ID),
			magenta(c.Summary),
			yellow(c.Command),
			green(hostdef),
		),
	)

	if !ask("Do you want to continue? [y/N] ") {
		fmt.Println("Doing nothing.")

		os.Exit(1)
	}

	repo := r.ParentRepo()
	host := repo.GetHost(hostdef)
	host.Execute(c.Command)
	return
}

func (c *Command) getHosts(args cli.Args) (names []string) {
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbs "github.com/brotherlogic/goserver/proto"
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	runner *Runner
	disk   diskChecker
	jobs   map[string]*pb.JobDetails
}

func (s *Server) monitor(job *pb.JobDetails) {
	for true {
		switch job.State {
		case pb.JobDetails_ACKNOWLEDGED:
			job.StartTime = 0
			job.State = pb.JobDetails_BUILDING
			s.runner.Checkout(job.GetSpec().Name)
			job.State = pb.JobDetails_BUILT
		case pb.JobDetails_BUILT:
			s.runner.Run(job)
			for job.StartTime == 0 {
				time.Sleep(waitTime)
			}
			job.State = pb.JobDetails_PENDING
		case pb.JobDetails_KILLING:
			s.runner.kill(job)
			if !isAlive(job.GetSpec()) {
				log.Printf("SET TO DEAD BECAUSE WE'RE KILLING: %v", job)
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_UPDATE_STARTING:
			s.runner.Update(job)
			job.State = pb.JobDetails_RUNNING
		case pb.JobDetails_PENDING:
			time.Sleep(time.Minute)
			if isAlive(job.GetSpec()) {
				job.State = pb.JobDetails_RUNNING
			} else {
				log.Printf("FOUND DEAD ON PENDING: %v", job)
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_RUNNING:
			time.Sleep(waitTime)
			if !isAlive(job.GetSpec()) {
				log.Printf("FOUND DEAD WHEN RUNNING: %v", job)
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_DEAD:
			log.Printf("RERUNNING BECAUSE WERE DEAD (%v)", job)
			job.State = pb.JobDetails_ACKNOWLEDGED
		}
	}
}

func getHash(file string) (string, error) {
	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"

	f, err := os.Open(strings.Replace(file, "$GOPATH", gpath, 1))
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}

	return string(h.Sum(nil)), nil
}

func getIP(name string, server string) (string, int) {
	conn, _ := grpc.Dial("192.168.86.64:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	entry := pbd.RegistryEntry{Name: name, Identifier: server}
	r, err := registry.Discover(context.Background(), &entry)

	if err != nil {
		return "", -1
	}

	return r.Ip, int(r.Port)
}

// updateState of the runner command
func isAlive(spec *pb.JobSpec) bool {
	elems := strings.Split(spec.Name, "/")
	dServer, dPort := getIP(elems[len(elems)-1], spec.Server)

	if dPort > 0 {
		dConn, err := grpc.Dial(dServer+":"+strconv.Itoa(dPort), grpc.WithInsecure())
		if err != nil {
			return false
		}
		defer dConn.Close()

		c := pbs.NewGoserverServiceClient(dConn)
		resp, err := c.IsAlive(context.Background(), &pbs.Alive{})

		if err != nil || resp.Name != elems[len(elems)-1] {
			log.Printf("FOUND DEAD SERVER: (%v) %v -> %v", spec, err, resp)
			return false
		}

		return true
	}

	//Mark as false if we can't locate the job
	return false
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildSlaveServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//Init builds the default runner framework
func Init() *Runner {
	r := &Runner{gopath: "goautobuild", m: &sync.Mutex{}}
	r.runner = runCommand
	go r.run()
	return r
}

func runCommand(c *runnerCommand) {
	if c == nil || c.command == nil {
		return
	}

	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"
	c.command.Path = strings.Replace(c.command.Path, "$GOPATH", gpath, -1)
	for i := range c.command.Args {
		c.command.Args[i] = strings.Replace(c.command.Args[i], "$GOPATH", gpath, -1)
	}

	path := fmt.Sprintf("GOPATH=" + home + "/gobuild")
	found := false
	envl := os.Environ()
	for i, blah := range envl {
		if strings.HasPrefix(blah, "GOPATH") {
			envl[i] = path
			found = true
		}
	}
	if !found {
		envl = append(envl, path)
	}
	c.command.Env = envl

	out, _ := c.command.StdoutPipe()

	c.command.Start()

	if !c.background {
		str := ""

		if out != nil {
			buf := new(bytes.Buffer)
			buf.ReadFrom(out)
			str = buf.String()
		}

		c.command.Wait()
		c.output = str
		c.complete = true
	} else {
		c.details.StartTime = time.Now().Unix()
	}
}

func (diskChecker prodDiskChecker) diskUsage(path string) int64 {
	return diskUsage(path)
}

func (s *Server) rebuildLoop() {
	for true {
		time.Sleep(time.Minute * 60)

		var rebuildList []*pb.JobDetails
		var hashList []string
		for _, job := range s.runner.backgroundTasks {
			if time.Since(job.started) > time.Hour {
				rebuildList = append(rebuildList, job.details)
				hashList = append(hashList, job.hash)
			}
		}

		for i := range rebuildList {
			s.runner.Rebuild(rebuildList[i], hashList[i])
		}
	}
}

func main() {
	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	s := Server{&goserver.GoServer{}, Init(), prodDiskChecker{}, make(map[string]*pb.JobDetails)}
	s.Register = s
	s.PrepServer()
	s.GoServer.Killme = false
	s.RegisterServingTask(s.rebuildLoop)
	s.RegisterServer("gobuildslave", false)
	s.Serve()
}
<commit_msg>More logging<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pbd "github.com/brotherlogic/discovery/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbs "github.com/brotherlogic/goserver/proto"
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	runner *Runner
	disk   diskChecker
	jobs   map[string]*pb.JobDetails
}

func (s *Server) monitor(job *pb.JobDetails) {
	for true {
		switch job.State {
		case pb.JobDetails_ACKNOWLEDGED:
			job.StartTime = 0
			job.State = pb.JobDetails_BUILDING
			s.runner.Checkout(job.GetSpec().Name)
			job.State = pb.JobDetails_BUILT
		case pb.JobDetails_BUILT:
			s.runner.Run(job)
			for job.StartTime == 0 {
				time.Sleep(waitTime)
			}
			job.State = pb.JobDetails_PENDING
		case pb.JobDetails_KILLING:
			s.runner.kill(job)
			if !isAlive(job.GetSpec()) {
				log.Printf("SET TO DEAD BECAUSE WE'RE KILLING: %v", job)
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_UPDATE_STARTING:
			s.runner.Update(job)
			job.State = pb.JobDetails_RUNNING
		case pb.JobDetails_PENDING:
			time.Sleep(time.Minute)
			if isAlive(job.GetSpec()) {
				job.State = pb.JobDetails_RUNNING
			} else {
				log.Printf("FOUND DEAD ON PENDING: %v", job)
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_RUNNING:
			time.Sleep(waitTime)
			if !isAlive(job.GetSpec()) {
				log.Printf("FOUND DEAD WHEN RUNNING: %v", job)
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_DEAD:
			log.Printf("RERUNNING BECAUSE WERE DEAD (%v)", job)
			job.State = pb.JobDetails_ACKNOWLEDGED
		}
	}
}

func getHash(file string) (string, error) {
	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"

	f, err := os.Open(strings.Replace(file, "$GOPATH", gpath, 1))
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}

	return string(h.Sum(nil)), nil
}

func getIP(name string, server string) (string, int) {
	conn, _ := grpc.Dial("192.168.86.64:50055", grpc.WithInsecure())
	defer conn.Close()

	registry := pbd.NewDiscoveryServiceClient(conn)
	entry := pbd.RegistryEntry{Name: name, Identifier: server}
	r, err := registry.Discover(context.Background(), &entry)

	if err != nil {
		return "", -1
	}

	return r.Ip, int(r.Port)
}

// updateState of the runner command
func isAlive(spec *pb.JobSpec) bool {
	elems := strings.Split(spec.Name, "/")
	dServer, dPort := getIP(elems[len(elems)-1], spec.Server)

	if dPort > 0 {
		dConn, err := grpc.Dial(dServer+":"+strconv.Itoa(dPort), grpc.WithInsecure())
		if err != nil {
			return false
		}
		defer dConn.Close()

		c := pbs.NewGoserverServiceClient(dConn)
		resp, err := c.IsAlive(context.Background(), &pbs.Alive{})

		if err != nil || resp.Name != elems[len(elems)-1] {
			log.Printf("FOUND DEAD SERVER: (%v) %v -> %v", spec, err, resp)
			return false
		}

		return true
	}

	log.Printf("Failed to locate %v ->%v (%v, %v)", spec, elems[len(elems)-1], dServer, dPort)
	//Mark as false if we can't locate the job
	return false
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildSlaveServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

//Init builds the default runner framework
func Init() *Runner {
	r := &Runner{gopath: "goautobuild", m: &sync.Mutex{}}
	r.runner = runCommand
	go r.run()
	return r
}

func runCommand(c *runnerCommand) {
	if c == nil || c.command == nil {
		return
	}

	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"
	c.command.Path = strings.Replace(c.command.Path, "$GOPATH", gpath, -1)
	for i := range c.command.Args {
		c.command.Args[i] = strings.Replace(c.command.Args[i], "$GOPATH", gpath, -1)
	}

	path := fmt.Sprintf("GOPATH=" + home + "/gobuild")
	found := false
	envl := os.Environ()
	for i, blah := range envl {
		if strings.HasPrefix(blah, "GOPATH") {
			envl[i] = path
			found = true
		}
	}
	if !found {
		envl = append(envl, path)
	}
	c.command.Env = envl

	out, _ := c.command.StdoutPipe()

	c.command.Start()

	if !c.background {
		str := ""

		if out != nil {
			buf := new(bytes.Buffer)
			buf.ReadFrom(out)
			str = buf.String()
		}

		c.command.Wait()
		c.output = str
		c.complete = true
	} else {
		c.details.StartTime = time.Now().Unix()
	}
}

func (diskChecker prodDiskChecker) diskUsage(path string) int64 {
	return diskUsage(path)
}

func (s *Server) rebuildLoop() {
	for true {
		time.Sleep(time.Minute * 60)

		var rebuildList []*pb.JobDetails
		var hashList []string
		for _, job := range s.runner.backgroundTasks {
			if time.Since(job.started) > time.Hour {
				rebuildList = append(rebuildList, job.details)
				hashList = append(hashList, job.hash)
			}
		}

		for i := range rebuildList {
			s.runner.Rebuild(rebuildList[i], hashList[i])
		}
	}
}

func main() {
	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	s := Server{&goserver.GoServer{}, Init(), prodDiskChecker{}, make(map[string]*pb.JobDetails)}
	s.Register = s
	s.PrepServer()
	s.GoServer.Killme = false
	s.RegisterServingTask(s.rebuildLoop)
	s.RegisterServer("gobuildslave", false)
	s.Serve()
}
<|endoftext|>


<commit_before>// Package ca provides easy to use certificate authority related functions.
// This is a lightweight wrapper around "crypto/x509" package for
// creating CA certs, client certs, signing requests, and more.
//
// This package is mostly based on the example code provided at:
// http://golang.org/src/crypto/tls/generate_cert.go
package ca

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"fmt"
	"math/big"
	"net"
	"strings"
	"time"
)

// GenCA generates a self-signed CA certificate and returns the PEM encoded X.509 certificate and private key pair.
// Note: while writing the binary cert/key pair to file system, it is useful to use standard naming like: 'cert.pem', 'key.pem'.
func GenCA() (cert, key []byte, err error) {
	return nil, nil, nil
}

// GenSigningCert generates an intermediate signing certificate for signing server or client certificates.
func GenSigningCert() {

}

// GenServerCert generates a hosting certificate for servers using TLS.
func GenServerCert() {

}

// GenClientCert generates a client certificate signed by the provided signing certificate.
// Generated certificate will have its extended key usage set to 'client authentication' and will be ready for use in TLS client authentication.
func GenClientCert(signCert *x509.Certificate, signKey *rsa.PrivateKey) (cert, key []byte, err error) {
	return nil, nil, nil
}

// genCert generates a PEM encoded X.509 certificate and private key pair (i.e. 'cert.pem', 'key.pem').
// This code is based on the sample from http://golang.org/src/crypto/tls/generate_cert.go (taken at Jan 30, 2015).
// If no private key is provided, the certificate is marked as self-signed CA.
// host = Comma-separated hostnames and IPs to generate a certificate for. i.e. "localhost,127.0.0.1"
// validFor = Validity period for the certificate. Defaults to time.Duration max (290 years).
// ca, caPriv = CA certificate/private key to sign the new certificate. If not given, the generated certificate will be a self-signed CA.
// keyLength = Key length for the new certificate. Defaults to 3248 bits RSA key.
// cn, org = Common name and organization fields of the certificate.
func genCert(host string, validFor time.Duration, ca *x509.Certificate, caPriv *rsa.PrivateKey, keyLength int, cn, org string) (pemBytes, privBytes []byte, err error) {
	isCA := ca == nil
	hosts := strings.Split(host, ",")
	if keyLength == 0 {
		keyLength = 3248
	}
	privKey, err := rsa.GenerateKey(rand.Reader, keyLength)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to generate certificate private RSA key: %v", err)
	}
	notBefore := time.Now()
	notAfter := notBefore.Add(290 * 365 * 24 * time.Hour) //290 years
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to generate the certificate serial number: %v", err)
	}
	if validFor != 0 {
		notAfter = notBefore.Add(validFor)
	}

	cert := x509.Certificate{
		IsCA:         isCA,
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			CommonName:   cn,
			Organization: []string{org},
		},
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		BasicConstraintsValid: true,
	}

	for _, h := range hosts {
		if ip := net.ParseIP(h); ip != nil {
			cert.IPAddresses = append(cert.IPAddresses, ip)
		} else {
			cert.DNSNames = append(cert.DNSNames, h)
		}
	}

	signerCert := &cert
	signerPriv := privKey
	if isCA {
		cert.KeyUsage |= x509.KeyUsageCertSign
		cert.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}
	} else {
		cert.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
		signerCert = ca
		signerPriv = caPriv
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, &cert, signerCert, &privKey.PublicKey, signerPriv)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create certificate: %v", err)
	}

	pemBytes = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	privBytes = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privKey)})
	return
}
<commit_msg>add per function docs<commit_after>// Package ca provides easy to use certificate authority related functions.
// This is a lightweight wrapper around "crypto/x509" package for
// creating CA certs, client certs, signing requests, and more.
//
// This package is mostly based on the example code provided at:
// http://golang.org/src/crypto/tls/generate_cert.go
package ca

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"fmt"
	"math/big"
	"net"
	"strings"
	"time"
)

// GenCA generates a self-signed CA certificate.
// Returns PEM encoded X.509 certificate and private key pair.
// Note: While writing the binary cert/key pair to file system, it is useful to use standard naming like: 'cert.pem', 'key.pem'.
func GenCA() (cert, key []byte, err error) {
	return nil, nil, nil
}

// GenSigningCert generates an intermediate signing certificate for signing server or client certificates.
// Returns PEM encoded X.509 certificate and private key pair.
func GenSigningCert() {

}

// GenServerCert generates a hosting certificate for servers using TLS.
// Returns PEM encoded X.509 certificate and private key pair.
func GenServerCert() {

}

// GenClientCert generates a client certificate signed by the provided signing certificate.
// Generated certificate will have its extended key usage set to 'client authentication' and will be ready for use in TLS client authentication.
// Returns PEM encoded X.509 certificate and private key pair.
func GenClientCert(signCert *x509.Certificate, signKey *rsa.PrivateKey) (cert, key []byte, err error) {
	return nil, nil, nil
}

// genCert generates a PEM encoded X.509 certificate and private key pair (i.e. 'cert.pem', 'key.pem').
// This code is based on the sample from http://golang.org/src/crypto/tls/generate_cert.go (taken at Jan 30, 2015).
// If no private key is provided, the certificate is marked as self-signed CA.
// host = Comma-separated hostnames and IPs to generate a certificate for. i.e. "localhost,127.0.0.1"
// validFor = Validity period for the certificate. Defaults to time.Duration max (290 years).
// ca, caPriv = CA certificate/private key to sign the new certificate. If not given, the generated certificate will be a self-signed CA.
// keyLength = Key length for the new certificate. Defaults to 3248 bits RSA key.
// cn, org = Common name and organization fields of the certificate.
func genCert(host string, validFor time.Duration, ca *x509.Certificate, caPriv *rsa.PrivateKey, keyLength int, cn, org string) (pemBytes, privBytes []byte, err error) {
	isCA := ca == nil
	hosts := strings.Split(host, ",")
	if keyLength == 0 {
		keyLength = 3248
	}
	privKey, err := rsa.GenerateKey(rand.Reader, keyLength)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to generate certificate private RSA key: %v", err)
	}
	notBefore := time.Now()
	notAfter := notBefore.Add(290 * 365 * 24 * time.Hour) //290 years
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to generate the certificate serial number: %v", err)
	}
	if validFor != 0 {
		notAfter = notBefore.Add(validFor)
	}

	cert := x509.Certificate{
		IsCA:         isCA,
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			CommonName:   cn,
			Organization: []string{org},
		},
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		BasicConstraintsValid: true,
	}

	for _, h := range hosts {
		if ip := net.ParseIP(h); ip != nil {
			cert.IPAddresses = append(cert.IPAddresses, ip)
		} else {
			cert.DNSNames = append(cert.DNSNames, h)
		}
	}

	signerCert := &cert
	signerPriv := privKey
	if isCA {
		cert.KeyUsage |= x509.KeyUsageCertSign
		cert.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}
	} else {
		cert.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
		signerCert = ca
		signerPriv = caPriv
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, &cert, signerCert, &privKey.PublicKey, signerPriv)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create certificate: %v", err)
	}

	pemBytes = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	privBytes = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privKey)})
	return
}
<|endoftext|>


<commit_before>package main

import (
  "net/http"
  "log"
  "io/ioutil"
  "html/template"
  "os"
  "os/user"
  "sort"
  "path"
  "flag"
  "strings"
)

var root string
var showHiddenFiles bool

type Entry struct {
  Name string
  FullName string
  IsDir bool
  IsParent bool
}

type ByIsDir []Entry

func (slice ByIsDir) Len() int {
  return len(slice)
}

func (slice ByIsDir) Less(i, j int) bool {
  if (slice[i].IsDir != slice[j].IsDir) {
    return slice[i].IsDir
  }

  return slice[i].Name < slice[j].Name
}

func (slice ByIsDir) Swap(i, j int) {
  slice[i], slice[j] = slice[j], slice[i]
}

func main() {
  currentUser, err := user.Current()
  if err != nil {
      log.Fatal(err)
  }

  flag.StringVar(&root, "root", currentUser.HomeDir, "Root folder. Default is hode dir.")
  flag.Parse()

  http.HandleFunc("/", indexHandler)
  log.Println("Server is started...")
  http.ListenAndServe(":8000", nil)
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
  currentPath := r.URL.Query().Get("path")
  if (currentPath == "") {
    currentPath = root
  }

  showHiddenFilesCookie, err := r.Cookie("showHiddenFiles")
  if err == nil {
    showHiddenFiles = showHiddenFilesCookie.Value == "1"
  } else {
    showHiddenFiles = false
  }

  stat, err := os.Stat(currentPath)

  if os.IsNotExist(err) {
    log.Println(err.Error())
    http.Error(w, http.StatusText(404), 404)
    return
  }

  if !stat.IsDir() {
    http.ServeFile(w, r, currentPath)
    return
  }

  t, err := template.ParseFiles("templates/fs.html")
  if err != nil {
    log.Println(err.Error())
    http.Error(w, http.StatusText(500), 500)
    return
  }

  t.Execute(w, getEntries(currentPath))
}

func getEntries(currentPath string) []Entry {
  entries := []Entry{}

  if (currentPath != root) {
    parent := Entry{
      Name: "..",
      FullName: path.Join(currentPath, ".."),
      IsDir: true,
      IsParent: true,
    }
    entries = append(entries, parent)
  }

  files, _ := ioutil.ReadDir(currentPath)
  for _, e := range files {
    if !showHiddenFiles && strings.HasPrefix(e.Name(), ".") {
      continue
    }

    entries = append(entries, Entry{
      Name: e.Name(),
      FullName: path.Join(currentPath, e.Name()),
      IsDir: e.IsDir(),
    })
  }

  sort.Sort(ByIsDir(entries))
  return entries
}
<commit_msg>Show relative path<commit_after>package main

import (
  "net/http"
  "log"
  "io/ioutil"
  "html/template"
  "os"
  "os/user"
  "sort"
  "path"
  "flag"
  "strings"
)

var root string
var showHiddenFiles bool

type Entry struct {
  Name string
  FullName string
  IsDir bool
  IsParent bool
}

type ByIsDir []Entry

func (slice ByIsDir) Len() int {
  return len(slice)
}

func (slice ByIsDir) Less(i, j int) bool {
  if (slice[i].IsDir != slice[j].IsDir) {
    return slice[i].IsDir
  }

  return slice[i].Name < slice[j].Name
}

func (slice ByIsDir) Swap(i, j int) {
  slice[i], slice[j] = slice[j], slice[i]
}

func main() {
  currentUser, err := user.Current()
  if err != nil {
      log.Fatal(err)
  }

  flag.StringVar(&root, "root", currentUser.HomeDir, "Root folder. Default is home dir.")
  flag.Parse()

  http.HandleFunc("/", indexHandler)
  log.Println("Server is started...")
  http.ListenAndServe(":8000", nil)
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
  currentPath := r.URL.Query().Get("path")
  fullCurrentPath := path.Join(root, currentPath)

  showHiddenFilesCookie, err := r.Cookie("showHiddenFiles")
  if err == nil {
    showHiddenFiles = showHiddenFilesCookie.Value == "1"
  } else {
    showHiddenFiles = false
  }

  stat, err := os.Stat(fullCurrentPath)

  if os.IsNotExist(err) {
    log.Println(err.Error())
    http.Error(w, http.StatusText(404), 404)
    return
  }

  if !stat.IsDir() {
    http.ServeFile(w, r, fullCurrentPath)
    return
  }

  t, err := template.ParseFiles("templates/fs.html")
  if err != nil {
    log.Println(err.Error())
    http.Error(w, http.StatusText(500), 500)
    return
  }

  t.Execute(w, getEntries(currentPath))
}

func getEntries(currentPath string) []Entry {
  entries := []Entry{}

  if (currentPath != "") {
    parent := Entry{
      Name: "..",
      FullName: path.Join(currentPath, ".."),
      IsDir: true,
      IsParent: true,
    }
    entries = append(entries, parent)
  }

  files, _ := ioutil.ReadDir(path.Join(root, currentPath))
  for _, e := range files {
    if !showHiddenFiles && strings.HasPrefix(e.Name(), ".") {
      continue
    }

    entries = append(entries, Entry{
      Name: e.Name(),
      FullName: path.Join(currentPath, e.Name()),
      IsDir: e.IsDir(),
    })
  }

  sort.Sort(ByIsDir(entries))
  return entries
}
<|endoftext|>


<commit_before>package fscache

import (
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"gopkg.in/djherbis/atime.v1"
	"gopkg.in/djherbis/stream.v1"
)

type FileSystemStater interface {
	// Stat takes a File.Name() and returns FileInfo interface
	Stat(name string) (FileInfo, error)
}

type FileInfo interface {
	os.FileInfo

	// AccessTimes returns the last time the file was read,
	// and the last time it was written to.
	// It will be used to check expiry of a file, and must be concurrent safe
	// with modifications to the FileSystem (writes, reads etc.)
	AccessTimes() (rt, wt time.Time, err error)
}

// FileSystem is used as the source for a Cache.
type FileSystem interface {
	// Stream FileSystem
	stream.FileSystem

	FileSystemStater

	// Reload should look through the FileSystem and call the supplied fn
	// with the key/filename pairs that are found.
	Reload(func(key, name string)) error

	// RemoveAll should empty the FileSystem of all files.
	RemoveAll() error
}

type stdFs struct {
	root string
	init func() error
}

type fileInfo struct {
	name string
	os.FileInfo
}

func (f *fileInfo) AccessTimes() (rt, wt time.Time, err error) {
	fi, err := os.Stat(f.name)
	if err != nil {
		return rt, wt, err
	}
	return atime.Get(fi), fi.ModTime(), nil
}

// NewFs returns a FileSystem rooted at directory dir.
// Dir is created with perms if it doesn't exist.
func NewFs(dir string, mode os.FileMode) (FileSystem, error) {
	fs := &stdFs{root: dir, init: func() error {
		return os.MkdirAll(dir, mode)
	}}
	return fs, fs.init()
}

func (fs *stdFs) Size(name string) (int64, error) {
	stat, err := os.Stat(name)
	if err == nil {
		return stat.Size(), nil
	}
	return 0, err
}

func (fs *stdFs) Reload(add func(key, name string)) error {
	files, err := ioutil.ReadDir(fs.root)
	if err != nil {
		return err
	}

	addfiles := make(map[string]struct {
		os.FileInfo
		key string
	})

	for _, f := range files {

		if strings.HasSuffix(f.Name(), ".key") {
			continue
		}

		key, err := fs.getKey(f.Name())
		if err != nil {
			fs.Remove(filepath.Join(fs.root, f.Name()))
			continue
		}
		fi, ok := addfiles[key]

		if !ok || fi.ModTime().Before(f.ModTime()) {
			if ok {
				fs.Remove(fi.Name())
			}
			addfiles[key] = struct {
				os.FileInfo
				key string
			}{
				FileInfo: f,
				key:      key,
			}
		} else {
			fs.Remove(f.Name())
		}

	}

	for _, f := range addfiles {
		path, err := filepath.Abs(filepath.Join(fs.root, f.Name()))
		if err != nil {
			return err
		}
		add(f.key, path)
	}

	return nil
}

func (fs *stdFs) Create(name string) (stream.File, error) {
	name, err := fs.makeName(name)
	if err != nil {
		return nil, err
	}
	return fs.create(name)
}

func (fs *stdFs) create(name string) (stream.File, error) {
	return os.OpenFile(filepath.Join(fs.root, name), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
}

func (fs *stdFs) Open(name string) (stream.File, error) {
	return os.Open(name)
}

func (fs *stdFs) Remove(name string) error {
	os.Remove(fmt.Sprintf("%s.key", name))
	return os.Remove(name)
}

func (fs *stdFs) RemoveAll() error {
	if err := os.RemoveAll(fs.root); err != nil {
		return err
	}
	return fs.init()
}

func (fs *stdFs) AccessTimes(name string) (rt, wt time.Time, err error) {
	fi, err := os.Stat(name)
	if err != nil {
		return rt, wt, err
	}
	return atime.Get(fi), fi.ModTime(), nil
}

func (fs *stdFs) Stat(name string) (FileInfo, error) {
	stat, err := os.Stat(name)
	if err != nil {
		return nil, err
	}

	return &fileInfo{name, stat}, nil
}

const (
	saltSize    = 8
	maxShort    = 20
	shortPrefix = "s"
	longPrefix  = "l"
)

func salt() string {
	buf := bytes.NewBufferString("")
	enc := base64.NewEncoder(base64.URLEncoding, buf)
	io.CopyN(enc, rand.Reader, saltSize)
	return buf.String()
}

func tob64(s string) string {
	buf := bytes.NewBufferString("")
	enc := base64.NewEncoder(base64.URLEncoding, buf)
	enc.Write([]byte(s))
	enc.Close()
	return buf.String()
}

func fromb64(s string) string {
	buf := bytes.NewBufferString(s)
	dec := base64.NewDecoder(base64.URLEncoding, buf)
	out := bytes.NewBufferString("")
	io.Copy(out, dec)
	return out.String()
}

func (fs *stdFs) makeName(key string) (string, error) {
	b64key := tob64(key)
	// short name
	if len(b64key) < maxShort {
		return fmt.Sprintf("%s%s%s", shortPrefix, salt(), b64key), nil
	}

	// long name
	hash := md5.Sum([]byte(key))
	name := fmt.Sprintf("%s%s%x", longPrefix, salt(), hash[:])
	f, err := fs.create(fmt.Sprintf("%s.key", name))
	if err != nil {
		return "", err
	}
	_, err = f.Write([]byte(key))
	f.Close()
	return name, err
}

func (fs *stdFs) getKey(name string) (string, error) {
	// short name
	if strings.HasPrefix(name, shortPrefix) {
		return fromb64(strings.TrimPrefix(name, shortPrefix)[saltSize:]), nil
	}

	// long name
	f, err := fs.Open(filepath.Join(fs.root, fmt.Sprintf("%s.key", name)))
	if err != nil {
		return "", err
	}
	defer f.Close()
	key, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}
	return string(key), nil
}
<commit_msg>`AccessTime` method of stdFs is simplified<commit_after>package fscache

import (
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"gopkg.in/djherbis/atime.v1"
	"gopkg.in/djherbis/stream.v1"
)

type FileSystemStater interface {
	// Stat takes a File.Name() and returns FileInfo interface
	Stat(name string) (FileInfo, error)
}

type FileInfo interface {
	os.FileInfo

	// AccessTimes returns the last time the file was read,
	// and the last time it was written to.
	// It will be used to check expiry of a file, and must be concurrent safe
	// with modifications to the FileSystem (writes, reads etc.)
	AccessTimes() (rt, wt time.Time, err error)
}

// FileSystem is used as the source for a Cache.
type FileSystem interface {
	// Stream FileSystem
	stream.FileSystem

	FileSystemStater

	// Reload should look through the FileSystem and call the supplied fn
	// with the key/filename pairs that are found.
	Reload(func(key, name string)) error

	// RemoveAll should empty the FileSystem of all files.
	RemoveAll() error
}

type stdFs struct {
	root string
	init func() error
}

type fileInfo struct {
	name string
	os.FileInfo
}

func (f *fileInfo) AccessTimes() (rt, wt time.Time, err error) {
	return atime.Get(f.FileInfo), f.FileInfo.ModTime(), nil
}

// NewFs returns a FileSystem rooted at directory dir.
// Dir is created with perms if it doesn't exist.
func NewFs(dir string, mode os.FileMode) (FileSystem, error) {
	fs := &stdFs{root: dir, init: func() error {
		return os.MkdirAll(dir, mode)
	}}
	return fs, fs.init()
}

func (fs *stdFs) Size(name string) (int64, error) {
	stat, err := os.Stat(name)
	if err == nil {
		return stat.Size(), nil
	}
	return 0, err
}

func (fs *stdFs) Reload(add func(key, name string)) error {
	files, err := ioutil.ReadDir(fs.root)
	if err != nil {
		return err
	}

	addfiles := make(map[string]struct {
		os.FileInfo
		key string
	})

	for _, f := range files {

		if strings.HasSuffix(f.Name(), ".key") {
			continue
		}

		key, err := fs.getKey(f.Name())
		if err != nil {
			fs.Remove(filepath.Join(fs.root, f.Name()))
			continue
		}
		fi, ok := addfiles[key]

		if !ok || fi.ModTime().Before(f.ModTime()) {
			if ok {
				fs.Remove(fi.Name())
			}
			addfiles[key] = struct {
				os.FileInfo
				key string
			}{
				FileInfo: f,
				key:      key,
			}
		} else {
			fs.Remove(f.Name())
		}

	}

	for _, f := range addfiles {
		path, err := filepath.Abs(filepath.Join(fs.root, f.Name()))
		if err != nil {
			return err
		}
		add(f.key, path)
	}

	return nil
}

func (fs *stdFs) Create(name string) (stream.File, error) {
	name, err := fs.makeName(name)
	if err != nil {
		return nil, err
	}
	return fs.create(name)
}

func (fs *stdFs) create(name string) (stream.File, error) {
	return os.OpenFile(filepath.Join(fs.root, name), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
}

func (fs *stdFs) Open(name string) (stream.File, error) {
	return os.Open(name)
}

func (fs *stdFs) Remove(name string) error {
	os.Remove(fmt.Sprintf("%s.key", name))
	return os.Remove(name)
}

func (fs *stdFs) RemoveAll() error {
	if err := os.RemoveAll(fs.root); err != nil {
		return err
	}
	return fs.init()
}

func (fs *stdFs) AccessTimes(name string) (rt, wt time.Time, err error) {
	fi, err := os.Stat(name)
	if err != nil {
		return rt, wt, err
	}
	return atime.Get(fi), fi.ModTime(), nil
}

func (fs *stdFs) Stat(name string) (FileInfo, error) {
	stat, err := os.Stat(name)
	if err != nil {
		return nil, err
	}

	return &fileInfo{name, stat}, nil
}

const (
	saltSize    = 8
	maxShort    = 20
	shortPrefix = "s"
	longPrefix  = "l"
)

func salt() string {
	buf := bytes.NewBufferString("")
	enc := base64.NewEncoder(base64.URLEncoding, buf)
	io.CopyN(enc, rand.Reader, saltSize)
	return buf.String()
}

func tob64(s string) string {
	buf := bytes.NewBufferString("")
	enc := base64.NewEncoder(base64.URLEncoding, buf)
	enc.Write([]byte(s))
	enc.Close()
	return buf.String()
}

func fromb64(s string) string {
	buf := bytes.NewBufferString(s)
	dec := base64.NewDecoder(base64.URLEncoding, buf)
	out := bytes.NewBufferString("")
	io.Copy(out, dec)
	return out.String()
}

func (fs *stdFs) makeName(key string) (string, error) {
	b64key := tob64(key)
	// short name
	if len(b64key) < maxShort {
		return fmt.Sprintf("%s%s%s", shortPrefix, salt(), b64key), nil
	}

	// long name
	hash := md5.Sum([]byte(key))
	name := fmt.Sprintf("%s%s%x", longPrefix, salt(), hash[:])
	f, err := fs.create(fmt.Sprintf("%s.key", name))
	if err != nil {
		return "", err
	}
	_, err = f.Write([]byte(key))
	f.Close()
	return name, err
}

func (fs *stdFs) getKey(name string) (string, error) {
	// short name
	if strings.HasPrefix(name, shortPrefix) {
		return fromb64(strings.TrimPrefix(name, shortPrefix)[saltSize:]), nil
	}

	// long name
	f, err := fs.Open(filepath.Join(fs.root, fmt.Sprintf("%s.key", name)))
	if err != nil {
		return "", err
	}
	defer f.Close()
	key, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}
	return string(key), nil
}
<|endoftext|>