<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
	hashic "github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	hraft "github.com/relab/raft/hashicorp"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "enable batching")
	serverMetrics     = flag.Bool("servermetrics", true, "enable server-side metrics")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 1024, "How many more times entries per message allowed during catch up")
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logger := logrus.New()

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0600,
	)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer()

	if *serverMetrics {
		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(fmt.Sprintf(":590%d", *id), nil))
		}()
	}

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes)
	case bhashicorp:
		runhashicorp(logger, lis, grpcServer, *id, ids, nodes)
	}

}

func runhashicorp(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	servers := make([]hashic.Server, len(nodes))
	for i, addr := range nodes {
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)
		addr = host + ":" + strconv.Itoa(p-100)

		suffrage := hashic.Voter

		if !contains(uint64(i+1), ids) {
			suffrage = hashic.Nonvoter
		}

		servers[i] = hashic.Server{
			Suffrage: suffrage,
			ID:       hashic.ServerID(addr),
			Address:  hashic.ServerAddress(addr),
		}
	}

	addr, err := net.ResolveTCPAddr("tcp", string(servers[id-1].Address))
	if err != nil {
		logger.Fatal(err)
	}
	trans, err := hashic.NewTCPTransport(string(servers[id-1].Address), addr, len(nodes)+1, 10*time.Second, os.Stderr)
	if err != nil {
		logger.Fatal(err)
	}

	path := fmt.Sprintf("hashicorp%.2d.bolt", id)
	overwrite := !*recover
	// Check if file already exists.
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// We don't need to overwrite a file that doesn't exist.
			overwrite = false
		} else {
			// If we are unable to verify the existence of the file,
			// there is probably a permission problem.
			logger.Fatal(err)
		}
	}
	if overwrite {
		if err := os.Remove(path); err != nil {
			logger.Fatal(err)
		}
	}
	logs, err := raftboltdb.NewBoltStore(path)
	if err != nil {
		logger.Fatal(err)
	}

	snaps := hashic.NewInmemSnapshotStore()

	cfg := &hashic.Config{
		LocalID:            servers[id-1].ID,
		ProtocolVersion:    hashic.ProtocolVersionMax,
		HeartbeatTimeout:   *electionTimeout,
		ElectionTimeout:    *electionTimeout,
		CommitTimeout:      *heartbeatTimeout,
		MaxAppendEntries:   int(*entriesPerMsg),
		ShutdownOnRemove:   true,
		TrailingLogs:       math.MaxUint64,
		SnapshotInterval:   120 * time.Hour,
		SnapshotThreshold:  math.MaxUint64,
		LeaderLeaseTimeout: *electionTimeout / 2,
	}

	node := hraft.NewRaft(logger, NewStore(), cfg, servers, trans, logs, logs, snaps, ids)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	logger.Fatal(grpcServer.Serve(lis))
}

func runetcd(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	peers := make([]etcdraft.Peer, len(ids))

	for i, nid := range ids {
		addr := nodes[i]
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	dir := fmt.Sprintf("etcdwal%.2d", id)

	if !wal.Exist(dir) {
		if err := os.Mkdir(dir, 0750); err != nil {
			logger.Fatalf("rkvd: cannot create dir for wal (%v)", err)
		}

		w, err := wal.Create(dir, nil)
		if err != nil {
			logger.Fatalf("rkvd: create wal error (%v)", err)
		}
		w.Close()
	}

	walsnap := walpb.Snapshot{}
	w, err := wal.Open(dir, walsnap)
	if err != nil {
		logger.Fatalf("rkvd: error loading wal (%v)", err)
	}

	_, st, ents, err := w.ReadAll()
	if err != nil {
		log.Fatalf("rkvd: failed to read WAL (%v)", err)
	}

	storage := etcdraft.NewMemoryStorage()
	storage.SetHardState(st)
	storage.Append(ents)

	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		w,
		&etcdraft.Config{
			ID:            id,
			ElectionTick:  int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick: 1,
			Storage:       storage,
			MaxSizePerMsg: *entriesPerMsg,
			// etcdserver says: Never overflow the rafthttp buffer,
			// which is 4096. We keep the same constant.
			MaxInflightMsgs: 4096 / 8,
			CheckQuorum:     true,
			PreVote:         true,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
		!contains(id, ids),
		nodes,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, 20000)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	})

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	logger.Fatal(node.Run(grpcServer))
}

func contains(x uint64, xs []uint64) bool {
	for _, y := range xs {
		if y == x {
			return true
		}
	}
	return false
}
<commit_msg>rkvd/main.go: Enable recovery for etcd<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
	hashic "github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	hraft "github.com/relab/raft/hashicorp"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "enable batching")
	serverMetrics     = flag.Bool("servermetrics", true, "enable server-side metrics")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 1024, "How many more times entries per message allowed during catch up")
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logger := logrus.New()

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0600,
	)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer()

	if *serverMetrics {
		go func() {
			http.Handle("/metrics", promhttp.Handler())
			logger.Fatal(http.ListenAndServe(fmt.Sprintf(":590%d", *id), nil))
		}()
	}

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes)
	case bhashicorp:
		runhashicorp(logger, lis, grpcServer, *id, ids, nodes)
	}

}

func runhashicorp(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	servers := make([]hashic.Server, len(nodes))
	for i, addr := range nodes {
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)
		addr = host + ":" + strconv.Itoa(p-100)

		suffrage := hashic.Voter

		if !contains(uint64(i+1), ids) {
			suffrage = hashic.Nonvoter
		}

		servers[i] = hashic.Server{
			Suffrage: suffrage,
			ID:       hashic.ServerID(addr),
			Address:  hashic.ServerAddress(addr),
		}
	}

	addr, err := net.ResolveTCPAddr("tcp", string(servers[id-1].Address))
	if err != nil {
		logger.Fatal(err)
	}
	trans, err := hashic.NewTCPTransport(string(servers[id-1].Address), addr, len(nodes)+1, 10*time.Second, os.Stderr)
	if err != nil {
		logger.Fatal(err)
	}

	path := fmt.Sprintf("hashicorp%.2d.bolt", id)
	overwrite := !*recover
	// Check if file already exists.
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// We don't need to overwrite a file that doesn't exist.
			overwrite = false
		} else {
			// If we are unable to verify the existence of the file,
			// there is probably a permission problem.
			logger.Fatal(err)
		}
	}
	if overwrite {
		if err := os.Remove(path); err != nil {
			logger.Fatal(err)
		}
	}
	logs, err := raftboltdb.NewBoltStore(path)
	if err != nil {
		logger.Fatal(err)
	}

	snaps := hashic.NewInmemSnapshotStore()

	cfg := &hashic.Config{
		LocalID:            servers[id-1].ID,
		ProtocolVersion:    hashic.ProtocolVersionMax,
		HeartbeatTimeout:   *electionTimeout,
		ElectionTimeout:    *electionTimeout,
		CommitTimeout:      *heartbeatTimeout,
		MaxAppendEntries:   int(*entriesPerMsg),
		ShutdownOnRemove:   true,
		TrailingLogs:       math.MaxUint64,
		SnapshotInterval:   120 * time.Hour,
		SnapshotThreshold:  math.MaxUint64,
		LeaderLeaseTimeout: *electionTimeout / 2,
	}

	node := hraft.NewRaft(logger, NewStore(), cfg, servers, trans, logs, logs, snaps, ids)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	logger.Fatal(grpcServer.Serve(lis))
}

func runetcd(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	peers := make([]etcdraft.Peer, len(ids))

	for i, nid := range ids {
		addr := nodes[i]
		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	dir := fmt.Sprintf("etcdwal%.2d", id)

	switch {
	case wal.Exist(dir) && !*recover:
		if err := os.RemoveAll(dir); err != nil {
			logger.Fatal(err)
		}
		fallthrough
	case !wal.Exist(dir):
		if err := os.Mkdir(dir, 0750); err != nil {
			logger.Fatalf("rkvd: cannot create dir for wal (%v)", err)
		}

		w, err := wal.Create(dir, nil)
		if err != nil {
			logger.Fatalf("rkvd: create wal error (%v)", err)
		}
		w.Close()
	}

	walsnap := walpb.Snapshot{}
	w, err := wal.Open(dir, walsnap)
	if err != nil {
		logger.Fatalf("rkvd: error loading wal (%v)", err)
	}

	_, st, ents, err := w.ReadAll()
	if err != nil {
		log.Fatalf("rkvd: failed to read WAL (%v)", err)
	}

	storage := etcdraft.NewMemoryStorage()
	storage.SetHardState(st)
	storage.Append(ents)

	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		w,
		&etcdraft.Config{
			ID:            id,
			ElectionTick:  int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick: 1,
			Storage:       storage,
			MaxSizePerMsg: *entriesPerMsg,
			// etcdserver says: Never overflow the rafthttp buffer,
			// which is 4096. We keep the same constant.
			MaxInflightMsgs: 4096 / 8,
			CheckQuorum:     true,
			PreVote:         true,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
		!contains(id, ids),
		nodes,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(
	logger logrus.FieldLogger,
	lis net.Listener, grpcServer *grpc.Server,
	id uint64, ids []uint64, nodes []string,
) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, 20000)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	})

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logger.Fatal(grpcServer.Serve(lis))
	}()

	logger.Fatal(node.Run(grpcServer))
}

func contains(x uint64, xs []uint64) bool {
	for _, y := range xs {
		if y == x {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains the pieces of the tool that use typechecking from the go/types package.

package main

import (
	"go/ast"
	"go/token"

	"code.google.com/p/go.tools/go/types"
)

func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) error {
	pkg.idents = make(map[*ast.Ident]types.Object)
	pkg.spans = make(map[types.Object]Span)
	pkg.types = make(map[ast.Expr]types.TypeAndValue)
	// By providing a Config with our own error function, it will continue
	// past the first error. There is no need for that function to do anything.
	config := types.Config{
		Error: func(error) {},
	}
	info := &types.Info{
		Types:   pkg.types,
		Objects: pkg.idents,
	}
	typesPkg, err := config.Check(pkg.path, fs, astFiles, info)
	pkg.typesPkg = typesPkg
	// update spans
	for id, obj := range pkg.idents {
		pkg.growSpan(id, obj)
	}
	return err
}

// isStruct reports whether the composite literal c is a struct.
// If it is not (probably a struct), it returns a printable form of the type.
func (pkg *Package) isStruct(c *ast.CompositeLit) (bool, string) {
	// Check that the CompositeLit's type is a slice or array (which needs no field keys), if possible.
	typ := pkg.types[c].Type
	// If it's a named type, pull out the underlying type. If it's not, the Underlying
	// method returns the type itself.
	actual := typ
	if actual != nil {
		actual = actual.Underlying()
	}
	if actual == nil {
		// No type information available. Assume true, so we do the check.
		return true, ""
	}
	switch actual.(type) {
	case *types.Struct:
		return true, typ.String()
	default:
		return false, ""
	}
}

var (
	stringerMethodType = types.New("func() string")
	errorType          = types.New("interface{ Error() string }").(*types.Interface)
	stringerType       = types.New("interface{ String() string }").(*types.Interface)
	// One day this might work. See issue 6259.
	// formatterType   = types.New("interface{Format(f fmt.State, c rune)}")
)

// matchArgType reports an error if printf verb t is not appropriate
// for operand arg.
//
// typ is used only for recursive calls; external callers must supply nil.
//
// (Recursion arises from the compound types {map,chan,slice} which
// may be printed with %d etc. if that is appropriate for their element
// types.)
func (f *File) matchArgType(t printfArgType, typ types.Type, arg ast.Expr) bool {
	return f.matchArgTypeInternal(t, typ, arg, make(map[types.Type]bool))
}

// matchArgTypeInternal is the internal version of matchArgType. It carries a map
// remembering what types are in progress so we don't recur when faced with recursive
// types or mutually recursive types.
func (f *File) matchArgTypeInternal(t printfArgType, typ types.Type, arg ast.Expr, inProgress map[types.Type]bool) bool {
	// %v, %T accept any argument type.
	if t == anyType {
		return true
	}
	if typ == nil {
		// external call
		typ = f.pkg.types[arg].Type
		if typ == nil {
			return true // probably a type check problem
		}
	}
	// If the type implements fmt.Formatter, we have nothing to check.
	// But (see issue 6259) that's not easy to verify, so instead we see
	// if its method set contains a Format function. We could do better,
	// even now, but we don't need to be 100% accurate. Wait for 6259 to
	// be fixed instead. TODO.
	if hasMethod(typ, "Format") {
		return true
	}
	// If we can use a string, might arg (dynamically) implement the Stringer or Error interface?
	if t&argString != 0 {
		if types.Implements(typ, errorType, false) || types.Implements(typ, stringerType, false) {
			return true
		}
	}

	typ = typ.Underlying()
	if inProgress[typ] {
		// We're already looking at this type. The call that started it will take care of it.
		return true
	}
	inProgress[typ] = true

	switch typ := typ.(type) {
	case *types.Signature:
		return t&argPointer != 0

	case *types.Map:
		// Recur: map[int]int matches %d.
		return t&argPointer != 0 ||
			(f.matchArgTypeInternal(t, typ.Key(), arg, inProgress) && f.matchArgTypeInternal(t, typ.Elem(), arg, inProgress))

	case *types.Chan:
		return t&argPointer != 0

	case *types.Array:
		// Same as slice.
		if types.Identical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 {
			return true // %s matches []byte
		}
		// Recur: []int matches %d.
		return t&argPointer != 0 || f.matchArgTypeInternal(t, typ.Elem().Underlying(), arg, inProgress)

	case *types.Slice:
		// Same as array.
		if types.Identical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 {
			return true // %s matches []byte
		}
		// Recur: []int matches %d. But watch out for
		//	type T []T
		// If the element is a pointer type (type T[]*T), it's handled fine by the Pointer case below.
		return t&argPointer != 0 || f.matchArgTypeInternal(t, typ.Elem(), arg, inProgress)

	case *types.Pointer:
		// Ugly, but dealing with an edge case: a known pointer to an invalid type,
		// probably something from a failed import.
		if typ.Elem().String() == "invalid type" {
			if *verbose {
				f.Warnf(arg.Pos(), "printf argument %v is pointer to invalid or unknown type", f.gofmt(arg))
			}
			return true // special case
		}
		// If it's actually a pointer with %p, it prints as one.
		if t == argPointer {
			return true
		}
		// If it's pointer to struct, that's equivalent in our analysis to whether we can print the struct.
		if str, ok := typ.Elem().Underlying().(*types.Struct); ok {
			return f.matchStructArgType(t, str, arg, inProgress)
		}
		// The rest can print with %p as pointers, or as integers with %x etc.
		return t&(argInt|argPointer) != 0

	case *types.Struct:
		return f.matchStructArgType(t, typ, arg, inProgress)

	case *types.Interface:
		// If the static type of the argument is empty interface, there's little we can do.
		// Example:
		//	func f(x interface{}) { fmt.Printf("%s", x) }
		// Whether x is valid for %s depends on the type of the argument to f. One day
		// we will be able to do better. For now, we assume that empty interface is OK
		// but non-empty interfaces, with Stringer and Error handled above, are errors.
		return typ.NumMethods() == 0

	case *types.Basic:
		switch typ.Kind() {
		case types.UntypedBool,
			types.Bool:
			return t&argBool != 0

		case types.UntypedInt,
			types.Int,
			types.Int8,
			types.Int16,
			types.Int32,
			types.Int64,
			types.Uint,
			types.Uint8,
			types.Uint16,
			types.Uint32,
			types.Uint64,
			types.Uintptr:
			return t&argInt != 0

		case types.UntypedFloat,
			types.Float32,
			types.Float64:
			return t&argFloat != 0

		case types.UntypedComplex,
			types.Complex64,
			types.Complex128:
			return t&argComplex != 0

		case types.UntypedString,
			types.String:
			return t&argString != 0

		case types.UnsafePointer:
			return t&(argPointer|argInt) != 0

		case types.UntypedRune:
			return t&(argInt|argRune) != 0

		case types.UntypedNil:
			return t&argPointer != 0 // TODO?

		case types.Invalid:
			if *verbose {
				f.Warnf(arg.Pos(), "printf argument %v has invalid or unknown type", f.gofmt(arg))
			}
			return true // Probably a type check problem.
		}
		panic("unreachable")
	}

	return false
}

// matchStructArgType reports whether all the elements of the struct match the expected
// type. For instance, with "%d" all the elements must be printable with the "%d" format.
func (f *File) matchStructArgType(t printfArgType, typ *types.Struct, arg ast.Expr, inProgress map[types.Type]bool) bool {
	for i := 0; i < typ.NumFields(); i++ {
		if !f.matchArgTypeInternal(t, typ.Field(i).Type(), arg, inProgress) {
			return false
		}
	}
	return true
}

// numArgsInSignature tells how many formal arguments the function type
// being called has.
func (f *File) numArgsInSignature(call *ast.CallExpr) int {
	// Check the type of the function or method declaration
	typ := f.pkg.types[call.Fun].Type
	if typ == nil {
		return 0
	}
	// The type must be a signature, but be sure for safety.
	sig, ok := typ.(*types.Signature)
	if !ok {
		return 0
	}
	return sig.Params().Len()
}

// isErrorMethodCall reports whether the call is of a method with signature
//	func Error() string
// where "string" is the universe's string type. We know the method is called "Error".
func (f *File) isErrorMethodCall(call *ast.CallExpr) bool {
	typ := f.pkg.types[call].Type
	if typ != nil {
		// We know it's called "Error", so just check the function signature.
		return types.Identical(f.pkg.types[call.Fun].Type, stringerMethodType)
	}
	// Without types, we can still check by hand.
	// Is it a selector expression? Otherwise it's a function call, not a method call.
	sel, ok := call.Fun.(*ast.SelectorExpr)
	if !ok {
		return false
	}
	// The package is type-checked, so if there are no arguments, we're done.
	if len(call.Args) > 0 {
		return false
	}
	// Check the type of the method declaration
	typ = f.pkg.types[sel].Type
	if typ == nil {
		return false
	}
	// The type must be a signature, but be sure for safety.
	sig, ok := typ.(*types.Signature)
	if !ok {
		return false
	}
	// There must be a receiver for it to be a method call. Otherwise it is
	// a function, not something that satisfies the error interface.
	if sig.Recv() == nil {
		return false
	}
	// There must be no arguments. Already verified by type checking, but be thorough.
	if sig.Params().Len() > 0 {
		return false
	}
	// Finally the real questions.
	// There must be one result.
	if sig.Results().Len() != 1 {
		return false
	}
	// It must have return type "string" from the universe.
	return sig.Results().At(0).Type() == types.Typ[types.String]
}

// hasMethod reports whether the type contains a method with the given name.
// It is part of the workaround for Formatters and should be deleted when
// that workaround is no longer necessary. TODO: delete when fixed.
func hasMethod(typ types.Type, name string) bool {
	set := typ.MethodSet()
	for i := 0; i < set.Len(); i++ {
		if set.At(i).Obj().Name() == name {
			return true
		}
	}
	return false
}
<commit_msg>go.tools/cmd/vet: use new Assertable predicate (fix build)<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains the pieces of the tool that use typechecking from the go/types package.

package main

import (
	"go/ast"
	"go/token"

	"code.google.com/p/go.tools/go/types"
)

func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) error {
	pkg.idents = make(map[*ast.Ident]types.Object)
	pkg.spans = make(map[types.Object]Span)
	pkg.types = make(map[ast.Expr]types.TypeAndValue)
	// By providing a Config with our own error function, it will continue
	// past the first error. There is no need for that function to do anything.
	config := types.Config{
		Error: func(error) {},
	}
	info := &types.Info{
		Types:   pkg.types,
		Objects: pkg.idents,
	}
	typesPkg, err := config.Check(pkg.path, fs, astFiles, info)
	pkg.typesPkg = typesPkg
	// update spans
	for id, obj := range pkg.idents {
		pkg.growSpan(id, obj)
	}
	return err
}

// isStruct reports whether the composite literal c is a struct.
// If it is not (probably a struct), it returns a printable form of the type.
func (pkg *Package) isStruct(c *ast.CompositeLit) (bool, string) {
	// Check that the CompositeLit's type is a slice or array (which needs no field keys), if possible.
	typ := pkg.types[c].Type
	// If it's a named type, pull out the underlying type. If it's not, the Underlying
	// method returns the type itself.
	actual := typ
	if actual != nil {
		actual = actual.Underlying()
	}
	if actual == nil {
		// No type information available. Assume true, so we do the check.
		return true, ""
	}
	switch actual.(type) {
	case *types.Struct:
		return true, typ.String()
	default:
		return false, ""
	}
}

var (
	stringerMethodType = types.New("func() string")
	errorType          = types.New("interface{ Error() string }").(*types.Interface)
	stringerType       = types.New("interface{ String() string }").(*types.Interface)
	// One day this might work. See issue 6259.
	// formatterType   = types.New("interface{Format(f fmt.State, c rune)}")
)

// matchArgType reports an error if printf verb t is not appropriate
// for operand arg.
//
// typ is used only for recursive calls; external callers must supply nil.
//
// (Recursion arises from the compound types {map,chan,slice} which
// may be printed with %d etc. if that is appropriate for their element
// types.)
func (f *File) matchArgType(t printfArgType, typ types.Type, arg ast.Expr) bool {
	return f.matchArgTypeInternal(t, typ, arg, make(map[types.Type]bool))
}

// matchArgTypeInternal is the internal version of matchArgType. It carries a map
// remembering what types are in progress so we don't recur when faced with recursive
// types or mutually recursive types.
func (f *File) matchArgTypeInternal(t printfArgType, typ types.Type, arg ast.Expr, inProgress map[types.Type]bool) bool {
	// %v, %T accept any argument type.
	if t == anyType {
		return true
	}
	if typ == nil {
		// external call
		typ = f.pkg.types[arg].Type
		if typ == nil {
			return true // probably a type check problem
		}
	}
	// If the type implements fmt.Formatter, we have nothing to check.
	// But (see issue 6259) that's not easy to verify, so instead we see
	// if its method set contains a Format function. We could do better,
	// even now, but we don't need to be 100% accurate. Wait for 6259 to
	// be fixed instead. TODO.
	if hasMethod(typ, "Format") {
		return true
	}
	// If we can use a string, might arg (dynamically) implement the Stringer or Error interface?
	if t&argString != 0 {
		if types.Assertable(errorType, typ) || types.Assertable(stringerType, typ) {
			return true
		}
	}

	typ = typ.Underlying()
	if inProgress[typ] {
		// We're already looking at this type. The call that started it will take care of it.
		return true
	}
	inProgress[typ] = true

	switch typ := typ.(type) {
	case *types.Signature:
		return t&argPointer != 0

	case *types.Map:
		// Recur: map[int]int matches %d.
		return t&argPointer != 0 ||
			(f.matchArgTypeInternal(t, typ.Key(), arg, inProgress) && f.matchArgTypeInternal(t, typ.Elem(), arg, inProgress))

	case *types.Chan:
		return t&argPointer != 0

	case *types.Array:
		// Same as slice.
		if types.Identical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 {
			return true // %s matches []byte
		}
		// Recur: []int matches %d.
		return t&argPointer != 0 || f.matchArgTypeInternal(t, typ.Elem().Underlying(), arg, inProgress)

	case *types.Slice:
		// Same as array.
		if types.Identical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 {
			return true // %s matches []byte
		}
		// Recur: []int matches %d. But watch out for
		//	type T []T
		// If the element is a pointer type (type T[]*T), it's handled fine by the Pointer case below.
		return t&argPointer != 0 || f.matchArgTypeInternal(t, typ.Elem(), arg, inProgress)

	case *types.Pointer:
		// Ugly, but dealing with an edge case: a known pointer to an invalid type,
		// probably something from a failed import.
		if typ.Elem().String() == "invalid type" {
			if *verbose {
				f.Warnf(arg.Pos(), "printf argument %v is pointer to invalid or unknown type", f.gofmt(arg))
			}
			return true // special case
		}
		// If it's actually a pointer with %p, it prints as one.
		if t == argPointer {
			return true
		}
		// If it's pointer to struct, that's equivalent in our analysis to whether we can print the struct.
		if str, ok := typ.Elem().Underlying().(*types.Struct); ok {
			return f.matchStructArgType(t, str, arg, inProgress)
		}
		// The rest can print with %p as pointers, or as integers with %x etc.
		return t&(argInt|argPointer) != 0

	case *types.Struct:
		return f.matchStructArgType(t, typ, arg, inProgress)

	case *types.Interface:
		// If the static type of the argument is empty interface, there's little we can do.
		// Example:
		//	func f(x interface{}) { fmt.Printf("%s", x) }
		// Whether x is valid for %s depends on the type of the argument to f. One day
		// we will be able to do better. For now, we assume that empty interface is OK
		// but non-empty interfaces, with Stringer and Error handled above, are errors.
		return typ.NumMethods() == 0

	case *types.Basic:
		switch typ.Kind() {
		case types.UntypedBool,
			types.Bool:
			return t&argBool != 0

		case types.UntypedInt,
			types.Int,
			types.Int8,
			types.Int16,
			types.Int32,
			types.Int64,
			types.Uint,
			types.Uint8,
			types.Uint16,
			types.Uint32,
			types.Uint64,
			types.Uintptr:
			return t&argInt != 0

		case types.UntypedFloat,
			types.Float32,
			types.Float64:
			return t&argFloat != 0

		case types.UntypedComplex,
			types.Complex64,
			types.Complex128:
			return t&argComplex != 0

		case types.UntypedString,
			types.String:
			return t&argString != 0

		case types.UnsafePointer:
			return t&(argPointer|argInt) != 0

		case types.UntypedRune:
			return t&(argInt|argRune) != 0

		case types.UntypedNil:
			return t&argPointer != 0 // TODO?

		case types.Invalid:
			if *verbose {
				f.Warnf(arg.Pos(), "printf argument %v has invalid or unknown type", f.gofmt(arg))
			}
			return true // Probably a type check problem.
		}
		panic("unreachable")
	}

	return false
}

// matchStructArgType reports whether all the elements of the struct match the expected
// type. For instance, with "%d" all the elements must be printable with the "%d" format.
func (f *File) matchStructArgType(t printfArgType, typ *types.Struct, arg ast.Expr, inProgress map[types.Type]bool) bool {
	for i := 0; i < typ.NumFields(); i++ {
		if !f.matchArgTypeInternal(t, typ.Field(i).Type(), arg, inProgress) {
			return false
		}
	}
	return true
}

// numArgsInSignature tells how many formal arguments the function type
// being called has.
func (f *File) numArgsInSignature(call *ast.CallExpr) int {
	// Check the type of the function or method declaration
	typ := f.pkg.types[call.Fun].Type
	if typ == nil {
		return 0
	}
	// The type must be a signature, but be sure for safety.
	sig, ok := typ.(*types.Signature)
	if !ok {
		return 0
	}
	return sig.Params().Len()
}

// isErrorMethodCall reports whether the call is of a method with signature
//	func Error() string
// where "string" is the universe's string type. We know the method is called "Error".
func (f *File) isErrorMethodCall(call *ast.CallExpr) bool {
	typ := f.pkg.types[call].Type
	if typ != nil {
		// We know it's called "Error", so just check the function signature.
		return types.Identical(f.pkg.types[call.Fun].Type, stringerMethodType)
	}
	// Without types, we can still check by hand.
	// Is it a selector expression? Otherwise it's a function call, not a method call.
	sel, ok := call.Fun.(*ast.SelectorExpr)
	if !ok {
		return false
	}
	// The package is type-checked, so if there are no arguments, we're done.
	if len(call.Args) > 0 {
		return false
	}
	// Check the type of the method declaration
	typ = f.pkg.types[sel].Type
	if typ == nil {
		return false
	}
	// The type must be a signature, but be sure for safety.
	sig, ok := typ.(*types.Signature)
	if !ok {
		return false
	}
	// There must be a receiver for it to be a method call. Otherwise it is
	// a function, not something that satisfies the error interface.
	if sig.Recv() == nil {
		return false
	}
	// There must be no arguments. Already verified by type checking, but be thorough.
	if sig.Params().Len() > 0 {
		return false
	}
	// Finally the real questions.
	// There must be one result.
	if sig.Results().Len() != 1 {
		return false
	}
	// It must have return type "string" from the universe.
	return sig.Results().At(0).Type() == types.Typ[types.String]
}

// hasMethod reports whether the type contains a method with the given name.
// It is part of the workaround for Formatters and should be deleted when
// that workaround is no longer necessary. TODO: delete when fixed.
func hasMethod(typ types.Type, name string) bool {
	set := typ.MethodSet()
	for i := 0; i < set.Len(); i++ {
		if set.At(i).Obj().Name() == name {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v2_3_experimental

import (
	"bytes"
	"errors"
	"reflect"

	"github.com/coreos/ignition/config/types"
	"github.com/coreos/ignition/config/v1"
	"github.com/coreos/ignition/config/v2_0"
	"github.com/coreos/ignition/config/v2_1"
	"github.com/coreos/ignition/config/v2_2"
	"github.com/coreos/ignition/config/validate"
	astjson "github.com/coreos/ignition/config/validate/astjson"
	"github.com/coreos/ignition/config/validate/report"

	json "github.com/ajeddeloh/go-json"
	"github.com/coreos/go-semver/semver"
	"go4.org/errorutil"
)

var (
	ErrCloudConfig           = errors.New("not a config (found coreos-cloudconfig)")
	ErrEmpty                 = errors.New("not a config (empty)")
	ErrScript                = errors.New("not a config (found coreos-cloudinit script)")
	ErrDeprecated            = errors.New("config format deprecated")
	ErrInvalid               = errors.New("config is not valid")
	ErrUnknownVersion        = errors.New("unsupported config version")
	ErrVersionIndeterminable = errors.New("unable to determine version")
)

// Parse parses the raw config into a types.Config struct and generates a report of any
// errors, warnings, info, and deprecations it encountered
func Parse(rawConfig []byte) (types.Config, report.Report, error) {
	if isEmpty(rawConfig) {
		return types.Config{}, report.Report{}, ErrEmpty
	} else if isCloudConfig(rawConfig) {
		return types.Config{}, report.Report{}, ErrCloudConfig
	} else if isScript(rawConfig) {
		return types.Config{}, report.Report{}, ErrScript
	}

	version, err := Version(rawConfig)
	if err != nil && err != ErrVersionIndeterminable {
		// If we can't determine the version, its probably invalid json and we want to fall through
		// to handle that in ParseFromLatest, since it will generate the highlight string and report
		// line and column.
		return types.Config{}, report.ReportFromError(err, report.EntryError), err
	}
	switch version {
	case semver.Version{Major: 1}:
		config, rpt, err := ParseFromV1(rawConfig)
		if err != nil {
			return types.Config{}, report.ReportFromError(err, report.EntryError), err
		}
		rpt.Merge(report.ReportFromError(ErrDeprecated, report.EntryDeprecated))

		return config, rpt, nil
	case types.MaxVersion:
		return ParseFromLatest(rawConfig)
	case semver.Version{Major: 2, Minor: 2}:
		return ParseFromV2_2(rawConfig)
	case semver.Version{Major: 2, Minor: 1}:
		return ParseFromV2_1(rawConfig)
	case semver.Version{Major: 2, Minor: 0}:
		return ParseFromV2_0(rawConfig)
	default:
		// It's not empty, it's not a cloud config, and it's not a script, but
		// we can't determine the version of it (or it's 0.0.0, hooray golang
		// zero values). We know it's not valid, but try to parse it anyway with
		// ParseFromLatest to generate any parse errors.
		config, rep, err := ParseFromLatest(rawConfig)
		if err != nil || rep.IsFatal() {
			return config, rep, err
		}
		// Somehow parsing succeeded without any errors or fatal reports. This
		// should be an impossible state, but since we're in this part of the
		// switch statement we know at the very least the version is invalid, so
		// return that.
		return types.Config{}, report.Report{}, ErrUnknownVersion
	}
}

func ParseFromLatest(rawConfig []byte) (types.Config, report.Report, error) {
	var err error
	var config types.Config

	// These errors are fatal and the config should not be further validated
	if err = json.Unmarshal(rawConfig, &config); err == nil {
		versionReport := config.Ignition.Validate()
		if versionReport.IsFatal() {
			return types.Config{}, versionReport, ErrInvalid
		}
	}

	// Handle json syntax and type errors first, since they are fatal but have offset info
	if serr, ok := err.(*json.SyntaxError); ok {
		line, col, highlight := errorutil.HighlightBytePosition(bytes.NewReader(rawConfig), serr.Offset)
		return types.Config{},
			report.Report{
				Entries: []report.Entry{{
					Kind:      report.EntryError,
					Message:   serr.Error(),
					Line:      line,
					Column:    col,
					Highlight: highlight,
				}},
			},
			ErrInvalid
	}

	if terr, ok := err.(*json.UnmarshalTypeError); ok {
		line, col, highlight := errorutil.HighlightBytePosition(bytes.NewReader(rawConfig), terr.Offset)
		return types.Config{},
			report.Report{
				Entries: []report.Entry{{
					Kind:      report.EntryError,
					Message:   terr.Error(),
					Line:      line,
					Column:    col,
					Highlight: highlight,
				}},
			},
			ErrInvalid
	}

	// Handle other fatal errors (i.e. invalid version)
	if err != nil {
		return types.Config{}, report.ReportFromError(err, report.EntryError), err
	}

	// Unmarshal again to a json.Node to get offset information for building a report
	var ast json.Node
	var r report.Report
	configValue := reflect.ValueOf(config)
	if err := json.Unmarshal(rawConfig, &ast); err != nil {
		r.Add(report.Entry{
			Kind:    report.EntryWarning,
			Message: "Ignition could not unmarshal your config for reporting line numbers. This should never happen. Please file a bug.",
		})
		r.Merge(validate.ValidateWithoutSource(configValue))
	} else {
		r.Merge(validate.Validate(configValue, astjson.FromJsonRoot(ast), bytes.NewReader(rawConfig), true))
	}

	if r.IsFatal() {
		return types.Config{}, r, ErrInvalid
	}

	return config, r, nil
}

func ParseFromV1(rawConfig []byte) (types.Config, report.Report, error) {
	config, err := v1.Parse(rawConfig)
	if err != nil {
		return types.Config{}, report.Report{}, err
	}

	convertedConfig := TranslateFromV1(config)

	rpt := validate.ValidateWithoutSource(reflect.ValueOf(convertedConfig))
	if rpt.IsFatal() {
		return types.Config{}, rpt, ErrInvalid
	}
	return convertedConfig, rpt, nil

}

func ParseFromV2_0(rawConfig []byte) (types.Config, report.Report, error) {
	cfg, report, err := v2_0.Parse(rawConfig)
	if err != nil {
		return types.Config{}, report, err
	}

	return TranslateFromV2_0(cfg), report, err
}

func ParseFromV2_1(rawConfig []byte) (types.Config, report.Report, error) {
	cfg, report, err := v2_1.Parse(rawConfig)
	if err != nil {
		return types.Config{}, report, err
	}

	return TranslateFromV2_1(cfg), report, err
}

func ParseFromV2_2(rawConfig []byte) (types.Config, report.Report, error) {
	cfg, report, err := v2_2.Parse(rawConfig)
	if err != nil {
		return types.Config{}, report, err
	}

	return TranslateFromV2_2(cfg), report, err
}

func Version(rawConfig []byte) (semver.Version, error) {
	var composite struct {
		Version  *int `json:"ignitionVersion"`
		Ignition struct {
			Version *string `json:"version"`
		} `json:"ignition"`
	}

	if json.Unmarshal(rawConfig, &composite) == nil {
		if composite.Ignition.Version != nil {
			v, err := types.Ignition{Version: *composite.Ignition.Version}.Semver()
			if err != nil {
				return semver.Version{}, err
			}
			return *v, nil
		} else if composite.Version != nil {
			return semver.Version{Major: int64(*composite.Version)}, nil
		}
	}

	return semver.Version{}, ErrVersionIndeterminable
}

func isEmpty(userdata []byte) bool {
	return len(userdata) == 0
}
<commit_msg>config/v2_3_experimental: only handle 2.3.0-experimental configs<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v2_3_experimental

import (
	"errors"

	"github.com/coreos/ignition/config/util"
	"github.com/coreos/ignition/config/v2_2"
	"github.com/coreos/ignition/config/v2_3_experimental/types"
	"github.com/coreos/ignition/config/validate/report"

	json "github.com/ajeddeloh/go-json"
	"github.com/coreos/go-semver/semver"
)

var (
	ErrCloudConfig           = errors.New("not a config (found coreos-cloudconfig)")
	ErrEmpty                 = errors.New("not a config (empty)")
	ErrScript                = errors.New("not a config (found coreos-cloudinit script)")
	ErrInvalid               = errors.New("config is not valid")
	ErrUnknownVersion        = errors.New("unsupported config version")
	ErrVersionIndeterminable = errors.New("unable to determine version")
)

// Parse parses the raw config into a types.Config struct and generates a report of any
// errors, warnings, info, and deprecations it encountered
func Parse(rawConfig []byte) (types.Config, report.Report, error) {
	if isEmpty(rawConfig) {
		return types.Config{}, report.Report{}, ErrEmpty
	} else if isCloudConfig(rawConfig) {
		return types.Config{}, report.Report{}, ErrCloudConfig
	} else if isScript(rawConfig) {
		return types.Config{}, report.Report{}, ErrScript
	}

	var err error
	var config types.Config

	err = json.Unmarshal(rawConfig, &config)

	version, semverErr := semver.NewVersion(config.Ignition.Version)

	if err != nil || semverErr != nil || version.LessThan(types.MaxVersion) {
		// We can fail unmarshaling if it's an older config. Attempt to parse
		// it as such.
		config, rpt, err := v2_2.Parse(rawConfig)
		if err != nil {
			return types.Config{}, rpt, err
		}
		return TranslateFromV2_2(config), rpt, err
	}

	if *version != types.MaxVersion {
		return types.Config{}, report.Report{}, ErrInvalid
	}

	rpt := util.ValidateConfig(rawConfig, config)
	if rpt.IsFatal() {
		return types.Config{}, rpt, ErrInvalid
	}

	return config, rpt, nil
}

func isEmpty(userdata []byte) bool {
	return len(userdata) == 0
}
<|endoftext|>


<commit_before>package fs

import (
	"encoding/json"

	"github.com/chzyer/madq/go/ptrace"
)

var Stat GStat

type GStat struct {
	Flusher struct {
		BlockCopy   ptrace.Size
		FlushTime   ptrace.Int
		FlushSize   ptrace.RatioSize
		ReadTime    ptrace.RatioTime
		WriteTime   ptrace.RatioTime
		HandleOp    ptrace.RatioTime
		FlushLoop   ptrace.RatioTime
		FlushBuffer ptrace.RatioTime
		Flush       ptrace.RatioTime
	}
	Inode struct {
		Cache struct {
			NextHit   ptrace.Ratio
			InoIdxHit ptrace.Ratio
		}

		ReadDisk    ptrace.Int
		PrevSeekCnt ptrace.Ratio
	}
	File struct {
		FlushSize     ptrace.RatioSize
		FlushDuration ptrace.RatioTime
		RegenBuffer   ptrace.Ratio
	}
	Cobuffer struct {
		Trytime            ptrace.Ratio
		NotifyFlushByWrite ptrace.Ratio
		GetDataLock        ptrace.RatioTime
		GetData            ptrace.RatioTime
		FlushDelay         ptrace.RatioTime
	}
}

func (c *GStat) String() string {
	ret, _ := json.MarshalIndent(c, "", "\t")
	return string(ret)
}
<commit_msg>[fs] add stat<commit_after>package fs

import (
	"encoding/json"

	"github.com/chzyer/madq/go/ptrace"
)

var Stat GStat

type GStat struct {
	Flusher struct {
		BlockCopy   ptrace.Size
		FlushTime   ptrace.Int
		FlushSize   ptrace.RatioSize
		ReadTime    ptrace.RatioTime
		WriteTime   ptrace.RatioTime
		HandleOp    ptrace.RatioTime
		FlushLoop   ptrace.RatioTime
		FlushBuffer ptrace.RatioTime
		Flush       ptrace.RatioTime
	}
	Inode struct {
		Cache struct {
			NextHit   ptrace.Ratio
			InoIdxHit ptrace.Ratio
		}

		ReadDisk    ptrace.Int
		PrevSeekCnt ptrace.Ratio
	}
	File struct {
		FlushSize     ptrace.RatioSize
		FlushDuration ptrace.RatioTime
		RegenBuffer   ptrace.Ratio
	}
	Cobuffer struct {
		Trytime            ptrace.Ratio
		NotifyFlushByWrite ptrace.Ratio
		GetDataLock        ptrace.RatioTime
		GetData            ptrace.RatioTime
		FlushDelay         ptrace.RatioTime
		FullTime           ptrace.RatioTime
		WriteTime          ptrace.RatioTime
	}
}

func (c *GStat) String() string {
	ret, _ := json.MarshalIndent(c, "", "\t")
	return string(ret)
}
<|endoftext|>


<commit_before>package gogtm_test

import (
	"fmt"

	"github.com/szydell/gogtm"
)

func ExampleSet() {
	err := gogtm.Set("^test", "aaa")
	if err != nil {
		fmt.Println("gogtm.Set failed: ", err.Error())
	}
}
<commit_msg>ExampleGet<commit_after>package gogtm_test

import (
	"fmt"

	"github.com/szydell/gogtm"
)

func ExampleSet() {
	err := gogtm.Set("^test", "aaa")
	if err != nil {
		fmt.Println("gogtm.Set failed: ", err.Error())
	}
}

func ExampleGet() {
	data, err := gogtm.Get("^test", "ababababababababa")
	if err != nil {
		fmt.Println("gogtm.Get failed: ", err.Error())
	}
	if data == "ababababababababa" {
		fmt.Println("Global ^test empty, default value returned")
	} else {
		fmt.Println("Value of global ^test is: " + data)
	}
}
<|endoftext|>


<commit_before>package golbin

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"
)

/*
Console is structure to contain Command Line, Input and Output.
*/
type Console struct {
	Command, StdInput, StdOutput string
	Process                      *os.Process
}

/*
startCommand runs passed command string and returns it.
*/
func startCommand(sysCommand string) *exec.Cmd {
	cmdTokens := strings.Split(sysCommand, " ")
	cmd := cmdTokens[0]
	if len(cmdTokens) == 1 {
		return exec.Command(cmd)
	}
	return exec.Command(cmd, strings.Join(cmdTokens[1:], " "))
}

/*
Run executes command from Console field with its input
and sets the output or error whatever gets prompted.
*/
func (konsole *Console) Run() (err error) {
	cmd := startCommand(konsole.Command)
	if konsole.StdInput != "" {
		cmd.Stdin = strings.NewReader(konsole.StdInput)
	}
	konsole.Process = cmd.Process

	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err == nil {
		konsole.StdOutput = out.String()
	} else {
		konsole.StdOutput = fmt.Sprintf("Stdout: %s\nError: %s", out.String(), err.Error())
	}
	return
}

/*
ExecOutput can be passed a command to quickly get its output or error.
*/
func ExecOutput(cmdline string) string {
	cmd := startCommand(cmdline)

	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err == nil {
		return out.String()
	}
	return fmt.Sprintf("Error: %s", err.Error())
}

/*
Run it for distro to manage opening with correct program
*/
func RunWithAssignedApp(runThis string) string {
	var openWith string
	if runtime.GOOS == "linux" {
		openWith = "xdg-open"
	} else if runtime.GOOS == "darwin" {
		openWith = "open"
	} else {
		return fmt.Sprintf("Error: %s is not supported as yet.", runtime.GOOS)
	}

	if !IsSystemCmd(openWith) {
		return fmt.Sprintf("Error: %s not found on this machine.", openWith)
	}

	cmdToRun := fmt.Sprintf("%s %s", openWith, runThis)
	return ExecOutput(cmdToRun)
}

func cmdStringToExecCmd(cmd string) *exec.Cmd {
	parts := strings.Fields(cmd)
	first := parts[0]
	rest := []string{}
	if len(parts) > 1 {
		rest = parts[1:len(parts)]
	}
	return exec.Command(first, rest...)
}

func Exec(cmd string) (out string, err error) {
	outBytes, err := cmdStringToExecCmd(cmd).Output()
	out = string(outBytes)
	return
}

func ExecWithEnv(cmd string, env map[string]string) (out string, err error) {
	cmdHandle := cmdStringToExecCmd(cmd)
	cmdHandle.Env = os.Environ()
	for envVar, envVal := range env {
		cmdHandle.Env = append(cmdHandle.Env, fmt.Sprintf("%s=%s", envVar, envVal))
	}
	outBytes, err := cmdHandle.Output()
	out = string(outBytes)
	return
}
<commit_msg>golbin Exec n ExecWithEnv to return err with trace of actual stderr<commit_after>package golbin

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"github.com/abhishekkr/gol/golerror"
)

/*
Console is structure to contain Command Line, Input and Output.
*/
type Console struct {
	Command, StdInput, StdOutput string
	Process                      *os.Process
}

/*
startCommand runs passed command string and returns it.
*/
func startCommand(sysCommand string) *exec.Cmd {
	cmdTokens := strings.Split(sysCommand, " ")
	cmd := cmdTokens[0]
	if len(cmdTokens) == 1 {
		return exec.Command(cmd)
	}
	return exec.Command(cmd, strings.Join(cmdTokens[1:], " "))
}

/*
Run executes command from Console field with its input
and sets the output or error whatever gets prompted.
*/
func (konsole *Console) Run() (err error) {
	cmd := startCommand(konsole.Command)
	if konsole.StdInput != "" {
		cmd.Stdin = strings.NewReader(konsole.StdInput)
	}
	konsole.Process = cmd.Process

	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err == nil {
		konsole.StdOutput = out.String()
	} else {
		konsole.StdOutput = fmt.Sprintf("Stdout: %s\nError: %s", out.String(), err.Error())
	}
	return
}

/*
ExecOutput can be passed a command to quickly get its output or error.
*/
func ExecOutput(cmdline string) string {
	cmd := startCommand(cmdline)

	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err == nil {
		return out.String()
	}
	return fmt.Sprintf("Error: %s", err.Error())
}

/*
Run it for distro to manage opening with correct program
*/
func RunWithAssignedApp(runThis string) string {
	var openWith string
	if runtime.GOOS == "linux" {
		openWith = "xdg-open"
	} else if runtime.GOOS == "darwin" {
		openWith = "open"
	} else {
		return fmt.Sprintf("Error: %s is not supported as yet.", runtime.GOOS)
	}

	if !IsSystemCmd(openWith) {
		return fmt.Sprintf("Error: %s not found on this machine.", openWith)
	}

	cmdToRun := fmt.Sprintf("%s %s", openWith, runThis)
	return ExecOutput(cmdToRun)
}

func cmdStringToExecCmd(cmd string) *exec.Cmd {
	parts := strings.Fields(cmd)
	first := parts[0]
	rest := []string{}
	if len(parts) > 1 {
		rest = parts[1:len(parts)]
	}
	return exec.Command(first, rest...)
}

func execCmd(cmdHandle *exec.Cmd) (out string, err error) {
	var stdout, stderr bytes.Buffer
	cmdHandle.Stdout = &stdout
	cmdHandle.Stderr = &stderr
	err = cmdHandle.Run()

	if err != nil {
		err = golerror.Error(127, fmt.Sprintf("'%s' :: '%s'", stderr.String(), err.Error()))
	}

	out = stdout.String()
	return
}

func Exec(cmd string) (out string, err error) {
	cmdHandle := cmdStringToExecCmd(cmd)
	return execCmd(cmdHandle)
}

func ExecWithEnv(cmd string, env map[string]string) (out string, err error) {
	cmdHandle := cmdStringToExecCmd(cmd)
	cmdHandle.Env = os.Environ()
	for envVar, envVal := range env {
		cmdHandle.Env = append(cmdHandle.Env, fmt.Sprintf("%s=%s", envVar, envVal))
	}

	return execCmd(cmdHandle)
}
<|endoftext|>


<commit_before>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
/*
*  File: gonew_main.go
*  Author: Bryan Matsuo [bmatsuo@soe.ucsc.edu]
*  Created: Sat Jul  2 19:17:53 PDT 2011
*  Usage: gonew [options]
 */
import (
	"os"
	//"io"
	"log"
	"fmt"
	"flag"
	//"bufio"
	"unicode"
	"strings"
	//"io/ioutil"
	"path/filepath"
	//"github.com/hoisie/mustache.go"
	//"github.com/kr/pretty.go"
)

func ArgumentError(msg string) {
	fmt.Fprintf(os.Stderr, "%s\n", msg)
	fs := setupFlags()
	fs.Usage()
}

var (
	usage          = `
gonew [options] cmd NAME
gonew [options] pkg NAME
gonew [options] lib NAME PKG
`
	printUsageHead = func() { fmt.Fprint(os.Stderr, usage, "\n") }
	userepo        = false
	norepo         = true
	VERBOSE        = false
	DEBUG          = false
	DEBUG_LEVEL    = -1
	name           string
	importlist     string
	ptype          string
	repo           string
	host           string
	user           string
	target         string
	license        string
	remote         string
	help           bool
)

func Debug(level int, msg string) {
	if DEBUG && DEBUG_LEVEL >= level {
		log.Print(msg)
	}
}

func Verbose(msg string) {
	if DEBUG || VERBOSE {
		fmt.Print(msg)
	}
}

func setupFlags() *flag.FlagSet {
	var fs = flag.NewFlagSet("gonew", flag.ExitOnError)
	fs.StringVar(&importlist,
		"import", "", "Packages to import in source .go files.")
	fs.StringVar(&repo,
		"repo", "", "Repository type (e.g. 'git').")
	fs.StringVar(&host,
		"host", "", "Repository host if any (e.g. 'github').")
	fs.StringVar(&user,
		"user", "", "Repo host username.")
	fs.StringVar(&remote,
		"remote", "", "Remote repository url to initialize and push to.")
	fs.StringVar(&target,
		"target", "", "Makefile target. Default based on NAME.")
	fs.StringVar(&license,
		"license", "", "Project license (e.g. 'newbsd').")
	fs.BoolVar(&AppConfig.Markdown,
		"markdown", false, "Markdown-enabled README.")
	fs.BoolVar(&(AppConfig.MakeTest),
		"test", AppConfig.MakeTest, "Produce test files with Go files.")
	fs.BoolVar(&(norepo), "norepo", false, "Don't start a repository.")
	fs.BoolVar(&VERBOSE,
		"v", false, "Verbose output.")
	fs.IntVar(&DEBUG_LEVEL,
		"debug", -1, "Change the amout of debug output.")
	fs.BoolVar(&help,
		"help", false, "Show this message.")
	fs.Usage = func() {
		printUsageHead()
		fs.PrintDefaults()
	}
	return fs
}

type Request int

const (
	NilRequest Request = iota
	ProjectRequest
	LibraryRequest
)

var RequestedFile File
var RequestedProject Project

func parseArgs() Request {
	var fs = setupFlags()
	fs.Parse(os.Args[1:])
	userepo = !norepo
	if DEBUG_LEVEL >= 0 {
		DEBUG = true
	}
	if help {
		fs.Usage()
		os.Exit(0)
	}
	var narg = fs.NArg()
	if narg < 1 {
		ArgumentError("Missing TYPE argument")
		os.Exit(1)
	}
	if narg < 2 {
		ArgumentError("Missing NAME argument")
		os.Exit(1)
	}
	ptype = fs.Arg(0)
	name = fs.Arg(1)

	if target == "" {
		target = DefaultTarget(name)
	}

	imports := make([]string, 0, 1+strings.Count(importlist, ":"))
	for _, s := range strings.Split(importlist, ":") {
		if pkg := strings.TrimFunc(s, unicode.IsSpace); pkg != "" {
			imports = append(imports, pkg)
		}
	}
	if len(imports) == 0 {
		imports = nil
	}

	var (
		file = File{
			Name: name, Pkg: "main",
			Repo: AppConfig.Repo, License: AppConfig.License,
			User: AppConfig.HostUser, Host: AppConfig.Host,
			ImportLibs: imports}
		project = Project{
			Name: name, Target: target,
			Type: NilProjectType, License: AppConfig.License, Remote: remote,
			Host: AppConfig.Host, User: AppConfig.HostUser,
			Repo:       AppConfig.Repo,
			ImportLibs: imports,
			Markdown:   AppConfig.Markdown}
		produceProject = true
		licObj         = NilLicenseType
		repoObj        = NilRepoType
		hostObj        = NilRepoHost
	)
	switch ptype {
	case "cmd":
		project.Type = CmdType
	case "pkg":
		project.Type = PkgType
	case "lib":
		produceProject = false
	default:
		fmt.Fprintf(os.Stderr, "Unknown TYPE %s\n", ptype)
		os.Exit(1)
	}
	switch license {
	case "":
		break
	case "newbsd":
		licObj = NewBSD
	default:
		fmt.Fprintf(os.Stderr, "Unknown TYPE %s\n", ptype)
		os.Exit(1)
	}
	if userepo {
		switch repo {
		case "":
			break
		case "git":
			repoObj = GitType
		case "mercurial":
			repoObj = HgType
		default:
			fmt.Fprintf(os.Stderr, "Unknown REPO %s\n", repo)
			os.Exit(1)
		}
		switch host {
		case "":
			break
		case "github":
			hostObj = GitHubHost
			repoObj = GitType
		/*
		   case "googlecode":
		       hostObj = GoogleCodeType
		       repoObj = HgType
		*/
		default:
			fmt.Fprintf(os.Stderr, "Unknown HOST %s\n", host)
			os.Exit(1)
		}
	}
	if produceProject {
		// TODO check target for improper characters.
		if user != "" {
			project.User = user
		}
		if licObj != NilLicenseType {
			project.License = licObj
		}
		if hostObj != NilRepoHost {
			project.Host = hostObj
		}
		if userepo && repoObj != NilRepoType {
			project.Repo = repoObj
		}
		RequestedProject = project
		return ProjectRequest
	} else {
		if narg < 3 {
			ArgumentError("Missing PKG argument")
			os.Exit(1)
		}
		file.Pkg = fs.Arg(2)
		if user != "" {
			file.User = user
		}
		if licObj != NilLicenseType {
			file.License = licObj
		}
		if hostObj != NilRepoHost {
			file.Host = hostObj
		}
		if userepo && repoObj != NilRepoType {
			file.Repo = repoObj
		}
		RequestedFile = file
		return LibraryRequest
	}
	return NilRequest
}

func FindTemplates() (TemplateMultiSet, error) {
	troots := make([]string, 0, 2)
	if alt := AppConfig.AltRoot; alt != "" {
		troots = append(troots, alt)
	}
	troots = append(troots, filepath.Join(GetTemplateRoot()...))
	return MakeTemplateMultiSet(DefaultFuncMap(), troots...)
}

func main() {
	if err := TouchConfig(); err != nil {
		fmt.Fprint(os.Stderr, err.Error(), "\n")
		os.Exit(1)
	}
	Verbose("Parsing config file.\n")
	ReadConfig()

	ms, err := FindTemplates()
	if err != nil {
		panic(err)
	}

	switch request := parseArgs(); request {
	case ProjectRequest:
		if DEBUG {
			fmt.Printf("Project requested %v\n", RequestedProject)
		} else if VERBOSE {
			fmt.Printf("Generating project %s\n", RequestedProject.Name)
		}
		if err := RequestedProject.Create(ms); err != nil {
			fmt.Fprint(os.Stderr, err.Error(), "\n")
			os.Exit(1)
		}
	case LibraryRequest:
		if DEBUG {
			fmt.Printf("Library requested %v\n", RequestedFile)
		} else if VERBOSE {
			fmt.Printf("Generating library %s (package %s)\n",
				RequestedFile.Name+".go", RequestedFile.Pkg)
		}
		if err := RequestedFile.Create(ms); err != nil {
			fmt.Fprint(os.Stderr, err.Error(), "\n")
			os.Exit(1)
		}
		if err := RequestedFile.TestFile().Create(ms); err != nil {
			fmt.Fprint(os.Stderr, err.Error(), "\n")
			os.Exit(1)
		}
	}
}
<commit_msg>Start work on a FindGonew function to find the source directory.<commit_after>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main

/*
*  File: gonew_main.go
*  Author: Bryan Matsuo [bmatsuo@soe.ucsc.edu]
*  Created: Sat Jul  2 19:17:53 PDT 2011
*  Usage: gonew [options]
 */
import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"unicode"
)

func ArgumentError(msg string) {
	fmt.Fprintf(os.Stderr, "%s\n", msg)
	fs := setupFlags()
	fs.Usage()
}

var (
	usage          = `
gonew [options] cmd NAME
gonew [options] pkg NAME
gonew [options] lib NAME PKG
`
	printUsageHead = func() { fmt.Fprint(os.Stderr, usage, "\n") }
	userepo        = false
	norepo         = true
	VERBOSE        = false
	DEBUG          = false
	DEBUG_LEVEL    = -1
	name           string
	importlist     string
	ptype          string
	repo           string
	host           string
	user           string
	target         string
	license        string
	remote         string
	help           bool
)

func Debug(level int, msg string) {
	if DEBUG && DEBUG_LEVEL >= level {
		log.Print(msg)
	}
}

func Verbose(msg string) {
	if DEBUG || VERBOSE {
		fmt.Print(msg)
	}
}

func setupFlags() *flag.FlagSet {
	var fs = flag.NewFlagSet("gonew", flag.ExitOnError)
	fs.StringVar(&importlist,
		"import", "", "Packages to import in source .go files.")
	fs.StringVar(&repo,
		"repo", "", "Repository type (e.g. 'git').")
	fs.StringVar(&host,
		"host", "", "Repository host if any (e.g. 'github').")
	fs.StringVar(&user,
		"user", "", "Repo host username.")
	fs.StringVar(&remote,
		"remote", "", "Remote repository url to initialize and push to.")
	fs.StringVar(&target,
		"target", "", "Makefile target. Default based on NAME.")
	fs.StringVar(&license,
		"license", "", "Project license (e.g. 'newbsd').")
	fs.BoolVar(&AppConfig.Markdown,
		"markdown", false, "Markdown-enabled README.")
	fs.BoolVar(&(AppConfig.MakeTest),
		"test", AppConfig.MakeTest, "Produce test files with Go files.")
	fs.BoolVar(&(norepo), "norepo", false, "Don't start a repository.")
	fs.BoolVar(&VERBOSE,
		"v", false, "Verbose output.")
	fs.IntVar(&DEBUG_LEVEL,
		"debug", -1, "Change the amout of debug output.")
	fs.BoolVar(&help,
		"help", false, "Show this message.")
	fs.Usage = func() {
		printUsageHead()
		fs.PrintDefaults()
	}
	return fs
}

// Returns a path to the directory containing Gonew's source (and the templates/ directory).
// This function first searches locates the 'gonew' executable in PATH. Then locates the directory
// it was built from (either GOROOT or a subdirectory of GOPATH).
func FindGonew() (dir string, err error) {
	var bin string
	if bin, err = exec.LookPath("gonew"); err != nil {
		return
	}
	var bindir string
	if bindir, err = filepath.Abs(filepath.Dir(bin)); err != nil {
		return
	}
	var gobin string
	var gopath string
	for _, env := range os.Environ() {
		if strings.HasPrefix(env, "GOBIN=") {
			if gobin, err = filepath.Abs(env[6:]); err != nil {
				return
			}
		}
		if strings.HasPrefix(env, "GOPATH=") {
			if gopath, err = filepath.Abs(env[7:]); err != nil {
				return
			}
		}
		if len(gobin) > 0 && len(gopath) > 0 {
			break
		}
	}
	if bindir == gobin {
		rootdir := filepath.Join(runtime.GOROOT(), "github.com", "bmtasuo", "gonew")
		var stat os.FileInfo
		switch stat, err = os.Stat(rootdir); {
		case err != nil:
			if err != os.ENOENT {
				return
			}
		case stat != nil:
			dir = rootdir
			return
		default:
			panic("unreachable")
		}
	}
	/*
		var stat os.FileInfo
		if !strings.HasSuffix(bindir, "bin") {
			err = fmt.Errorf("%s not under a GOPATH", bindir)
		} else if stat, err = filepath.Join(filepath.Dir(bindir), "src"); err != nil {
			err = fmt.Errorf("%s not under a GOPATH: %s", bindir)
		} else if !stat.IsDir() {
			err = fmt.Errorf("%s not under a GOPATH", bindir)
		} else if stat, err = filepath.Join(filepath.Join(filepath.Dir(bindir), "src", "github.com/bmatsuo/gonew")); err != nil {
			if err == os.ENOENT {
				if stat, err = filepath.Join(filepath.Join(filepath.Dir(bindir), "src", "gonew")); err != nil {
					bin = filepath.Join(filepath.Join(filepath.Dir(bindir), "src", "gonew"))
				}
			}
		} else {
			bin = filepath.Join(filepath.Join(filepath.Dir(bindir), "src", "github.com/bmatsuo/gonew"))
		}
	*/
	if err != nil {
		return
	}
	return
}

type Request int

const (
	NilRequest Request = iota
	ProjectRequest
	LibraryRequest
)

var RequestedFile File
var RequestedProject Project

func parseArgs() Request {
	var fs = setupFlags()
	fs.Parse(os.Args[1:])
	userepo = !norepo
	if DEBUG_LEVEL >= 0 {
		DEBUG = true
	}
	if help {
		fs.Usage()
		os.Exit(0)
	}
	var narg = fs.NArg()
	if narg < 1 {
		ArgumentError("Missing TYPE argument")
		os.Exit(1)
	}
	if narg < 2 {
		ArgumentError("Missing NAME argument")
		os.Exit(1)
	}
	ptype = fs.Arg(0)
	name = fs.Arg(1)

	if target == "" {
		target = DefaultTarget(name)
	}

	imports := make([]string, 0, 1+strings.Count(importlist, ":"))
	for _, s := range strings.Split(importlist, ":") {
		if pkg := strings.TrimFunc(s, unicode.IsSpace); pkg != "" {
			imports = append(imports, pkg)
		}
	}
	if len(imports) == 0 {
		imports = nil
	}

	var (
		file = File{
			Name: name, Pkg: "main",
			Repo: AppConfig.Repo, License: AppConfig.License,
			User: AppConfig.HostUser, Host: AppConfig.Host,
			ImportLibs: imports}
		project = Project{
			Name: name, Target: target,
			Type: NilProjectType, License: AppConfig.License, Remote: remote,
			Host: AppConfig.Host, User: AppConfig.HostUser,
			Repo:       AppConfig.Repo,
			ImportLibs: imports,
			Markdown:   AppConfig.Markdown}
		produceProject = true
		licObj         = NilLicenseType
		repoObj        = NilRepoType
		hostObj        = NilRepoHost
	)
	switch ptype {
	case "cmd":
		project.Type = CmdType
	case "pkg":
		project.Type = PkgType
	case "lib":
		produceProject = false
	default:
		fmt.Fprintf(os.Stderr, "Unknown TYPE %s\n", ptype)
		os.Exit(1)
	}
	switch license {
	case "":
		break
	case "newbsd":
		licObj = NewBSD
	default:
		fmt.Fprintf(os.Stderr, "Unknown TYPE %s\n", ptype)
		os.Exit(1)
	}
	if userepo {
		switch repo {
		case "":
			break
		case "git":
			repoObj = GitType
		case "mercurial":
			repoObj = HgType
		default:
			fmt.Fprintf(os.Stderr, "Unknown REPO %s\n", repo)
			os.Exit(1)
		}
		switch host {
		case "":
			break
		case "github":
			hostObj = GitHubHost
			repoObj = GitType
		/*
		   case "googlecode":
		       hostObj = GoogleCodeType
		       repoObj = HgType
		*/
		default:
			fmt.Fprintf(os.Stderr, "Unknown HOST %s\n", host)
			os.Exit(1)
		}
	}
	if produceProject {
		// TODO check target for improper characters.
		if user != "" {
			project.User = user
		}
		if licObj != NilLicenseType {
			project.License = licObj
		}
		if hostObj != NilRepoHost {
			project.Host = hostObj
		}
		if userepo && repoObj != NilRepoType {
			project.Repo = repoObj
		}
		RequestedProject = project
		return ProjectRequest
	} else {
		if narg < 3 {
			ArgumentError("Missing PKG argument")
			os.Exit(1)
		}
		file.Pkg = fs.Arg(2)
		if user != "" {
			file.User = user
		}
		if licObj != NilLicenseType {
			file.License = licObj
		}
		if hostObj != NilRepoHost {
			file.Host = hostObj
		}
		if userepo && repoObj != NilRepoType {
			file.Repo = repoObj
		}
		RequestedFile = file
		return LibraryRequest
	}
	return NilRequest
}

func FindTemplates() (TemplateMultiSet, error) {
	troots := make([]string, 0, 2)
	if alt := AppConfig.AltRoot; alt != "" {
		troots = append(troots, alt)
	}
	troots = append(troots, filepath.Join(GetTemplateRoot()...))
	return MakeTemplateMultiSet(DefaultFuncMap(), troots...)
}

func main() {
	if err := TouchConfig(); err != nil {
		fmt.Fprint(os.Stderr, err.Error(), "\n")
		os.Exit(1)
	}
	Verbose("Parsing config file.\n")
	ReadConfig()

	ms, err := FindTemplates()
	if err != nil {
		panic(err)
	}

	switch request := parseArgs(); request {
	case ProjectRequest:
		if DEBUG {
			fmt.Printf("Project requested %v\n", RequestedProject)
		} else if VERBOSE {
			fmt.Printf("Generating project %s\n", RequestedProject.Name)
		}
		if err := RequestedProject.Create(ms); err != nil {
			fmt.Fprint(os.Stderr, err.Error(), "\n")
			os.Exit(1)
		}
	case LibraryRequest:
		if DEBUG {
			fmt.Printf("Library requested %v\n", RequestedFile)
		} else if VERBOSE {
			fmt.Printf("Generating library %s (package %s)\n",
				RequestedFile.Name+".go", RequestedFile.Pkg)
		}
		if err := RequestedFile.Create(ms); err != nil {
			fmt.Fprint(os.Stderr, err.Error(), "\n")
			os.Exit(1)
		}
		if err := RequestedFile.TestFile().Create(ms); err != nil {
			fmt.Fprint(os.Stderr, err.Error(), "\n")
			os.Exit(1)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"sync"
	"time"

	"golang.org/x/net/context"

	storage "google.golang.org/api/storage/v1"

	"github.com/marcacohen/gcslock"
	logrus "github.com/sirupsen/logrus"
)

type JobCheckByGcslock struct {
	Bucket  string
	DirPath string
	Timeout time.Duration
	Storage Storage

	working bool
	mux     sync.Mutex
}

func (jc *JobCheckByGcslock) Check(job_id string, _ack func() error, f func() error) error {
	object := jc.DirPath + "/" + job_id + ".gcslock"
	url := fmt.Sprintf("gs://%s/%s", jc.Bucket, object)

	logger := log.WithFields(logrus.Fields{"lock": url})
	logger.Infoln("JobCheckByGcslock Start")

	ok, err := jc.DeleteIfTimedout(object)
	if err != nil {
		log.Errorf("Failed to DeleteIfTimedout %s because of %v\n", url, err)
		return err
	}
	if !ok {
		log.Warningf("Quit running job because %s already exists\n", url)
		return nil
	}

	m, err := gcslock.New(nil, jc.Bucket, object)
	if err != nil {
		log.Errorf("Failed to gcslock.New because of %v\n", err)
		return err
	}

	if err := jc.Lock(m); err != nil {
		return err
	}
	defer jc.Unlock(m)

	go jc.StartTouching(object, time.Duration(int64(jc.Timeout)/10))

	logger.Infoln("JobCheckByGcslock passed")
	defer logger.Infoln("JobCheckByGcslock done")

	err = f()

	jc.mux.Lock()
	defer jc.mux.Unlock()

	jc.working = false

	return err
}

func (jc *JobCheckByGcslock) DeleteIfTimedout(object string) (bool, error) {
	prefix := "JobCheckByGcslock.DeleteIfTimedout"
	logger := log.WithFields(logrus.Fields{"lock": fmt.Sprintf("gs://%s/%s", jc.Bucket, object)})
	logger.Debugf("%s Start\n", prefix)
	defer logger.Debugf("%s Done\n", prefix)

	f, err := jc.Storage.Get(jc.Bucket, object)
	if err != nil {
		return false, err
	}
	// Ok unless the file exists
	if f == nil {
		return true, nil
	}

	ut, err := time.Parse(time.RFC3339, f.Updated)
	if err != nil {
		logger.Errorf("%s error parsing file update time %q of %v because of %v\n", prefix, f.Updated, f, err)
		return false, err
	}

	deadline := ut.Add(jc.Timeout)
	if deadline.After(time.Now()) {
		// deadline hasn't come yet
		logger.Warningf("%s deadline hasn't come yet. It seems another process is working.\n", prefix)
		return false, nil
	}

	logger.Debugf("%s delete exceeded lock file starting.\n", prefix)
	err = jc.Storage.Delete(jc.Bucket, object)
	if err != nil {
		logger.Errorf("%s delete exceeded lock file error.\n", prefix)
		return false, err
	}
	logger.Infof("%s delete exceeded lock file successfully.\n", prefix)

	return true, nil
}

func (jc *JobCheckByGcslock) Lock(m gcslock.ContextLocker) error {
	log.Debugln("JobCheckByGcslock.Lock start")

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	// Wait up to 10 seconds to acquire a lock.
	if err := m.ContextLock(ctx); err != nil {
		log.Errorf("Failed to ContextLock because of %v\n", err)
		return err
	}

	log.Infoln("JobCheckByGcslock.Lock done")
	return nil
}

func (jc *JobCheckByGcslock) Unlock(m gcslock.ContextLocker) error {
	log.Debugln("JobCheckByGcslock.Unlock start")

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	if err := m.ContextUnlock(ctx); err != nil {
		log.Errorf("Failed to ContextUnlock because of %v\n", err)
		return err
	}

	log.Infoln("JobCheckByGcslock.Unlock done")
	return nil
}

func (jc *JobCheckByGcslock) StartTouching(object string, interval time.Duration) error {
	logger := log.WithFields(logrus.Fields{"lock": fmt.Sprintf("gs://%s/%s", jc.Bucket, object), "interval": interval})
	logger.Infoln("JobCheckByGcslock.StartTouching Start")
	defer logger.Infoln("JobCheckByGcslock.StartTouching Finished")

	jc.working = true
	for {
		nextLimit := time.Now().Add(interval)
		err := jc.WaitAndTouch(object, nextLimit)
		if err != nil {
			log.WithFields(logrus.Fields{"error": err}).Errorln("Error in StartTouching")
			return err
		}
		if !jc.working {
			return nil
		}
	}
	// return nil
}

func (jc *JobCheckByGcslock) WaitAndTouch(object string, nextLimit time.Time) error {
	ticker := time.NewTicker(100 * time.Millisecond)
	for now := range ticker.C {
		if !jc.working {
			ticker.Stop()
			return nil
		}
		if now.After(nextLimit) {
			ticker.Stop()
			break
		}
	}

	jc.mux.Lock()
	defer jc.mux.Unlock()

	logger := log.WithFields(logrus.Fields{"bucket": jc.Bucket, "object": object})
	logger.Debugln("WaitAndTouch")

	if !jc.working {
		logger.Infoln("WaitAndTouch working is done")
		return nil
	}

	metadata := map[string]string{
		"JobCheckByGcslock": time.Now().Format(time.RFC3339),
	}
	msg := &storage.Object{Metadata: metadata}

	logger = logger.WithFields(logrus.Fields{"metadata": metadata})
	logger.Debugln("WaitAndTouch Update lock file starting")
	_, err := jc.Storage.Update(jc.Bucket, object, msg)
	if err != nil {
		logger.Errorf("WaitAndTouch Update lock file error because of %v\n", err)
		return err
	}
	logger.Debugln("WaitAndTouch Update lock file success")
	return nil
}
<commit_msg>:+1: Improve logging messages<commit_after>package main

import (
	"fmt"
	"sync"
	"time"

	"golang.org/x/net/context"

	storage "google.golang.org/api/storage/v1"

	"github.com/marcacohen/gcslock"
	logrus "github.com/sirupsen/logrus"
)

type JobCheckByGcslock struct {
	Bucket  string
	DirPath string
	Timeout time.Duration
	Storage Storage

	working bool
	mux     sync.Mutex
}

func (jc *JobCheckByGcslock) Check(job_id string, _ack func() error, f func() error) error {
	object := jc.DirPath + "/" + job_id + ".gcslock"
	url := fmt.Sprintf("gs://%s/%s", jc.Bucket, object)

	logger := log.WithFields(logrus.Fields{"lock": url})
	logger.Infoln("JobCheckByGcslock Start")
	defer logger.Infoln("JobCheckByGcslock done")

	ok, err := jc.DeleteIfTimedout(object)
	if err != nil {
		log.Errorf("Failed to DeleteIfTimedout %s because of %v\n", url, err)
		return err
	}
	if !ok {
		log.Warningf("Quit running job because %s already exists\n", url)
		return nil
	}

	m, err := gcslock.New(nil, jc.Bucket, object)
	if err != nil {
		log.Errorf("Failed to gcslock.New because of %v\n", err)
		return err
	}

	if err := jc.Lock(m); err != nil {
		return err
	}
	defer jc.Unlock(m)

	go jc.StartTouching(object, time.Duration(int64(jc.Timeout)/10))

	logger.Debugln("JobCheckByGcslock handler starting")
	defer logger.Debugln("JobCheckByGcslock handler done")

	err = f()

	jc.mux.Lock()
	defer jc.mux.Unlock()

	jc.working = false

	return err
}

func (jc *JobCheckByGcslock) DeleteIfTimedout(object string) (bool, error) {
	prefix := "JobCheckByGcslock.DeleteIfTimedout"
	logger := log.WithFields(logrus.Fields{"lock": fmt.Sprintf("gs://%s/%s", jc.Bucket, object)})
	logger.Debugf("%s Start\n", prefix)
	defer logger.Debugf("%s Done\n", prefix)

	f, err := jc.Storage.Get(jc.Bucket, object)
	if err != nil {
		return false, err
	}
	// Ok unless the file exists
	if f == nil {
		return true, nil
	}

	ut, err := time.Parse(time.RFC3339, f.Updated)
	if err != nil {
		logger.Errorf("%s error parsing file update time %q of %v because of %v\n", prefix, f.Updated, f, err)
		return false, err
	}

	deadline := ut.Add(jc.Timeout)
	if deadline.After(time.Now()) {
		// deadline hasn't come yet
		logger.Warningf("%s deadline hasn't come yet. It seems another process is working.\n", prefix)
		return false, nil
	}

	logger.Debugf("%s delete exceeded lock file starting.\n", prefix)
	err = jc.Storage.Delete(jc.Bucket, object)
	if err != nil {
		logger.Errorf("%s delete exceeded lock file error.\n", prefix)
		return false, err
	}
	logger.Infof("%s delete exceeded lock file successfully.\n", prefix)

	return true, nil
}

func (jc *JobCheckByGcslock) Lock(m gcslock.ContextLocker) error {
	log.Debugln("JobCheckByGcslock.Lock start")

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	// Wait up to 10 seconds to acquire a lock.
	if err := m.ContextLock(ctx); err != nil {
		log.Errorf("Failed to ContextLock because of %v\n", err)
		return err
	}

	log.Infoln("JobCheckByGcslock.Lock done")
	return nil
}

func (jc *JobCheckByGcslock) Unlock(m gcslock.ContextLocker) error {
	log.Debugln("JobCheckByGcslock.Unlock start")

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	if err := m.ContextUnlock(ctx); err != nil {
		log.Errorf("Failed to ContextUnlock because of %v\n", err)
		return err
	}

	log.Infoln("JobCheckByGcslock.Unlock done")
	return nil
}

func (jc *JobCheckByGcslock) StartTouching(object string, interval time.Duration) error {
	logger := log.WithFields(logrus.Fields{"lock": fmt.Sprintf("gs://%s/%s", jc.Bucket, object), "interval": interval})
	logger.Infoln("JobCheckByGcslock.StartTouching Start")
	defer logger.Infoln("JobCheckByGcslock.StartTouching Finished")

	jc.working = true
	for {
		nextLimit := time.Now().Add(interval)
		err := jc.WaitAndTouch(object, nextLimit)
		if err != nil {
			log.WithFields(logrus.Fields{"error": err}).Errorln("Error in StartTouching")
			return err
		}
		if !jc.working {
			return nil
		}
	}
	// return nil
}

func (jc *JobCheckByGcslock) WaitAndTouch(object string, nextLimit time.Time) error {
	ticker := time.NewTicker(100 * time.Millisecond)
	for now := range ticker.C {
		if !jc.working {
			ticker.Stop()
			return nil
		}
		if now.After(nextLimit) {
			ticker.Stop()
			break
		}
	}

	jc.mux.Lock()
	defer jc.mux.Unlock()

	logger := log.WithFields(logrus.Fields{"bucket": jc.Bucket, "object": object})
	logger.Debugln("WaitAndTouch")

	if !jc.working {
		logger.Infoln("WaitAndTouch working is done")
		return nil
	}

	metadata := map[string]string{
		"JobCheckByGcslock": time.Now().Format(time.RFC3339),
	}
	msg := &storage.Object{Metadata: metadata}

	logger = logger.WithFields(logrus.Fields{"metadata": metadata})
	logger.Debugln("WaitAndTouch Update lock file starting")
	_, err := jc.Storage.Update(jc.Bucket, object, msg)
	if err != nil {
		logger.Errorf("WaitAndTouch Update lock file error because of %v\n", err)
		return err
	}
	logger.Debugln("WaitAndTouch Update lock file success")
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileserver

import (
	"bytes"
	_ "embed"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
	"text/template"

	"github.com/caddyserver/caddy/v2"
	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
	"github.com/caddyserver/caddy/v2/modules/caddyhttp/templates"
	"go.uber.org/zap"
)

//go:embed browse.html
var defaultBrowseTemplate string

// Browse configures directory browsing.
type Browse struct {
	// Use this template file instead of the default browse template.
	TemplateFile string `json:"template_file,omitempty"`
}

func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
	fsrv.logger.Debug("browse enabled; listing directory contents",
		zap.String("path", dirPath),
		zap.String("root", root))

	// Navigation on the client-side gets messed up if the
	// URL doesn't end in a trailing slash because hrefs to
	// "b/c" at path "/a" end up going to "/b/c" instead
	// of "/a/b/c" - so we have to redirect in this case
	// so that the path is "/a/" and the client constructs
	// relative hrefs "b/c" to be "/a/b/c".
	//
	// Only redirect if the last element of the path (the filename) was not
	// rewritten; if the admin wanted to rewrite to the canonical path, they
	// would have, and we have to be very careful not to introduce unwanted
	// redirects and especially redirect loops! (Redirecting using the
	// original URI is necessary because that's the URI the browser knows,
	// we don't want to redirect from internally-rewritten URIs.)
	// See https://github.com/caddyserver/caddy/issues/4205.
	origReq := r.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request)
	if path.Base(origReq.URL.Path) == path.Base(r.URL.Path) {
		if !strings.HasSuffix(origReq.URL.Path, "/") {
			fsrv.logger.Debug("redirecting to trailing slash to preserve hrefs", zap.String("request_path", r.URL.Path))
			origReq.URL.Path += "/"
			http.Redirect(w, r, origReq.URL.String(), http.StatusMovedPermanently)
			return nil
		}
	}

	dir, err := fsrv.openFile(dirPath, w)
	if err != nil {
		return err
	}
	defer dir.Close()

	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)

	// calling path.Clean here prevents weird breadcrumbs when URL paths are sketchy like /%2e%2e%2f
	listing, err := fsrv.loadDirectoryContents(dir, root, path.Clean(r.URL.Path), repl)
	switch {
	case os.IsPermission(err):
		return caddyhttp.Error(http.StatusForbidden, err)
	case os.IsNotExist(err):
		return fsrv.notFound(w, r, next)
	case err != nil:
		return caddyhttp.Error(http.StatusInternalServerError, err)
	}

	fsrv.browseApplyQueryParams(w, r, &listing)

	buf := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(buf)

	acceptHeader := strings.ToLower(strings.Join(r.Header["Accept"], ","))

	// write response as either JSON or HTML
	if strings.Contains(acceptHeader, "application/json") {
		if err := json.NewEncoder(buf).Encode(listing.Items); err != nil {
			return caddyhttp.Error(http.StatusInternalServerError, err)
		}
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
	} else {
		var fs http.FileSystem
		if fsrv.Root != "" {
			fs = http.Dir(repl.ReplaceAll(fsrv.Root, "."))
		}

		var tplCtx = &templateContext{
			TemplateContext: templates.TemplateContext{
				Root:       fs,
				Req:        r,
				RespHeader: templates.WrappedHeader{Header: w.Header()},
			},
			browseTemplateContext: listing,
		}

		tpl, err := fsrv.makeBrowseTemplate(tplCtx)
		if err != nil {
			return fmt.Errorf("parsing browse template: %v", err)
		}
		if err := tpl.Execute(buf, tplCtx); err != nil {
			return caddyhttp.Error(http.StatusInternalServerError, err)
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
	}

	_, _ = buf.WriteTo(w)

	return nil
}

func (fsrv *FileServer) loadDirectoryContents(dir *os.File, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
	files, err := dir.Readdir(-1)
	if err != nil {
		return browseTemplateContext{}, err
	}

	// user can presumably browse "up" to parent folder if path is longer than "/"
	canGoUp := len(urlPath) > 1

	return fsrv.directoryListing(files, canGoUp, root, urlPath, repl), nil
}

// browseApplyQueryParams applies query parameters to the listing.
// It mutates the listing and may set cookies.
func (fsrv *FileServer) browseApplyQueryParams(w http.ResponseWriter, r *http.Request, listing *browseTemplateContext) {
	sortParam := r.URL.Query().Get("sort")
	orderParam := r.URL.Query().Get("order")
	limitParam := r.URL.Query().Get("limit")
	offsetParam := r.URL.Query().Get("offset")

	// first figure out what to sort by
	switch sortParam {
	case "":
		sortParam = sortByNameDirFirst
		if sortCookie, sortErr := r.Cookie("sort"); sortErr == nil {
			sortParam = sortCookie.Value
		}
	case sortByName, sortByNameDirFirst, sortBySize, sortByTime:
		http.SetCookie(w, &http.Cookie{Name: "sort", Value: sortParam, Secure: r.TLS != nil})
	}

	// then figure out the order
	switch orderParam {
	case "":
		orderParam = "asc"
		if orderCookie, orderErr := r.Cookie("order"); orderErr == nil {
			orderParam = orderCookie.Value
		}
	case "asc", "desc":
		http.SetCookie(w, &http.Cookie{Name: "order", Value: orderParam, Secure: r.TLS != nil})
	}

	// finally, apply the sorting and limiting
	listing.applySortAndLimit(sortParam, orderParam, limitParam, offsetParam)
}

// makeBrowseTemplate creates the template to be used for directory listings.
func (fsrv *FileServer) makeBrowseTemplate(tplCtx *templateContext) (*template.Template, error) {
	var tpl *template.Template
	var err error

	if fsrv.Browse.TemplateFile != "" {
		tpl = tplCtx.NewTemplate(path.Base(fsrv.Browse.TemplateFile))
		tpl, err = tpl.ParseFiles(fsrv.Browse.TemplateFile)
		if err != nil {
			return nil, fmt.Errorf("parsing browse template file: %v", err)
		}
	} else {
		tpl = tplCtx.NewTemplate("default_listing")
		tpl, err = tpl.Parse(defaultBrowseTemplate)
		if err != nil {
			return nil, fmt.Errorf("parsing default browse template: %v", err)
		}
	}

	return tpl, nil
}

// isSymlink return true if f is a symbolic link
func isSymlink(f os.FileInfo) bool {
	return f.Mode()&os.ModeSymlink != 0
}

// isSymlinkTargetDir returns true if f's symbolic link target
// is a directory.
func isSymlinkTargetDir(f os.FileInfo, root, urlPath string) bool {
	if !isSymlink(f) {
		return false
	}
	target := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
	targetInfo, err := os.Stat(target)
	if err != nil {
		return false
	}
	return targetInfo.IsDir()
}

// templateContext powers the context used when evaluating the browse template.
// It combines browse-specific features with the standard templates handler
// features.
type templateContext struct {
	templates.TemplateContext
	browseTemplateContext
}

// bufPool is used to increase the efficiency of file listings.
var bufPool = sync.Pool{
	New: func() interface{} {
		return new(bytes.Buffer)
	},
}
<commit_msg>fileserver: Canonical redir when whole path is stripped (#4549)<commit_after>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileserver

import (
	"bytes"
	_ "embed"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
	"text/template"

	"github.com/caddyserver/caddy/v2"
	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
	"github.com/caddyserver/caddy/v2/modules/caddyhttp/templates"
	"go.uber.org/zap"
)

//go:embed browse.html
var defaultBrowseTemplate string

// Browse configures directory browsing.
type Browse struct {
	// Use this template file instead of the default browse template.
	TemplateFile string `json:"template_file,omitempty"`
}

func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
	fsrv.logger.Debug("browse enabled; listing directory contents",
		zap.String("path", dirPath),
		zap.String("root", root))

	// Navigation on the client-side gets messed up if the
	// URL doesn't end in a trailing slash because hrefs to
	// "b/c" at path "/a" end up going to "/b/c" instead
	// of "/a/b/c" - so we have to redirect in this case
	// so that the path is "/a/" and the client constructs
	// relative hrefs "b/c" to be "/a/b/c".
	//
	// Only redirect if the last element of the path (the filename) was not
	// rewritten; if the admin wanted to rewrite to the canonical path, they
	// would have, and we have to be very careful not to introduce unwanted
	// redirects and especially redirect loops! (Redirecting using the
	// original URI is necessary because that's the URI the browser knows,
	// we don't want to redirect from internally-rewritten URIs.)
	// See https://github.com/caddyserver/caddy/issues/4205.
	// We also redirect if the path is empty, because this implies the path
	// prefix was fully stripped away by a `handle_path` handler for example.
	// See https://github.com/caddyserver/caddy/issues/4466.
	origReq := r.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request)
	if r.URL.Path == "" || path.Base(origReq.URL.Path) == path.Base(r.URL.Path) {
		if !strings.HasSuffix(origReq.URL.Path, "/") {
			fsrv.logger.Debug("redirecting to trailing slash to preserve hrefs", zap.String("request_path", r.URL.Path))
			origReq.URL.Path += "/"
			http.Redirect(w, r, origReq.URL.String(), http.StatusMovedPermanently)
			return nil
		}
	}

	dir, err := fsrv.openFile(dirPath, w)
	if err != nil {
		return err
	}
	defer dir.Close()

	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)

	// calling path.Clean here prevents weird breadcrumbs when URL paths are sketchy like /%2e%2e%2f
	listing, err := fsrv.loadDirectoryContents(dir, root, path.Clean(r.URL.Path), repl)
	switch {
	case os.IsPermission(err):
		return caddyhttp.Error(http.StatusForbidden, err)
	case os.IsNotExist(err):
		return fsrv.notFound(w, r, next)
	case err != nil:
		return caddyhttp.Error(http.StatusInternalServerError, err)
	}

	fsrv.browseApplyQueryParams(w, r, &listing)

	buf := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(buf)

	acceptHeader := strings.ToLower(strings.Join(r.Header["Accept"], ","))

	// write response as either JSON or HTML
	if strings.Contains(acceptHeader, "application/json") {
		if err := json.NewEncoder(buf).Encode(listing.Items); err != nil {
			return caddyhttp.Error(http.StatusInternalServerError, err)
		}
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
	} else {
		var fs http.FileSystem
		if fsrv.Root != "" {
			fs = http.Dir(repl.ReplaceAll(fsrv.Root, "."))
		}

		var tplCtx = &templateContext{
			TemplateContext: templates.TemplateContext{
				Root:       fs,
				Req:        r,
				RespHeader: templates.WrappedHeader{Header: w.Header()},
			},
			browseTemplateContext: listing,
		}

		tpl, err := fsrv.makeBrowseTemplate(tplCtx)
		if err != nil {
			return fmt.Errorf("parsing browse template: %v", err)
		}
		if err := tpl.Execute(buf, tplCtx); err != nil {
			return caddyhttp.Error(http.StatusInternalServerError, err)
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
	}

	_, _ = buf.WriteTo(w)

	return nil
}

func (fsrv *FileServer) loadDirectoryContents(dir *os.File, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
	files, err := dir.Readdir(-1)
	if err != nil {
		return browseTemplateContext{}, err
	}

	// user can presumably browse "up" to parent folder if path is longer than "/"
	canGoUp := len(urlPath) > 1

	return fsrv.directoryListing(files, canGoUp, root, urlPath, repl), nil
}

// browseApplyQueryParams applies query parameters to the listing.
// It mutates the listing and may set cookies.
func (fsrv *FileServer) browseApplyQueryParams(w http.ResponseWriter, r *http.Request, listing *browseTemplateContext) {
	sortParam := r.URL.Query().Get("sort")
	orderParam := r.URL.Query().Get("order")
	limitParam := r.URL.Query().Get("limit")
	offsetParam := r.URL.Query().Get("offset")

	// first figure out what to sort by
	switch sortParam {
	case "":
		sortParam = sortByNameDirFirst
		if sortCookie, sortErr := r.Cookie("sort"); sortErr == nil {
			sortParam = sortCookie.Value
		}
	case sortByName, sortByNameDirFirst, sortBySize, sortByTime:
		http.SetCookie(w, &http.Cookie{Name: "sort", Value: sortParam, Secure: r.TLS != nil})
	}

	// then figure out the order
	switch orderParam {
	case "":
		orderParam = "asc"
		if orderCookie, orderErr := r.Cookie("order"); orderErr == nil {
			orderParam = orderCookie.Value
		}
	case "asc", "desc":
		http.SetCookie(w, &http.Cookie{Name: "order", Value: orderParam, Secure: r.TLS != nil})
	}

	// finally, apply the sorting and limiting
	listing.applySortAndLimit(sortParam, orderParam, limitParam, offsetParam)
}

// makeBrowseTemplate creates the template to be used for directory listings.
func (fsrv *FileServer) makeBrowseTemplate(tplCtx *templateContext) (*template.Template, error) {
	var tpl *template.Template
	var err error

	if fsrv.Browse.TemplateFile != "" {
		tpl = tplCtx.NewTemplate(path.Base(fsrv.Browse.TemplateFile))
		tpl, err = tpl.ParseFiles(fsrv.Browse.TemplateFile)
		if err != nil {
			return nil, fmt.Errorf("parsing browse template file: %v", err)
		}
	} else {
		tpl = tplCtx.NewTemplate("default_listing")
		tpl, err = tpl.Parse(defaultBrowseTemplate)
		if err != nil {
			return nil, fmt.Errorf("parsing default browse template: %v", err)
		}
	}

	return tpl, nil
}

// isSymlink return true if f is a symbolic link
func isSymlink(f os.FileInfo) bool {
	return f.Mode()&os.ModeSymlink != 0
}

// isSymlinkTargetDir returns true if f's symbolic link target
// is a directory.
func isSymlinkTargetDir(f os.FileInfo, root, urlPath string) bool {
	if !isSymlink(f) {
		return false
	}
	target := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
	targetInfo, err := os.Stat(target)
	if err != nil {
		return false
	}
	return targetInfo.IsDir()
}

// templateContext powers the context used when evaluating the browse template.
// It combines browse-specific features with the standard templates handler
// features.
type templateContext struct {
	templates.TemplateContext
	browseTemplateContext
}

// bufPool is used to increase the efficiency of file listings.
var bufPool = sync.Pool{
	New: func() interface{} {
		return new(bytes.Buffer)
	},
}
<|endoftext|>