<commit_before>package application

import (
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"database/sql"
	"io/ioutil"
	"log"
	"os"
	"path"
	"time"

	"github.com/cloudfoundry-incubator/notifications/db"
	"github.com/cloudfoundry-incubator/notifications/gobble"
	"github.com/cloudfoundry-incubator/notifications/util"
	v1models "github.com/cloudfoundry-incubator/notifications/v1/models"
	"github.com/go-sql-driver/mysql"
)

type DBProvider struct {
	sqlDB *sql.DB
	env   Environment
}

func NewDBProvider(env Environment) *DBProvider {
	var err error
	databaseURL := env.DatabaseURL
	if env.DatabaseCACertFile != "" {
		registerTLSConfig(env)
		databaseURL += "&tls=custom"
		if !env.DatabaseEnableIdentityVerification {
			databaseURL += "&trustServerCertificate=true&verifyServerCertificate=true&disableSslHostnameVerification=true"
		}
	}

	sqlDB, err := sql.Open("mysql", databaseURL)
	if err != nil {
		panic(err)
	}

	if err := sqlDB.Ping(); err != nil {
		panic(err)
	}

	sqlDB.SetMaxOpenConns(env.DBMaxOpenConns)
	return &DBProvider{
		sqlDB: sqlDB,
		env:   env,
	}
}

func (d *DBProvider) GobbleDatabase() gobble.DatabaseInterface {
	return gobble.NewDatabase(d.sqlDB)
}

func (d *DBProvider) Queue() gobble.QueueInterface {
	return gobble.NewQueue(d.GobbleDatabase(), util.NewClock(), gobble.Config{
		WaitMaxDuration: time.Duration(d.env.GobbleWaitMaxDuration) * time.Millisecond,
	})
}

func (d *DBProvider) Database() db.DatabaseInterface {
	database := v1models.NewDatabase(d.sqlDB, v1models.Config{
		DefaultTemplatePath: path.Join(d.env.RootPath, "templates", "default.json"),
	})

	if d.env.DBLoggingEnabled {
		database.TraceOn("[DB]", log.New(os.Stdout, "", 0))
	}

	return database
}

func (d *DBProvider) MessagesRepo() v1models.MessagesRepo {
	return v1models.NewMessagesRepo(util.NewIDGenerator(rand.Reader).Generate)
}

func registerTLSConfig(env Environment) {
	ca, err := ioutil.ReadFile(env.DatabaseCACertFile)
	if err != nil {
		panic(err)
	}
	rootCertPool := x509.NewCertPool()
	if ok := rootCertPool.AppendCertsFromPEM(ca); !ok {
		panic("Failed to append PEM when creating database connection")
	}
	mysql.RegisterTLSConfig("custom", &tls.Config{
		RootCAs:    rootCertPool,
		ServerName: env.DatabaseCommonName,
	})
}
<commit_msg>Implement skip hostname verification, fix weird imports<commit_after>package application

import (
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"database/sql"
	"io/ioutil"
	"log"
	"os"
	"path"
	"time"

	"github.com/cloudfoundry-incubator/notifications/db"
	"github.com/cloudfoundry-incubator/notifications/gobble"
	"github.com/cloudfoundry-incubator/notifications/util"
	v1models "github.com/cloudfoundry-incubator/notifications/v1/models"
	"github.com/go-sql-driver/mysql"
	"errors"
)

type DBProvider struct {
	sqlDB *sql.DB
	env   Environment
}

func NewDBProvider(env Environment) *DBProvider {
	var err error
	databaseURL := env.DatabaseURL
	if env.DatabaseCACertFile != "" {
		registerTLSConfig(env)
		databaseURL += "&tls=custom"
	}

	sqlDB, err := sql.Open("mysql", databaseURL)
	if err != nil {
		panic(err)
	}

	if err := sqlDB.Ping(); err != nil {
		panic(err)
	}

	sqlDB.SetMaxOpenConns(env.DBMaxOpenConns)
	return &DBProvider{
		sqlDB: sqlDB,
		env:   env,
	}
}

func (d *DBProvider) GobbleDatabase() gobble.DatabaseInterface {
	return gobble.NewDatabase(d.sqlDB)
}

func (d *DBProvider) Queue() gobble.QueueInterface {
	return gobble.NewQueue(d.GobbleDatabase(), util.NewClock(), gobble.Config{
		WaitMaxDuration: time.Duration(d.env.GobbleWaitMaxDuration) * time.Millisecond,
	})
}

func (d *DBProvider) Database() db.DatabaseInterface {
	database := v1models.NewDatabase(d.sqlDB, v1models.Config{
		DefaultTemplatePath: path.Join(d.env.RootPath, "templates", "default.json"),
	})

	if d.env.DBLoggingEnabled {
		database.TraceOn("[DB]", log.New(os.Stdout, "", 0))
	}

	return database
}

func (d *DBProvider) MessagesRepo() v1models.MessagesRepo {
	return v1models.NewMessagesRepo(util.NewIDGenerator(rand.Reader).Generate)
}

func registerTLSConfig(env Environment) {
	ca, err := ioutil.ReadFile(env.DatabaseCACertFile)
	if err != nil {
		panic(err)
	}
	rootCertPool := x509.NewCertPool()
	if ok := rootCertPool.AppendCertsFromPEM(ca); !ok {
		panic("Failed to append PEM when creating database connection")
	}

	tlsConfig := &tls.Config{
		RootCAs:    rootCertPool,
		ServerName: env.DatabaseCommonName,
		InsecureSkipVerify: false,
	}

	if !env.DatabaseEnableIdentityVerification {
		tlsConfig.InsecureSkipVerify = true
		tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
			return VerifyCertificatesIgnoreHostname(rawCerts, rootCertPool)
		}
	}

	mysql.RegisterTLSConfig("custom", tlsConfig)
}

func VerifyCertificatesIgnoreHostname(rawCerts [][]byte, caCertPool *x509.CertPool) error {
	certs := make([]*x509.Certificate, len(rawCerts))

	for i, asn1Data := range rawCerts {
		cert, err := x509.ParseCertificate(asn1Data)
		if err != nil {
			return errors.New("tls: failed to parse certificate from server: " + err.Error())
		}

		certs[i] = cert
	}

	opts := x509.VerifyOptions{
		Roots:         caCertPool,
		CurrentTime:   time.Now(),
		Intermediates: x509.NewCertPool(),
	}

	for i, cert := range certs {
		if i == 0 {
			continue
		}

		opts.Intermediates.AddCert(cert)
	}

	_, err := certs[0].Verify(opts)
	return err
}<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/google/cadvisor/container"
	info "github.com/google/cadvisor/info/v1"
	"github.com/google/cadvisor/utils"
	"github.com/karrick/godirwalk"
	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"k8s.io/klog/v2"
)

func DebugInfo(watches map[string][]string) map[string][]string {
	out := make(map[string][]string)

	lines := make([]string, 0, len(watches))
	for containerName, cgroupWatches := range watches {
		lines = append(lines, fmt.Sprintf("%s:", containerName))
		for _, cg := range cgroupWatches {
			lines = append(lines, fmt.Sprintf("\t%s", cg))
		}
	}
	out["Inotify watches"] = lines

	return out
}

// findFileInAncestorDir returns the path to the parent directory that contains the specified file.
// "" is returned if the lookup reaches the limit.
func findFileInAncestorDir(current, file, limit string) (string, error) {
	for {
		fpath := path.Join(current, file)
		_, err := os.Stat(fpath)
		if err == nil {
			return current, nil
		}
		if !os.IsNotExist(err) {
			return "", err
		}
		if current == limit {
			return "", nil
		}
		current = filepath.Dir(current)
	}
}

var bootTime = func() time.Time {
	now := time.Now()
	var sysinfo unix.Sysinfo_t
	if err := unix.Sysinfo(&sysinfo); err != nil {
		return now
	}
	sinceBoot := time.Duration(sysinfo.Uptime) * time.Second
	return now.Add(-1 * sinceBoot).Truncate(time.Minute)
}()

func GetSpec(cgroupPaths map[string]string, machineInfoFactory info.MachineInfoFactory, hasNetwork, hasFilesystem bool) (info.ContainerSpec, error) {
	var spec info.ContainerSpec

	// Assume unified hierarchy containers.
	// Get the lowest creation time from all hierarchies as the container creation time.
	now := time.Now()
	lowestTime := now
	for _, cgroupPath := range cgroupPaths {
		fi, err := os.Stat(cgroupPath)
		if err == nil && fi.ModTime().Before(lowestTime) {
			lowestTime = fi.ModTime()
		}
	}

	for _, cgroupPath := range cgroupPaths {
		// The modified time of the cgroup directory sometimes changes whenever a subcontainer is created.
		// eg. /docker will have creation time matching the creation of latest docker container.
		// Use clone_children/events as a workaround as it isn't usually modified. It is only likely changed
		// immediately after creating a container. If the directory modified time is lower, we use that.
		if cgroups.IsCgroup2UnifiedMode() {
			cgroupPath = path.Join(cgroupPath, "cgroup.events")
		} else {
			cgroupPath = path.Join(cgroupPath, "cgroup.clone_children")
		}
		fi, err := os.Stat(cgroupPath)
		if err == nil && fi.ModTime().Before(lowestTime) {
			lowestTime = fi.ModTime()
		}
	}
	if lowestTime.Before(bootTime) {
		lowestTime = bootTime
	}

	if lowestTime != now {
		spec.CreationTime = lowestTime
	}

	// Get machine info.
	mi, err := machineInfoFactory.GetMachineInfo()
	if err != nil {
		return spec, err
	}

	// CPU.
	cpuRoot, ok := cgroupPaths["cpu"]
	if ok {
		if utils.FileExists(cpuRoot) {
			spec.HasCpu = true
			spec.Cpu.Limit = readUInt64(cpuRoot, "cpu.shares")
			spec.Cpu.Period = readUInt64(cpuRoot, "cpu.cfs_period_us")
			quota := readString(cpuRoot, "cpu.cfs_quota_us")

			if quota != "" && quota != "-1" {
				val, err := strconv.ParseUint(quota, 10, 64)
				if err != nil {
					klog.Errorf("GetSpec: Failed to parse CPUQuota from %q: %s", path.Join(cpuRoot, "cpu.cfs_quota_us"), err)
				} else {
					spec.Cpu.Quota = val
				}
			}
		}
	}

	// Cpu Mask.
	// This will fail for non-unified hierarchies. We'll return the whole machine mask in that case.
	cpusetRoot, ok := cgroupPaths["cpuset"]
	if ok {
		if utils.FileExists(cpusetRoot) {
			spec.HasCpu = true
			mask := ""
			if cgroups.IsCgroup2UnifiedMode() {
				mask = readString(cpusetRoot, "cpuset.cpus.effective")
			} else {
				mask = readString(cpusetRoot, "cpuset.cpus")
			}
			spec.Cpu.Mask = utils.FixCpuMask(mask, mi.NumCores)
		}
	}

	// Memory
	memoryRoot, ok := cgroupPaths["memory"]
	if ok {
		if !cgroups.IsCgroup2UnifiedMode() {
			if utils.FileExists(memoryRoot) {
				spec.HasMemory = true
				spec.Memory.Limit = readUInt64(memoryRoot, "memory.limit_in_bytes")
				spec.Memory.SwapLimit = readUInt64(memoryRoot, "memory.memsw.limit_in_bytes")
				spec.Memory.Reservation = readUInt64(memoryRoot, "memory.soft_limit_in_bytes")
			}
		} else {
			memoryRoot, err := findFileInAncestorDir(memoryRoot, "memory.max", "/sys/fs/cgroup")
			if err != nil {
				return spec, err
			}
			if memoryRoot != "" {
				spec.HasMemory = true
				spec.Memory.Reservation = readUInt64(memoryRoot, "memory.high")
				spec.Memory.Limit = readUInt64(memoryRoot, "memory.max")
				spec.Memory.SwapLimit = readUInt64(memoryRoot, "memory.swap.max")
			}
		}
	}

	// Hugepage
	hugepageRoot, ok := cgroupPaths["hugetlb"]
	if ok {
		if utils.FileExists(hugepageRoot) {
			spec.HasHugetlb = true
		}
	}

	// Processes, read it's value from pids path directly
	pidsRoot, ok := cgroupPaths["pids"]
	if ok {
		if utils.FileExists(pidsRoot) {
			spec.HasProcesses = true
			spec.Processes.Limit = readUInt64(pidsRoot, "pids.max")
		}
	}

	spec.HasNetwork = hasNetwork
	spec.HasFilesystem = hasFilesystem

	ioControllerName := "blkio"
	if cgroups.IsCgroup2UnifiedMode() {
		ioControllerName = "io"
	}
	if blkioRoot, ok := cgroupPaths[ioControllerName]; ok && utils.FileExists(blkioRoot) {
		spec.HasDiskIo = true
	}

	return spec, nil
}

func readString(dirpath string, file string) string {
	cgroupFile := path.Join(dirpath, file)

	// Read
	out, err := ioutil.ReadFile(cgroupFile)
	if err != nil {
		// Ignore non-existent files
		if !os.IsNotExist(err) {
			klog.Warningf("readString: Failed to read %q: %s", cgroupFile, err)
		}
		return ""
	}
	return strings.TrimSpace(string(out))
}

func readUInt64(dirpath string, file string) uint64 {
	out := readString(dirpath, file)
	if out == "" || out == "max" {
		return 0
	}

	val, err := strconv.ParseUint(out, 10, 64)
	if err != nil {
		klog.Errorf("readUInt64: Failed to parse int %q from file %q: %s", out, path.Join(dirpath, file), err)
		return 0
	}

	return val
}

// Lists all directories under "path" and outputs the results as children of "parent".
func ListDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}) error {
	buf := make([]byte, godirwalk.MinimumScratchBufferSize)
	return listDirectories(dirpath, parent, recursive, output, buf)
}

func listDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}, buf []byte) error {
	dirents, err := godirwalk.ReadDirents(dirpath, buf)
	if err != nil {
		// Ignore if this hierarchy does not exist.
		if os.IsNotExist(errors.Cause(err)) {
			err = nil
		}
		return err
	}
	for _, dirent := range dirents {
		// We only grab directories.
		if !dirent.IsDir() {
			continue
		}
		dirname := dirent.Name()

		name := path.Join(parent, dirname)
		output[name] = struct{}{}

		// List subcontainers if asked to.
		if recursive {
			err := listDirectories(path.Join(dirpath, dirname), name, true, output, buf)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func MakeCgroupPaths(mountPoints map[string]string, name string) map[string]string {
	cgroupPaths := make(map[string]string, len(mountPoints))
	for key, val := range mountPoints {
		cgroupPaths[key] = path.Join(val, name)
	}

	return cgroupPaths
}

func CgroupExists(cgroupPaths map[string]string) bool {
	// If any cgroup exists, the container is still alive.
	for _, cgroupPath := range cgroupPaths {
		if utils.FileExists(cgroupPath) {
			return true
		}
	}
	return false
}

func ListContainers(name string, cgroupPaths map[string]string, listType container.ListType) ([]info.ContainerReference, error) {
	containers := make(map[string]struct{})
	for _, cgroupPath := range cgroupPaths {
		err := ListDirectories(cgroupPath, name, listType == container.ListRecursive, containers)
		if err != nil {
			return nil, err
		}
	}

	// Make into container references.
	ret := make([]info.ContainerReference, 0, len(containers))
	for cont := range containers {
		ret = append(ret, info.ContainerReference{
			Name: cont,
		})
	}

	return ret, nil
}

// AssignDeviceNamesToDiskStats assigns the Device field on the provided DiskIoStats by looking up
// the device major and minor identifiers in the provided device namer.
func AssignDeviceNamesToDiskStats(namer DeviceNamer, stats *info.DiskIoStats) {
	assignDeviceNamesToPerDiskStats(
		namer,
		stats.IoMerged,
		stats.IoQueued,
		stats.IoServiceBytes,
		stats.IoServiceTime,
		stats.IoServiced,
		stats.IoTime,
		stats.IoWaitTime,
		stats.Sectors,
	)
}

// assignDeviceNamesToPerDiskStats looks up device names for the provided stats, caching names
// if necessary.
func assignDeviceNamesToPerDiskStats(namer DeviceNamer, diskStats ...[]info.PerDiskStats) {
	devices := make(deviceIdentifierMap)
	for _, stats := range diskStats {
		for i, stat := range stats {
			stats[i].Device = devices.Find(stat.Major, stat.Minor, namer)
		}
	}
}

// DeviceNamer returns string names for devices by their major and minor id.
type DeviceNamer interface {
	// DeviceName returns the name of the device by its major and minor ids, or false if no
	// such device is recognized.
	DeviceName(major, minor uint64) (string, bool)
}

type MachineInfoNamer info.MachineInfo

func (n *MachineInfoNamer) DeviceName(major, minor uint64) (string, bool) {
	for _, info := range n.DiskMap {
		if info.Major == major && info.Minor == minor {
			return "/dev/" + info.Name, true
		}
	}
	for _, info := range n.Filesystems {
		if info.DeviceMajor == major && info.DeviceMinor == minor {
			return info.Device, true
		}
	}
	return "", false
}

type deviceIdentifier struct {
	major uint64
	minor uint64
}

type deviceIdentifierMap map[deviceIdentifier]string

// Find locates the device name by device identifier out of from, caching the result as necessary.
func (m deviceIdentifierMap) Find(major, minor uint64, namer DeviceNamer) string {
	d := deviceIdentifier{major, minor}
	if s, ok := m[d]; ok {
		return s
	}
	s, _ := namer.DeviceName(major, minor)
	m[d] = s
	return s
}
<commit_msg>Refactor container start time calculation<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/google/cadvisor/container"
	info "github.com/google/cadvisor/info/v1"
	"github.com/google/cadvisor/utils"
	"github.com/karrick/godirwalk"
	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"k8s.io/klog/v2"
)

func DebugInfo(watches map[string][]string) map[string][]string {
	out := make(map[string][]string)

	lines := make([]string, 0, len(watches))
	for containerName, cgroupWatches := range watches {
		lines = append(lines, fmt.Sprintf("%s:", containerName))
		for _, cg := range cgroupWatches {
			lines = append(lines, fmt.Sprintf("\t%s", cg))
		}
	}
	out["Inotify watches"] = lines

	return out
}

// findFileInAncestorDir returns the path to the parent directory that contains the specified file.
// "" is returned if the lookup reaches the limit.
func findFileInAncestorDir(current, file, limit string) (string, error) {
	for {
		fpath := path.Join(current, file)
		_, err := os.Stat(fpath)
		if err == nil {
			return current, nil
		}
		if !os.IsNotExist(err) {
			return "", err
		}
		if current == limit {
			return "", nil
		}
		current = filepath.Dir(current)
	}
}

var bootTime = func() time.Time {
	now := time.Now()
	var sysinfo unix.Sysinfo_t
	if err := unix.Sysinfo(&sysinfo); err != nil {
		return now
	}
	sinceBoot := time.Duration(sysinfo.Uptime) * time.Second
	return now.Add(-1 * sinceBoot).Truncate(time.Minute)
}()

func GetSpec(cgroupPaths map[string]string, machineInfoFactory info.MachineInfoFactory, hasNetwork, hasFilesystem bool) (info.ContainerSpec, error) {
	var spec info.ContainerSpec

	// Assume unified hierarchy containers.
	// Get the lowest creation time from all hierarchies as the container creation time.
	now := time.Now()
	lowestTime := now
	for _, cgroupPathDir := range cgroupPaths {
		dir, err := os.Stat(cgroupPathDir)
		if err == nil && dir.ModTime().Before(lowestTime) {
			lowestTime = dir.ModTime()
		}
		// The modified time of the cgroup directory sometimes changes whenever a subcontainer is created.
		// eg. /docker will have creation time matching the creation of latest docker container.
		// Use clone_children/events as a workaround as it isn't usually modified. It is only likely changed
		// immediately after creating a container. If the directory modified time is lower, we use that.
		cgroupPathFile := path.Join(cgroupPathDir, "cgroup.clone_children")
		if cgroups.IsCgroup2UnifiedMode() {
			cgroupPathFile = path.Join(cgroupPathDir, "cgroup.events")
		}
		fi, err := os.Stat(cgroupPathFile)
		if err == nil && fi.ModTime().Before(lowestTime) {
			lowestTime = fi.ModTime()
		}
	}
	if lowestTime.Before(bootTime) {
		lowestTime = bootTime
	}

	if lowestTime != now {
		spec.CreationTime = lowestTime
	}

	// Get machine info.
	mi, err := machineInfoFactory.GetMachineInfo()
	if err != nil {
		return spec, err
	}

	// CPU.
	cpuRoot, ok := cgroupPaths["cpu"]
	if ok {
		if utils.FileExists(cpuRoot) {
			spec.HasCpu = true
			spec.Cpu.Limit = readUInt64(cpuRoot, "cpu.shares")
			spec.Cpu.Period = readUInt64(cpuRoot, "cpu.cfs_period_us")
			quota := readString(cpuRoot, "cpu.cfs_quota_us")

			if quota != "" && quota != "-1" {
				val, err := strconv.ParseUint(quota, 10, 64)
				if err != nil {
					klog.Errorf("GetSpec: Failed to parse CPUQuota from %q: %s", path.Join(cpuRoot, "cpu.cfs_quota_us"), err)
				} else {
					spec.Cpu.Quota = val
				}
			}
		}
	}

	// Cpu Mask.
	// This will fail for non-unified hierarchies. We'll return the whole machine mask in that case.
	cpusetRoot, ok := cgroupPaths["cpuset"]
	if ok {
		if utils.FileExists(cpusetRoot) {
			spec.HasCpu = true
			mask := ""
			if cgroups.IsCgroup2UnifiedMode() {
				mask = readString(cpusetRoot, "cpuset.cpus.effective")
			} else {
				mask = readString(cpusetRoot, "cpuset.cpus")
			}
			spec.Cpu.Mask = utils.FixCpuMask(mask, mi.NumCores)
		}
	}

	// Memory
	memoryRoot, ok := cgroupPaths["memory"]
	if ok {
		if !cgroups.IsCgroup2UnifiedMode() {
			if utils.FileExists(memoryRoot) {
				spec.HasMemory = true
				spec.Memory.Limit = readUInt64(memoryRoot, "memory.limit_in_bytes")
				spec.Memory.SwapLimit = readUInt64(memoryRoot, "memory.memsw.limit_in_bytes")
				spec.Memory.Reservation = readUInt64(memoryRoot, "memory.soft_limit_in_bytes")
			}
		} else {
			memoryRoot, err := findFileInAncestorDir(memoryRoot, "memory.max", "/sys/fs/cgroup")
			if err != nil {
				return spec, err
			}
			if memoryRoot != "" {
				spec.HasMemory = true
				spec.Memory.Reservation = readUInt64(memoryRoot, "memory.high")
				spec.Memory.Limit = readUInt64(memoryRoot, "memory.max")
				spec.Memory.SwapLimit = readUInt64(memoryRoot, "memory.swap.max")
			}
		}
	}

	// Hugepage
	hugepageRoot, ok := cgroupPaths["hugetlb"]
	if ok {
		if utils.FileExists(hugepageRoot) {
			spec.HasHugetlb = true
		}
	}

	// Processes, read it's value from pids path directly
	pidsRoot, ok := cgroupPaths["pids"]
	if ok {
		if utils.FileExists(pidsRoot) {
			spec.HasProcesses = true
			spec.Processes.Limit = readUInt64(pidsRoot, "pids.max")
		}
	}

	spec.HasNetwork = hasNetwork
	spec.HasFilesystem = hasFilesystem

	ioControllerName := "blkio"
	if cgroups.IsCgroup2UnifiedMode() {
		ioControllerName = "io"
	}
	if blkioRoot, ok := cgroupPaths[ioControllerName]; ok && utils.FileExists(blkioRoot) {
		spec.HasDiskIo = true
	}

	return spec, nil
}

func readString(dirpath string, file string) string {
	cgroupFile := path.Join(dirpath, file)

	// Read
	out, err := ioutil.ReadFile(cgroupFile)
	if err != nil {
		// Ignore non-existent files
		if !os.IsNotExist(err) {
			klog.Warningf("readString: Failed to read %q: %s", cgroupFile, err)
		}
		return ""
	}
	return strings.TrimSpace(string(out))
}

func readUInt64(dirpath string, file string) uint64 {
	out := readString(dirpath, file)
	if out == "" || out == "max" {
		return 0
	}

	val, err := strconv.ParseUint(out, 10, 64)
	if err != nil {
		klog.Errorf("readUInt64: Failed to parse int %q from file %q: %s", out, path.Join(dirpath, file), err)
		return 0
	}

	return val
}

// Lists all directories under "path" and outputs the results as children of "parent".
func ListDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}) error {
	buf := make([]byte, godirwalk.MinimumScratchBufferSize)
	return listDirectories(dirpath, parent, recursive, output, buf)
}

func listDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}, buf []byte) error {
	dirents, err := godirwalk.ReadDirents(dirpath, buf)
	if err != nil {
		// Ignore if this hierarchy does not exist.
		if os.IsNotExist(errors.Cause(err)) {
			err = nil
		}
		return err
	}
	for _, dirent := range dirents {
		// We only grab directories.
		if !dirent.IsDir() {
			continue
		}
		dirname := dirent.Name()

		name := path.Join(parent, dirname)
		output[name] = struct{}{}

		// List subcontainers if asked to.
		if recursive {
			err := listDirectories(path.Join(dirpath, dirname), name, true, output, buf)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func MakeCgroupPaths(mountPoints map[string]string, name string) map[string]string {
	cgroupPaths := make(map[string]string, len(mountPoints))
	for key, val := range mountPoints {
		cgroupPaths[key] = path.Join(val, name)
	}

	return cgroupPaths
}

func CgroupExists(cgroupPaths map[string]string) bool {
	// If any cgroup exists, the container is still alive.
	for _, cgroupPath := range cgroupPaths {
		if utils.FileExists(cgroupPath) {
			return true
		}
	}
	return false
}

func ListContainers(name string, cgroupPaths map[string]string, listType container.ListType) ([]info.ContainerReference, error) {
	containers := make(map[string]struct{})
	for _, cgroupPath := range cgroupPaths {
		err := ListDirectories(cgroupPath, name, listType == container.ListRecursive, containers)
		if err != nil {
			return nil, err
		}
	}

	// Make into container references.
	ret := make([]info.ContainerReference, 0, len(containers))
	for cont := range containers {
		ret = append(ret, info.ContainerReference{
			Name: cont,
		})
	}

	return ret, nil
}

// AssignDeviceNamesToDiskStats assigns the Device field on the provided DiskIoStats by looking up
// the device major and minor identifiers in the provided device namer.
func AssignDeviceNamesToDiskStats(namer DeviceNamer, stats *info.DiskIoStats) {
	assignDeviceNamesToPerDiskStats(
		namer,
		stats.IoMerged,
		stats.IoQueued,
		stats.IoServiceBytes,
		stats.IoServiceTime,
		stats.IoServiced,
		stats.IoTime,
		stats.IoWaitTime,
		stats.Sectors,
	)
}

// assignDeviceNamesToPerDiskStats looks up device names for the provided stats, caching names
// if necessary.
func assignDeviceNamesToPerDiskStats(namer DeviceNamer, diskStats ...[]info.PerDiskStats) {
	devices := make(deviceIdentifierMap)
	for _, stats := range diskStats {
		for i, stat := range stats {
			stats[i].Device = devices.Find(stat.Major, stat.Minor, namer)
		}
	}
}

// DeviceNamer returns string names for devices by their major and minor id.
type DeviceNamer interface {
	// DeviceName returns the name of the device by its major and minor ids, or false if no
	// such device is recognized.
	DeviceName(major, minor uint64) (string, bool)
}

type MachineInfoNamer info.MachineInfo

func (n *MachineInfoNamer) DeviceName(major, minor uint64) (string, bool) {
	for _, info := range n.DiskMap {
		if info.Major == major && info.Minor == minor {
			return "/dev/" + info.Name, true
		}
	}
	for _, info := range n.Filesystems {
		if info.DeviceMajor == major && info.DeviceMinor == minor {
			return info.Device, true
		}
	}
	return "", false
}

type deviceIdentifier struct {
	major uint64
	minor uint64
}

type deviceIdentifierMap map[deviceIdentifier]string

// Find locates the device name by device identifier out of from, caching the result as necessary.
func (m deviceIdentifierMap) Find(major, minor uint64, namer DeviceNamer) string {
	d := deviceIdentifier{major, minor}
	if s, ok := m[d]; ok {
		return s
	}
	s, _ := namer.DeviceName(major, minor)
	m[d] = s
	return s
}
<|endoftext|>


<commit_before>package etcdconfig

import (
   "io"
   "fmt"
   "regexp"
   "io/ioutil"
   "encoding/json"
   "golang.org/x/net/context"
   "github.com/luisfurquim/goose"
   etcd "github.com/coreos/etcd/client"
)

var reArrayIndex *regexp.Regexp = regexp.MustCompile("/\\[([0-9]+)\\]$")
var reMapIndex   *regexp.Regexp = regexp.MustCompile("/([^/]*)$")

type EtcdconfigG struct {
   Setter  goose.Alert
   Getter  goose.Alert
   Updater goose.Alert
}

var Goose EtcdconfigG

func rSetConfig(path string, config map[string]interface{}, etcdcli etcd.KeysAPI) error {
   var key           string
   var key2          int
   var value, value2 interface{}
   var err           error
   var resp         *etcd.Response
   var optDir       *etcd.SetOptions
   var ctx           context.Context

   optDir = &etcd.SetOptions{Dir:true}
   ctx    = context.Background()

   resp, err = etcdcli.Set(ctx, path, "",optDir)
   if err != nil {
      Goose.Setter.Logf(1,"Error setting configuration, creating diretory.1 (%s): %s",path,err)
      Goose.Setter.Fatalf(5,"path:%s,   key:%s     Metadata: %q",path, key, resp)
   }

   for key, value = range config {
      switch value.(type) {
         case map[string]interface{} :
            Goose.Setter.Logf(5,"Found %s/%s=%q => map[string]interface{}", path, key, value.(map[string]interface{}))
            err = rSetConfig(path + "/" + key, value.(map[string]interface{}), etcdcli)
            if err != nil {
               return err
            }
         case []interface{} :
            Goose.Setter.Logf(5,"Found %s/%s=%q => []interface{}", path, key, value.([]interface{}))
            resp, err = etcdcli.Set(ctx, fmt.Sprintf("%s/%s",path,key), "", optDir)
            if err != nil {
               Goose.Setter.Fatalf(1,"Error setting configuration, creating diretory.2 (%s/%s): %s",path,key,err)
            }

            for key2, value2 = range value.([]interface{}) {
               switch value2.(type) {
                  case map[string]interface{} :
                     Goose.Setter.Logf(5,"Found %s/%s=%q => []interface{map[string]interface{}}", path, key, value2.(map[string]interface{}))
                     err = rSetConfig(fmt.Sprintf("%s/%s/[%d]",path,key,key2), value2.(map[string]interface{}), etcdcli)
                     if err != nil {
                        return err
                     }
                  case string :
                     Goose.Setter.Logf(5,"Found %s/%s=%q => []interface{string}", path, key, value2.(string))
                     resp, err = etcdcli.Set(ctx, fmt.Sprintf("%s/%s/[%d]",path,key,key2), value2.(string), nil)
                     if err != nil {
                        Goose.Setter.Fatalf(1,"Error setting configuration.1: %s",err)
                     } else {
                        // print common key info
                        Goose.Setter.Logf(5,"Configuration set. Metadata: %q\n", resp)
                     }
                  default:
                     Goose.Setter.Fatalf(1,"Invalid type: key=%s, key2=%d, value=%v",key,key2,value2)
               }
            }
         case string :
            resp, err = etcdcli.Set(ctx, path + "/" + key, value.(string), nil)
            if err != nil {
               Goose.Setter.Logf(1,"Error setting configuration.2: %s",err)
               Goose.Setter.Fatalf(5,"path:%s,   key:%s     Metadata: %q",path, key, resp)
            } else {
               // print common key info
               Goose.Setter.Logf(5,"Configuration %s/%s=%s set. Metadata: %q", path, key, value.(string), resp)
            }

         default:
            Goose.Setter.Fatalf(1,"Invalid type: key=%s, value=%v",key,value)

      }
   }

   return nil
}

func SetConfig(cfg string, etcdcli etcd.Client, key string) error {
   var err         error
   var configbuf []byte
   var config       map[string]interface{}

   configbuf, err = ioutil.ReadFile(cfg)
   if err != nil {
      Goose.Setter.Logf(1,"Error reading config file (%s)\n",err)
      return err
   }

   err = json.Unmarshal(configbuf, &config);
   if err != nil {
      Goose.Setter.Logf(1,"Error parsing config (%s)\n",err)
      return err
   }

   err = rSetConfig("/" + key,config,etcd.NewKeysAPI(etcdcli))
   if err != nil {
      Goose.Setter.Logf(1,"Error setting config cluster (%s)\n",err)
      return err
   }

   return nil
}

func SetConfigFromReader(cfg io.Reader, etcdcli etcd.Client, key string) error {
   var err         error
   var configbuf []byte
   var config       map[string]interface{}

   configbuf, err = ioutil.ReadAll(cfg)
   if err != nil {
      Goose.Setter.Logf(1,"Error reading config file (%s)\n",err)
      return err
   }

   err = json.Unmarshal(configbuf, &config);
   if err != nil {
      Goose.Setter.Logf(1,"Error parsing config (%s)\n",err)
      return err
   }

   err = rSetConfig("/" + key,config,etcd.NewKeysAPI(etcdcli))
   if err != nil {
      Goose.Setter.Logf(1,"Error setting config cluster (%s)\n",err)
      return err
   }

   return nil
}

func SetConfigFromMap(config map[string]interface{}, etcdcli etcd.Client, key string) error {
   var err         error

   err = rSetConfig("/" + key,config,etcd.NewKeysAPI(etcdcli))
   if err != nil {
      Goose.Setter.Logf(1,"Error setting config cluster (%s)\n",err)
      return err
   }

   return nil
}

func rShowConfig(node *etcd.Node) error {
   var err     error
   var child  *etcd.Node

   if !node.Dir {
      Goose.Getter.Logf(1,"[%s] => %s",node.Key,node.Value)
      return nil
   }

   Goose.Getter.Logf(1,"[%s]",node.Key)
   for _, child = range node.Nodes {
     if child != nil {
        err = rShowConfig(child)
        if err != nil {
           Goose.Getter.Logf(1,"Error reading child node: %s",err)
           return err
        }
     }
   }

   return nil
}

func rGetConfig(node *etcd.Node) (interface{}, interface{}, error) {
   var err       error
   var child    *etcd.Node
   var i         int
   var data      interface{}
   var data2     interface{}
   var index     interface{}
   var index2    interface{}
   var array   []interface{}
   var matched []string

   matched = reArrayIndex.FindStringSubmatch(node.Key)
   if len(matched) > 0  {
      fmt.Sscanf(matched[1],"%d",&i)
      index = i
   } else {
      matched = reMapIndex.FindStringSubmatch(node.Key)
      if len(matched) <= 1  {
         Goose.Getter.Fatalf(1,"Error invalid index")
      }
      index = matched[1]
   }

   if !node.Dir {
      Goose.Getter.Logf(4,"[%s] => %s",node.Key,node.Value)
      return index, node.Value, nil
   }

   Goose.Getter.Logf(4,"[%s]",node.Key)
   for _, child = range node.Nodes {
      if child != nil {
         index2, data2, err = rGetConfig(child)
         if err != nil {
            Goose.Getter.Logf(1,"Error reading child node: %s",err)
            return nil, nil, err
         }
         switch index2.(type) {
            case string:
               if data == nil {
                  data = map[string]interface{}{}
               }
               data.(map[string]interface{})[index2.(string)] = data2
            case int:
               if array == nil {
                  array = make([]interface{},index2.(int)+1)
               } else if len(array) <= index2.(int) {
                  array = append(array,make([]interface{},index2.(int)-len(array)+1)...)
               }
               array[index2.(int)] = data2
               data = array
         }
      }
   }

   return index, data, nil
}

func GetConfig(etcdcli etcd.Client, key string) (interface{}, interface{}, error) {
   var err         error
   var resp       *etcd.Response

   resp, err = etcd.NewKeysAPI(etcdcli).Get(context.Background(), "/" + key, &etcd.GetOptions{Recursive:true})
   if err != nil {
      Goose.Getter.Logf(1,"Error fetching configuration: %s",err)
      return nil, nil, err
   }

   return rGetConfig(resp.Node)
}

func DeleteConfig(etcdcli etcd.Client, key string) error {
   var err         error

   _, err = etcd.NewKeysAPI(etcdcli).Delete(context.Background(), "/" + key, &etcd.DeleteOptions{Recursive:true,Dir:true})
   if err != nil {
      Goose.Updater.Logf(1,"Error deleting configuration: %s",err)
      return err
   }

   return nil
}

func SetKey(etcdcli etcd.Client, key string, value string) error {
   var err         error
   var resp         *etcd.Response
   var ctx           context.Context

   ctx    = context.Background()
   Goose.Setter.Logf(6,"Entering SetKey /%s <- %#v", key, value)
   resp, err = etcd.NewKeysAPI(etcdcli).Set(ctx, "/" + key, value, nil)
   if err != nil {
      Goose.Setter.Logf(1,"Error setting configuration.2: %s",err)
      Goose.Setter.Fatalf(5,"key:%s     Metadata: %q", key, resp)
   } else {
      // print common key info
      Goose.Setter.Logf(5,"Configuration set. Metadata: %q", resp)
   }

   return nil
}

func OnUpdate(etcdCli etcd.Client, key string, fn func(val string)) {
   var kapi           etcd.KeysAPI
   var ctx            context.Context

   kapi = etcd.NewKeysAPI(etcdCli)
   ctx  = context.Background()

   go func (w etcd.Watcher) {
      var err error
      var resp         *etcd.Response

      for {
         resp, err = w.Next(ctx)
         if err == nil {
            Goose.Updater.Logf(3,"Updating config variable %s = %s",key,resp.Node.Value)
            fn(resp.Node.Value)
         } else {
            Goose.Updater.Logf(1,"Error updating config variable %s (%s)",key,err)
         }
      }
   }(kapi.Watcher("/" + key,nil))
}

func OnUpdateIFace(etcdCli etcd.Client, key string, fn func(val interface{})) {
   var kapi           etcd.KeysAPI
   var ctx            context.Context

   kapi = etcd.NewKeysAPI(etcdCli)
   ctx  = context.Background()

   go func (w etcd.Watcher) {
      var err error
      var resp         *etcd.Response

      for {
         resp, err = w.Next(ctx)
         if err == nil {
            Goose.Updater.Logf(3,"Updating config variable %s = %s",key,resp.Node.Value)
            fn(resp.Node.Value)
         } else {
            Goose.Updater.Logf(1,"Error updating config variable %s (%s)",key,err)
         }
      }
   }(kapi.Watcher("/" + key,nil))
}

func OnUpdateTree(etcdCli etcd.Client, key string, fn func(key string, val interface{}, action string)) {
   var kapi           etcd.KeysAPI
   var ctx            context.Context

   kapi = etcd.NewKeysAPI(etcdCli)
   ctx  = context.Background()

   go func (w etcd.Watcher) {
      var err error
      var resp         *etcd.Response

      for {
         resp, err = w.Next(ctx)
         if err == nil {
            Goose.Updater.Logf(3,"Updating (%s) config variable %s = %s", resp.Action, key, resp.Node.Value)
            fn(resp.Node.Key, resp.Node.Value, resp.Action)
         } else {
            Goose.Updater.Logf(1,"Error updating config variable %s (%s)",key,err)
         }
      }
   }(kapi.Watcher("/" + key,&etcd.WatcherOptions{Recursive:true}))
}

<commit_msg>@{newfeature}Added a key watcher to track when an update happens on a key and run a user defined callback function<commit_after>package etcdconfig

import (
   "io"
   "fmt"
   "sort"
   "regexp"
//   "reflect"
   "io/ioutil"
   "encoding/json"
   "golang.org/x/net/context"
   "github.com/luisfurquim/goose"
   etcd "github.com/coreos/etcd/client"
)

type watchReq struct {
   key string
   id  string
   resp chan bool
}

type EtcdconfigG struct {
   Setter  goose.Alert
   Getter  goose.Alert
   Updater goose.Alert
}

var Goose EtcdconfigG

var reArrayIndex *regexp.Regexp = regexp.MustCompile("/\\[([0-9]+)\\]$")
var reMapIndex   *regexp.Regexp = regexp.MustCompile("/([^/]*)$")
var keysWatched map[string][]string
var wreq chan watchReq

func rSetConfig(path string, config map[string]interface{}, etcdcli etcd.KeysAPI) error {
   var key           string
   var key2          int
   var value, value2 interface{}
   var err           error
   var resp         *etcd.Response
   var optDir       *etcd.SetOptions
   var ctx           context.Context

   optDir = &etcd.SetOptions{Dir:true}
   ctx    = context.Background()

   resp, err = etcdcli.Set(ctx, path, "",optDir)
   if err != nil {
      Goose.Setter.Logf(1,"Error setting configuration, creating diretory.1 (%s): %s",path,err)
      Goose.Setter.Fatalf(5,"path:%s,   key:%s     Metadata: %q",path, key, resp)
   }

   for key, value = range config {
      switch value.(type) {
         case map[string]interface{} :
            Goose.Setter.Logf(5,"Found %s/%s=%q => map[string]interface{}", path, key, value.(map[string]interface{}))
            err = rSetConfig(path + "/" + key, value.(map[string]interface{}), etcdcli)
            if err != nil {
               return err
            }
         case []interface{} :
            Goose.Setter.Logf(5,"Found %s/%s=%q => []interface{}", path, key, value.([]interface{}))
            resp, err = etcdcli.Set(ctx, fmt.Sprintf("%s/%s",path,key), "", optDir)
            if err != nil {
               Goose.Setter.Fatalf(1,"Error setting configuration, creating diretory.2 (%s/%s): %s",path,key,err)
            }

            for key2, value2 = range value.([]interface{}) {
               switch value2.(type) {
                  case map[string]interface{} :
                     Goose.Setter.Logf(5,"Found %s/%s=%q => []interface{map[string]interface{}}", path, key, value2.(map[string]interface{}))
                     err = rSetConfig(fmt.Sprintf("%s/%s/[%d]",path,key,key2), value2.(map[string]interface{}), etcdcli)
                     if err != nil {
                        return err
                     }
                  case string :
                     Goose.Setter.Logf(5,"Found %s/%s=%q => []interface{string}", path, key, value2.(string))
                     resp, err = etcdcli.Set(ctx, fmt.Sprintf("%s/%s/[%d]",path,key,key2), value2.(string), nil)
                     if err != nil {
                        Goose.Setter.Fatalf(1,"Error setting configuration.1: %s",err)
                     } else {
                        // print common key info
                        Goose.Setter.Logf(5,"Configuration set. Metadata: %q\n", resp)
                     }
                  default:
                     Goose.Setter.Fatalf(1,"Invalid type: key=%s, key2=%d, value=%v",key,key2,value2)
               }
            }
         case string :
            resp, err = etcdcli.Set(ctx, path + "/" + key, value.(string), nil)
            if err != nil {
               Goose.Setter.Logf(1,"Error setting configuration.2: %s",err)
               Goose.Setter.Fatalf(5,"path:%s,   key:%s     Metadata: %q",path, key, resp)
            } else {
               // print common key info
               Goose.Setter.Logf(5,"Configuration %s/%s=%s set. Metadata: %q", path, key, value.(string), resp)
            }

         default:
            Goose.Setter.Fatalf(1,"Invalid type: key=%s, value=%v",key,value)

      }
   }

   return nil
}

func SetConfig(cfg string, etcdcli etcd.Client, key string) error {
   var err         error
   var configbuf []byte
   var config       map[string]interface{}

   configbuf, err = ioutil.ReadFile(cfg)
   if err != nil {
      Goose.Setter.Logf(1,"Error reading config file (%s)\n",err)
      return err
   }

   err = json.Unmarshal(configbuf, &config);
   if err != nil {
      Goose.Setter.Logf(1,"Error parsing config (%s)\n",err)
      return err
   }

   err = rSetConfig("/" + key,config,etcd.NewKeysAPI(etcdcli))
   if err != nil {
      Goose.Setter.Logf(1,"Error setting config cluster (%s)\n",err)
      return err
   }

   return nil
}

func SetConfigFromReader(cfg io.Reader, etcdcli etcd.Client, key string) error {
   var err         error
   var configbuf []byte
   var config       map[string]interface{}

   configbuf, err = ioutil.ReadAll(cfg)
   if err != nil {
      Goose.Setter.Logf(1,"Error reading config file (%s)\n",err)
      return err
   }

   err = json.Unmarshal(configbuf, &config);
   if err != nil {
      Goose.Setter.Logf(1,"Error parsing config (%s)\n",err)
      return err
   }

   err = rSetConfig("/" + key,config,etcd.NewKeysAPI(etcdcli))
   if err != nil {
      Goose.Setter.Logf(1,"Error setting config cluster (%s)\n",err)
      return err
   }

   return nil
}

func SetConfigFromMap(config map[string]interface{}, etcdcli etcd.Client, key string) error {
   var err         error

   err = rSetConfig("/" + key,config,etcd.NewKeysAPI(etcdcli))
   if err != nil {
      Goose.Setter.Logf(1,"Error setting config cluster (%s)\n",err)
      return err
   }

   return nil
}

func rShowConfig(node *etcd.Node) error {
   var err     error
   var child  *etcd.Node

   if !node.Dir {
      Goose.Getter.Logf(1,"[%s] => %s",node.Key,node.Value)
      return nil
   }

   Goose.Getter.Logf(1,"[%s]",node.Key)
   for _, child = range node.Nodes {
     if child != nil {
        err = rShowConfig(child)
        if err != nil {
           Goose.Getter.Logf(1,"Error reading child node: %s",err)
           return err
        }
     }
   }

   return nil
}

func rGetConfig(node *etcd.Node) (interface{}, interface{}, error) {
   var err       error
   var child    *etcd.Node
   var i         int
   var data      interface{}
   var data2     interface{}
   var index     interface{}
   var index2    interface{}
   var array   []interface{}
   var matched []string

   matched = reArrayIndex.FindStringSubmatch(node.Key)
   if len(matched) > 0  {
      fmt.Sscanf(matched[1],"%d",&i)
      index = i
   } else {
      matched = reMapIndex.FindStringSubmatch(node.Key)
      if len(matched) <= 1  {
         Goose.Getter.Fatalf(1,"Error invalid index")
      }
      index = matched[1]
   }

   if !node.Dir {
      Goose.Getter.Logf(4,"[%s] => %s",node.Key,node.Value)
      return index, node.Value, nil
   }

   Goose.Getter.Logf(4,"[%s]",node.Key)
   for _, child = range node.Nodes {
      if child != nil {
         index2, data2, err = rGetConfig(child)
         if err != nil {
            Goose.Getter.Logf(1,"Error reading child node: %s",err)
            return nil, nil, err
         }
         switch index2.(type) {
            case string:
               if data == nil {
                  data = map[string]interface{}{}
               }
               data.(map[string]interface{})[index2.(string)] = data2
            case int:
               if array == nil {
                  array = make([]interface{},index2.(int)+1)
               } else if len(array) <= index2.(int) {
                  array = append(array,make([]interface{},index2.(int)-len(array)+1)...)
               }
               array[index2.(int)] = data2
               data = array
         }
      }
   }

   return index, data, nil
}

func GetConfig(etcdcli etcd.Client, key string) (interface{}, interface{}, error) {
   var err         error
   var resp       *etcd.Response

   resp, err = etcd.NewKeysAPI(etcdcli).Get(context.Background(), "/" + key, &etcd.GetOptions{Recursive:true})
   if err != nil {
      Goose.Getter.Logf(1,"Error fetching configuration: %s",err)
      return nil, nil, err
   }

   return rGetConfig(resp.Node)
}

func DeleteConfig(etcdcli etcd.Client, key string) error {
   var err         error

   _, err = etcd.NewKeysAPI(etcdcli).Delete(context.Background(), "/" + key, &etcd.DeleteOptions{Recursive:true,Dir:true})
   if err != nil {
      Goose.Updater.Logf(1,"Error deleting configuration: %s",err)
      return err
   }

   return nil
}

func SetKey(etcdcli etcd.Client, key string, value string) error {
   var err         error
   var resp         *etcd.Response
   var ctx           context.Context

   ctx    = context.Background()
   Goose.Setter.Logf(6,"Entering SetKey /%s <- %#v", key, value)
   resp, err = etcd.NewKeysAPI(etcdcli).Set(ctx, "/" + key, value, nil)
   if err != nil {
      Goose.Setter.Logf(1,"Error setting configuration.2: %s",err)
      Goose.Setter.Logf(5,"key:%s     Metadata: %q", key, resp)
      panic(fmt.Sprintf("Error setting configuration: %s",err))
   } else {
      // print common key info
      Goose.Setter.Logf(5,"Configuration set. Metadata: %q", resp)
   }

   return nil
}

func watcherChecker(req chan watchReq) {
   var reqKey watchReq
   var ok bool
   var j int
   var kw []string

   for {
      reqKey = <-req
      if reqKey.key == "" {
         return
      }

      if kw, ok = keysWatched[reqKey.key]; !ok {
         reqKey.resp<- false
         keysWatched[reqKey.key] = []string{reqKey.id}
      }

      j = sort.Search(len(kw), func(i int) bool { return kw[i] >= reqKey.id })
      if j < len(kw) && kw[j] == reqKey.id {
         // reqKey.fn is present at kw[j]
         reqKey.resp <- true
      } else {
         reqKey.resp <- false
         // reqKey.id is not present at kw[j]
         // but j is the index where it would be inserted.
         if j==len(kw) {
            keysWatched[reqKey.key] = append(kw,reqKey.id)
         } else {
            kw = append(kw,reqKey.id)
            copy(kw[j+1:],kw[j:len(kw)-1])
            kw[j] = reqKey.id
            keysWatched[reqKey.key] = kw
         }
      }
   }
}

func chkWatcher(key string, id string) bool {
   var resp chan bool

   resp = make(chan bool,1)
   wreq<- watchReq{key:key, id:id, resp: resp}

   return <-resp
}

func OnUpdate(etcdCli etcd.Client, key string, fn func(val string), id ...interface{}) {
   var kapi           etcd.KeysAPI
   var ctx            context.Context
   var sid            string

   if len(id) > 0 {
      switch id[0].(type) {
         case string:
            sid = id[0].(string)
         default:
            Goose.Updater.Logf(2,"Ignoring wrong parameter type for key %s (%#v)",key,id[0])
            return
      }
   }

   if chkWatcher(key, sid) {
      return
   }

   kapi = etcd.NewKeysAPI(etcdCli)
   ctx  = context.Background()

   go func (w etcd.Watcher) {
      var err error
      var resp         *etcd.Response

      for {
         resp, err = w.Next(ctx)
         if err == nil {
            Goose.Updater.Logf(3,"Updating config variable %s = %s", key, resp.Node.Value)
            fn(resp.Node.Value)
         } else {
            Goose.Updater.Logf(1,"Error updating config variable %s (%s)",key,err)
         }
      }
   }(kapi.Watcher("/" + key,nil))
}

func OnUpdateIFace(etcdCli etcd.Client, key string, fn func(val interface{}), id ...interface{}) {
   var kapi           etcd.KeysAPI
   var ctx            context.Context
   var sid            string

   if len(id) > 0 {
      switch id[0].(type) {
         case string:
            sid = id[0].(string)
         default:
            Goose.Updater.Logf(2,"Ignoring wrong parameter type for key %s (%#v)",key,id[0])
            return
      }
   }

   if chkWatcher(key, sid) {
      return
   }

   kapi = etcd.NewKeysAPI(etcdCli)
   ctx  = context.Background()

   go func (w etcd.Watcher) {
      var err error
      var resp         *etcd.Response

      for {
         resp, err = w.Next(ctx)
         if err == nil {
            Goose.Updater.Logf(3,"Updating config variable %s = %s",key,resp.Node.Value)
            fn(resp.Node.Value)
         } else {
            Goose.Updater.Logf(1,"Error updating config variable %s (%s)",key,err)
         }
      }
   }(kapi.Watcher("/" + key,nil))
}

func OnUpdateTree(etcdCli etcd.Client, key string, fn func(key string, val interface{}, action string), id ...interface{}) {
   var kapi           etcd.KeysAPI
   var ctx            context.Context
   var sid            string

   if len(id) > 0 {
      switch id[0].(type) {
         case string:
            sid = id[0].(string)
         default:
            Goose.Updater.Logf(2,"Ignoring wrong parameter type for key %s (%#v)",key,id[0])
            return
      }
   }

   if chkWatcher(key, sid) {
      return
   }

   kapi = etcd.NewKeysAPI(etcdCli)
   ctx  = context.Background()

   go func (w etcd.Watcher) {
      var err error
      var resp         *etcd.Response

      for {
         resp, err = w.Next(ctx)
         if err == nil {
            Goose.Updater.Logf(3,"Updating (%s) config variable %s = %s", resp.Action, key, resp.Node.Value)
            fn(resp.Node.Key, resp.Node.Value, resp.Action)
         } else {
            Goose.Updater.Logf(1,"Error updating config variable %s (%s)",key,err)
         }
      }
   }(kapi.Watcher("/" + key,&etcd.WatcherOptions{Recursive:true}))
}

func init() {
   keysWatched = map[string][]string{}
   wreq = make(chan watchReq,1)
   go watcherChecker(wreq)
}

<|endoftext|>


<commit_before>package eth

import (
	"context"
	"errors"
	"math/big"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/livepeer/go-livepeer/common"
	"github.com/livepeer/go-livepeer/monitor"
)

// GasPriceOracle defines methods for fetching a suggested gas price
// for submitting transactions
type GasPriceOracle interface {
	SuggestGasPrice(ctx context.Context) (*big.Int, error)
}

// GasPriceMonitor polls for gas price updates and updates its
// own view of the current gas price that can be used by others
type GasPriceMonitor struct {
	gpo GasPriceOracle
	// The following fields should be protected by `pollingMu`
	polling         bool
	pollingInterval time.Duration
	cancel          context.CancelFunc
	// pollingMu protects access to polling related fields
	pollingMu sync.Mutex

	// gasPriceMu protects access to gasPrice
	gasPriceMu sync.RWMutex
	// gasPrice is the current gas price to be returned to users
	gasPrice *big.Int

	// update is a channel used to send notifications to a listener
	// when the gas price is updated
	update chan struct{}
}

// NewGasPriceMonitor returns a GasPriceMonitor
func NewGasPriceMonitor(gpo GasPriceOracle, pollingInterval time.Duration) *GasPriceMonitor {
	return &GasPriceMonitor{
		gpo:             gpo,
		pollingInterval: pollingInterval,
		gasPrice:        big.NewInt(0),
	}
}

// GasPrice returns the current gas price
func (gpm *GasPriceMonitor) GasPrice() *big.Int {
	gpm.gasPriceMu.RLock()
	defer gpm.gasPriceMu.RUnlock()

	return gpm.gasPrice
}

// Start starts polling for gas price updates and returns a channel to receive
// notifications of gas price changes
func (gpm *GasPriceMonitor) Start(ctx context.Context) (chan struct{}, error) {
	gpm.pollingMu.Lock()
	defer gpm.pollingMu.Unlock()

	if gpm.polling {
		return nil, errors.New("already polling")
	}

	// Initialize gasPrice before starting to poll
	if err := gpm.fetchAndUpdateGasPrice(ctx); err != nil {
		return nil, err
	}

	gpm.update = make(chan struct{})

	cctx, cancel := context.WithCancel(ctx)
	gpm.cancel = cancel

	ticker := time.NewTicker(gpm.pollingInterval)

	go func(ctx context.Context) {
		for {
			select {
			case <-ticker.C:
				if err := gpm.fetchAndUpdateGasPrice(ctx); err != nil {
					glog.Errorf("error getting gas price: %v", err)
				}
			case <-ctx.Done():
				gpm.pollingMu.Lock()
				gpm.cancel = nil
				gpm.polling = false
				gpm.pollingMu.Unlock()
				return
			}
		}
	}(cctx)

	gpm.polling = true

	return gpm.update, nil
}

// Stop stops polling for gas price updates
func (gpm *GasPriceMonitor) Stop() error {
	gpm.pollingMu.Lock()
	defer gpm.pollingMu.Unlock()

	if !gpm.polling {
		return errors.New("not polling")
	}

	gpm.cancel()
	gpm.cancel = nil
	gpm.polling = false

	// Close the update channel when gpm is stopped
	close(gpm.update)

	return nil
}

func (gpm *GasPriceMonitor) fetchAndUpdateGasPrice(ctx context.Context) error {
	gasPrice, err := gpm.gpo.SuggestGasPrice(ctx)
	if err != nil {
		return err
	}

	gpm.updateGasPrice(gasPrice)

	if monitor.Enabled {
		monitor.SuggestedGasPrice(gasPrice)
	}

	glog.V(common.DEBUG).Infof("Cached gas price: %v", gasPrice)

	return nil
}

func (gpm *GasPriceMonitor) updateGasPrice(gasPrice *big.Int) {
	gpm.gasPriceMu.Lock()
	defer gpm.gasPriceMu.Unlock()

	gpm.gasPrice = gasPrice
}
<commit_msg>eth: Remove gas price logging<commit_after>package eth

import (
	"context"
	"errors"
	"math/big"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/livepeer/go-livepeer/monitor"
)

// GasPriceOracle defines methods for fetching a suggested gas price
// for submitting transactions
type GasPriceOracle interface {
	SuggestGasPrice(ctx context.Context) (*big.Int, error)
}

// GasPriceMonitor polls for gas price updates and updates its
// own view of the current gas price that can be used by others
type GasPriceMonitor struct {
	gpo GasPriceOracle
	// The following fields should be protected by `pollingMu`
	polling         bool
	pollingInterval time.Duration
	cancel          context.CancelFunc
	// pollingMu protects access to polling related fields
	pollingMu sync.Mutex

	// gasPriceMu protects access to gasPrice
	gasPriceMu sync.RWMutex
	// gasPrice is the current gas price to be returned to users
	gasPrice *big.Int

	// update is a channel used to send notifications to a listener
	// when the gas price is updated
	update chan struct{}
}

// NewGasPriceMonitor returns a GasPriceMonitor
func NewGasPriceMonitor(gpo GasPriceOracle, pollingInterval time.Duration) *GasPriceMonitor {
	return &GasPriceMonitor{
		gpo:             gpo,
		pollingInterval: pollingInterval,
		gasPrice:        big.NewInt(0),
	}
}

// GasPrice returns the current gas price
func (gpm *GasPriceMonitor) GasPrice() *big.Int {
	gpm.gasPriceMu.RLock()
	defer gpm.gasPriceMu.RUnlock()

	return gpm.gasPrice
}

// Start starts polling for gas price updates and returns a channel to receive
// notifications of gas price changes
func (gpm *GasPriceMonitor) Start(ctx context.Context) (chan struct{}, error) {
	gpm.pollingMu.Lock()
	defer gpm.pollingMu.Unlock()

	if gpm.polling {
		return nil, errors.New("already polling")
	}

	// Initialize gasPrice before starting to poll
	if err := gpm.fetchAndUpdateGasPrice(ctx); err != nil {
		return nil, err
	}

	gpm.update = make(chan struct{})

	cctx, cancel := context.WithCancel(ctx)
	gpm.cancel = cancel

	ticker := time.NewTicker(gpm.pollingInterval)

	go func(ctx context.Context) {
		for {
			select {
			case <-ticker.C:
				if err := gpm.fetchAndUpdateGasPrice(ctx); err != nil {
					glog.Errorf("error getting gas price: %v", err)
				}
			case <-ctx.Done():
				gpm.pollingMu.Lock()
				gpm.cancel = nil
				gpm.polling = false
				gpm.pollingMu.Unlock()
				return
			}
		}
	}(cctx)

	gpm.polling = true

	return gpm.update, nil
}

// Stop stops polling for gas price updates
func (gpm *GasPriceMonitor) Stop() error {
	gpm.pollingMu.Lock()
	defer gpm.pollingMu.Unlock()

	if !gpm.polling {
		return errors.New("not polling")
	}

	gpm.cancel()
	gpm.cancel = nil
	gpm.polling = false

	// Close the update channel when gpm is stopped
	close(gpm.update)

	return nil
}

func (gpm *GasPriceMonitor) fetchAndUpdateGasPrice(ctx context.Context) error {
	gasPrice, err := gpm.gpo.SuggestGasPrice(ctx)
	if err != nil {
		return err
	}

	gpm.updateGasPrice(gasPrice)

	if monitor.Enabled {
		monitor.SuggestedGasPrice(gasPrice)
	}

	return nil
}

func (gpm *GasPriceMonitor) updateGasPrice(gasPrice *big.Int) {
	gpm.gasPriceMu.Lock()
	defer gpm.gasPriceMu.Unlock()

	gpm.gasPrice = gasPrice
}
<|endoftext|>


<commit_before>package qga

import "sync"

var (
	storeMutext sync.Mutex
	store       = make(map[string]map[interface{}]interface{})
)

// StoreSet set variable k with value v to namespace ns
func StoreSet(ns string, k interface{}, v interface{}) {
	storeMutext.Lock()
	defer storeMutext.Unlock()
	m := make(map[interface{}]interface{})
	m[k] = v
	store[ns] = m
}

// StoreGet get variable k from namespace ns
func StoreGet(ns string, k interface{}) (interface{}, bool) {
	storeMutext.Lock()
	defer storeMutext.Unlock()
	m, ok := store[ns]
	if !ok {
		return nil, false
	}
	v, ok := m[k]
	return v, ok
}

// StoreDel del variable k from namespace ns
func StoreDel(ns string, k interface{}) {
	storeMutext.Lock()
	defer storeMutext.Unlock()
	m, ok := store[ns]
	if !ok {
		return
	}
	delete(m, k)
}
<commit_msg>fix for store<commit_after>package qga

import "sync"

var (
	storeMutext sync.Mutex
	store       = make(map[string]map[interface{}]interface{})
)

// StoreSet set variable k with value v to namespace ns
func StoreSet(ns string, k interface{}, v interface{}) {
	storeMutext.Lock()
	defer storeMutext.Unlock()
	m, ok := store[ns]
	if !ok {
		m = make(map[interface{}]interface{})
	}
	m[k] = v
	store[ns] = m
}

// StoreGet get variable k from namespace ns
func StoreGet(ns string, k interface{}) (interface{}, bool) {
	storeMutext.Lock()
	defer storeMutext.Unlock()
	m, ok := store[ns]
	if !ok {
		return nil, false
	}
	v, ok := m[k]
	return v, ok
}

// StoreDel del variable k from namespace ns
func StoreDel(ns string, k interface{}) {
	storeMutext.Lock()
	defer storeMutext.Unlock()
	m, ok := store[ns]
	if !ok {
		return
	}
	delete(m, k)
}
<|endoftext|>


<commit_before>package qh

import (
	"os"
	"time"
//	"fmt"
	"github.com/percona/percona-cloud-tools/agent"
	agentLog "github.com/percona/percona-cloud-tools/agent/log"
	mysqlLog "github.com/percona/percona-go-mysql/log"
	"github.com/percona/percona-go-mysql/log/parser"
)

type Worker struct {
	cc *agent.ControlChannels
	job *Job
	log *agentLog.LogWriter
}

type Job struct {
	SlowLogFile string
	Runtime time.Duration
	StartOffset uint64
	StopOffset uint64
	ExampleQueries bool
	Cc *agent.ControlChannels
	ResultChan chan *Result
	DoneChan chan *Worker
}

type Result struct {
	Error error `json:",omitempty"`
	Global *mysqlLog.GlobalClass `json:",omitempty"`
	Classes []*mysqlLog.QueryClass `json:",omitempty"`
}

func NewWorker(job *Job) *Worker {
	w := &Worker{
		job: job,
		log: agentLog.NewLogWriter(job.Cc.LogChan, "qh-worker"),
	}
	return w
}

func (w *Worker) Run() {
	// Whenever and however we return, send qh-manager our result and
	// tell it we're done so it frees our spot for another concurrent
	// worker.
	result := new(Result)
	defer func() {
		w.job.ResultChan <- result
		w.job.DoneChan <- w
	}()

	// Open the slow log file.
	file, err := os.Open(w.job.SlowLogFile)
	if err != nil {
		result.Error = err
		return
	}

	// Create a slow log parser and run it.  It sends events log events
	// via its channel.
	p := parser.NewSlowLogParser(file, false) // false=debug off
	if err != nil {
		result.Error = err
		return
	}
	go p.Run()

	// The global class has info and stats for all events.
	// Each query has its own class, defined by the checksum of its fingerprint.
	global := mysqlLog.NewGlobalClass()
	queries := make(map[string]*mysqlLog.QueryClass)
	for event := range p.EventChan {
		// Add the event to the global class.
		global.AddEvent(event)

		// Get the query class to which the event belongs.
		fingerprint := mysqlLog.Fingerprint(event.Query)
		classId := mysqlLog.Checksum(fingerprint)
		class, haveClass := queries[classId]
		if !haveClass {
			class = mysqlLog.NewQueryClass(classId, fingerprint)
			queries[classId] = class
		}

		// Add the event to its query class.
		class.AddEvent(event)
	}

	// Done parsing the slow log.  Finalize the global and query classes (calculate
	// averages, etc.).
	for _, class := range queries {
		class.Finalize()
	}
	global.Finalize(uint64(len(queries)))

	nQueries := len(queries)
	classes := make([]*mysqlLog.QueryClass, nQueries)
	for _, class := range queries {
		// Decr before use; can't classes[--nQueries] in Go.
		nQueries--
		classes[nQueries] = class
	}

	// Save the result.  It will be sent by defer when we return.
	result.Error = nil
	result.Global = global
	result.Classes = classes

	return
}
<commit_msg>Fix for failing test: stop processing events if event.Offset > w.job.StopOffset.<commit_after>package qh

import (
	"os"
	"time"
//	"fmt"
	"github.com/percona/percona-cloud-tools/agent"
	agentLog "github.com/percona/percona-cloud-tools/agent/log"
	mysqlLog "github.com/percona/percona-go-mysql/log"
	"github.com/percona/percona-go-mysql/log/parser"
)

type Worker struct {
	cc *agent.ControlChannels
	job *Job
	log *agentLog.LogWriter
}

type Job struct {
	SlowLogFile string
	Runtime time.Duration
	StartOffset uint64
	StopOffset uint64
	ExampleQueries bool
	Cc *agent.ControlChannels
	ResultChan chan *Result
	DoneChan chan *Worker
}

type Result struct {
	Error error `json:",omitempty"`
	Global *mysqlLog.GlobalClass `json:",omitempty"`
	Classes []*mysqlLog.QueryClass `json:",omitempty"`
}

func NewWorker(job *Job) *Worker {
	w := &Worker{
		job: job,
		log: agentLog.NewLogWriter(job.Cc.LogChan, "qh-worker"),
	}
	return w
}

func (w *Worker) Run() {
	// Whenever and however we return, send qh-manager our result and
	// tell it we're done so it frees our spot for another concurrent
	// worker.
	result := new(Result)
	defer func() {
		w.job.ResultChan <- result
		w.job.DoneChan <- w
	}()

	// Open the slow log file.
	file, err := os.Open(w.job.SlowLogFile)
	if err != nil {
		result.Error = err
		return
	}

	// Create a slow log parser and run it.  It sends events log events
	// via its channel.
	p := parser.NewSlowLogParser(file, false) // false=debug off
	if err != nil {
		result.Error = err
		return
	}
	go p.Run()

	// The global class has info and stats for all events.
	// Each query has its own class, defined by the checksum of its fingerprint.
	global := mysqlLog.NewGlobalClass()
	queries := make(map[string]*mysqlLog.QueryClass)
	for event := range p.EventChan {
		if event.Offset > w.job.StopOffset {
			break
		}

		// Add the event to the global class.
		global.AddEvent(event)

		// Get the query class to which the event belongs.
		fingerprint := mysqlLog.Fingerprint(event.Query)
		classId := mysqlLog.Checksum(fingerprint)
		class, haveClass := queries[classId]
		if !haveClass {
			class = mysqlLog.NewQueryClass(classId, fingerprint)
			queries[classId] = class
		}

		// Add the event to its query class.
		class.AddEvent(event)
	}

	// Done parsing the slow log.  Finalize the global and query classes (calculate
	// averages, etc.).
	for _, class := range queries {
		class.Finalize()
	}
	global.Finalize(uint64(len(queries)))

	nQueries := len(queries)
	classes := make([]*mysqlLog.QueryClass, nQueries)
	for _, class := range queries {
		// Decr before use; can't classes[--nQueries] in Go.
		nQueries--
		classes[nQueries] = class
	}

	// Save the result.  It will be sent by defer when we return.
	result.Error = nil
	result.Global = global
	result.Classes = classes

	return
}
<|endoftext|>


<commit_before>package router

import (
	"github.com/labstack/echo"
	"github.com/photoshelf/photoshelf-storage/infrastructure/container"
	"github.com/photoshelf/photoshelf-storage/presentation/controller"
)

func Load() (*echo.Echo, error) {
	e := echo.New()

	photoController := controller.New()
	container.Get(&photoController)

	g := e.Group("photos")
	g.GET("/:id", photoController.Get)
	g.POST("/", photoController.Post)
	g.PUT("/:id", photoController.Put)
	g.DELETE("/:id", photoController.Delete)

	return e, nil
}
<commit_msg>Add logger middleware<commit_after>package router

import (
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/photoshelf/photoshelf-storage/infrastructure/container"
	"github.com/photoshelf/photoshelf-storage/presentation/controller"
)

func Load() (*echo.Echo, error) {
	e := echo.New()

	photoController := controller.New()
	container.Get(&photoController)

	g := e.Group("photos")
	g.GET("/:id", photoController.Get)
	g.POST("/", photoController.Post)
	g.PUT("/:id", photoController.Put)
	g.DELETE("/:id", photoController.Delete)

	e.Use(middleware.Logger())

	return e, nil
}
<|endoftext|>


<commit_before>package fireauth

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"
)

const (
	// Version used for creating token
	Version = 0
	// TokenSep used as a delimiter for the token
	TokenSep = "."
)

// Firebase specific values for header
const (
	TokenAlgorithm = "HS256"
	TokenType      = "JWT"
)

var encodedHeader = encode([]byte(fmt.Sprintf(`{"alg": "%s", "typ": "%s"}`, TokenAlgorithm, TokenType)))

// Generic errors
var (
	ErrNoUIDKey           = errors.New(`Data payload must contain a "uid" key`)
	ErrUIDNotString       = errors.New(`Data payload key "uid" must be a string`)
	ErrUIDTooLong         = errors.New(`Data payload key "uid" must not be longer than 256 characters`)
	ErrEmptyDataNoOptions = errors.New("Data is empty and no options are set.  This token will have no effect on Firebase.")
	ErrTokenTooLong       = errors.New("Generated token is too long. The token cannot be longer than 1024 bytes.")
)

// Generator represents a token generator
type Generator struct {
	secret string
}

// Option represent the claims used when creating an authentication token
// https://www.firebase.com/docs/rest/guide/user-auth.html#section-rest-tokens-without-helpers
type Option struct {
	// NotBefote is the token "not before" date as a number of seconds since the Unix epoch.
	// If specified, the token will not be considered valid until after this date.
	NotBefore int64 `json:"nbf,omitempty"`

	// Expiration is the token expiration date as a number of seconds since the Unix epoch.
	// If not specified, by default the token will expire 24 hours after the "issued at" date (iat).
	Expiration int64 `json:"exp,omitempty"`

	// Admin when set to true to make this an "admin" token, which grants full read and
	// write access to all data.
	Admin bool `json:"admin,omitempty"`

	// Debug when set to true to enable debug mode, which provides verbose error messages
	// when Security and Firebase Rules fail.
	Debug bool `json:"debug,omitempty"`
}

// Data is used to create a token. The token data can contain any data of your choosing,
// however it must contain a `uid` key, which must be a string of less than 256 characters
type Data map[string]interface{}

// New creates a new Generator
func New(secret string) *Generator {
	return &Generator{
		secret: secret,
	}
}

// CreateToken generates a new token with the given Data and options
func (t *Generator) CreateToken(data Data, options *Option) (string, error) {
	// make sure we have valid parameters
	if data == nil && (options == nil || (!options.Admin && !options.Debug)) {
		return "", ErrEmptyDataNoOptions
	}

	// validate the data
	if err := validate(data, (options != nil && options.Admin)); err != nil {
		return "", err
	}

	// setup the claims for the token
	claim := struct {
		*Option
		Version  int   `json:"v"`
		Data     Data  `json:"d"`
		IssuedAt int64 `json:"iat"`
	}{
		Option:   options,
		Version:  Version,
		Data:     data,
		IssuedAt: time.Now().Unix(),
	}

	// generate the encoded claims
	claimBytes, err := json.Marshal(claim)
	if err != nil {
		return "", err
	}

	// create the token
	secureString := encodedHeader + TokenSep + encode(claimBytes)
	signature := sign(secureString, t.secret)
	token := secureString + TokenSep + signature

	if len(token) > 1024 {
		return "", ErrTokenTooLong
	}
	return token, nil
}

func validate(data Data, isAdmind bool) error {
	uid, containsID := data["uid"]
	if !containsID && !isAdmind {
		return ErrNoUIDKey
	}

	if _, isString := uid.(string); containsID && !isString {
		return ErrUIDNotString
	}

	if containsID && len(uid.(string)) > 256 {
		return ErrUIDTooLong
	}
	return nil
}

func encode(data []byte) string {
	return strings.Replace(base64.URLEncoding.EncodeToString(data), "=", "", -1)
}

func sign(message, secret string) string {
	return encode(hmac.New(sha256.New, []byte(secret)).Sum([]byte(message)))
}
<commit_msg>Fix the Options json tag<commit_after>package fireauth

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"
)

const (
	// Version used for creating token
	Version = 0
	// TokenSep used as a delimiter for the token
	TokenSep = "."
)

// Firebase specific values for header
const (
	TokenAlgorithm = "HS256"
	TokenType      = "JWT"
)

var encodedHeader = encode([]byte(fmt.Sprintf(`{"alg": "%s", "typ": "%s"}`, TokenAlgorithm, TokenType)))

// Generic errors
var (
	ErrNoUIDKey           = errors.New(`Data payload must contain a "uid" key`)
	ErrUIDNotString       = errors.New(`Data payload key "uid" must be a string`)
	ErrUIDTooLong         = errors.New(`Data payload key "uid" must not be longer than 256 characters`)
	ErrEmptyDataNoOptions = errors.New("Data is empty and no options are set.  This token will have no effect on Firebase.")
	ErrTokenTooLong       = errors.New("Generated token is too long. The token cannot be longer than 1024 bytes.")
)

// Generator represents a token generator
type Generator struct {
	secret string
}

// Option represent the claims used when creating an authentication token
// https://www.firebase.com/docs/rest/guide/user-auth.html#section-rest-tokens-without-helpers
type Option struct {
	// NotBefote is the token "not before" date as a number of seconds since the Unix epoch.
	// If specified, the token will not be considered valid until after this date.
	NotBefore int64 `json:"nbf,omitempty"`

	// Expiration is the token expiration date as a number of seconds since the Unix epoch.
	// If not specified, by default the token will expire 24 hours after the "issued at" date (iat).
	Expiration int64 `json:"exp,omitempty"`

	// Admin when set to true to make this an "admin" token, which grants full read and
	// write access to all data.
	Admin bool `json:"admin,omitempty"`

	// Debug when set to true to enable debug mode, which provides verbose error messages
	// when Security and Firebase Rules fail.
	Debug bool `json:"debug,omitempty"`
}

// Data is used to create a token. The token data can contain any data of your choosing,
// however it must contain a `uid` key, which must be a string of less than 256 characters
type Data map[string]interface{}

// New creates a new Generator
func New(secret string) *Generator {
	return &Generator{
		secret: secret,
	}
}

// CreateToken generates a new token with the given Data and options
func (t *Generator) CreateToken(data Data, options *Option) (string, error) {
	// make sure we have valid parameters
	if data == nil && (options == nil || (!options.Admin && !options.Debug)) {
		return "", ErrEmptyDataNoOptions
	}

	// validate the data
	if err := validate(data, (options != nil && options.Admin)); err != nil {
		return "", err
	}

	// setup the claims for the token
	claim := struct {
		*Option  `json:"Options,omitempty"`
		Version  int   `json:"v"`
		Data     Data  `json:"d"`
		IssuedAt int64 `json:"iat"`
	}{
		Option:   options,
		Version:  Version,
		Data:     data,
		IssuedAt: time.Now().Unix(),
	}

	// generate the encoded claims
	claimBytes, err := json.Marshal(claim)
	if err != nil {
		return "", err
	}

	// create the token
	secureString := encodedHeader + TokenSep + encode(claimBytes)
	signature := sign(secureString, t.secret)
	token := secureString + TokenSep + signature

	if len(token) > 1024 {
		return "", ErrTokenTooLong
	}
	return token, nil
}

func validate(data Data, isAdmind bool) error {
	uid, containsID := data["uid"]
	if !containsID && !isAdmind {
		return ErrNoUIDKey
	}

	if _, isString := uid.(string); containsID && !isString {
		return ErrUIDNotString
	}

	if containsID && len(uid.(string)) > 256 {
		return ErrUIDTooLong
	}
	return nil
}

func encode(data []byte) string {
	return strings.Replace(base64.URLEncoding.EncodeToString(data), "=", "", -1)
}

func sign(message, secret string) string {
	return encode(hmac.New(sha256.New, []byte(secret)).Sum([]byte(message)))
}
<|endoftext|>


<commit_before>package file

import (
	"bytes"
	"fmt"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"github.com/realglobe-Inc/go-lib-rg/rglog"
	"github.com/realglobe-Inc/go-lib-rg/run"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"time"
)

var log rglog.Logger

func init() {
	log = rglog.GetLogger("github.com/realglobe-Inc/go-lib-rg/file")
}

// ファイル・ディレクトリに関するユーティリティ。

const (
	dirPerm  = 0755
	filePerm = 0644
)

func IsExist(path string) (bool, error) {
	_, err := os.Lstat(path) // リンクも検知する。
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		} else {
			return false, erro.Wrap(err)
		}
	}
	return true, nil
}

func IsDir(path string) (bool, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, erro.Wrap(err)
	}
	return fi.IsDir(), nil
}

func Size(path string) (int64, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	return fi.Size(), nil
}

// /a/b/c から [ /a/b /a / ] をつくる
func DirList(path string) []string {
	list := []string{}
	for i := 0; i < 10; i++ {
		newPath := filepath.Dir(path)
		if newPath == path {
			break
		}
		path = newPath
		list = append(list, path)
	}
	return list
}

// ファイルの中身を丸ごと比較する。
func Compare(path1, path2 string) (int, error) {
	fi1, err := os.Stat(path1)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	fi2, err := os.Stat(path2)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	if fi1.Size() < fi2.Size() {
		return -1, nil
	} else if fi1.Size() > fi2.Size() {
		return 1, nil
	}

	// 読み込みつつ逐次比較するように改善できるが、めんどう。
	bytes1, err := ioutil.ReadFile(path1)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	bytes2, err := ioutil.ReadFile(path2)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	return bytes.Compare(bytes1, bytes2), nil
}

// ファイルまたはディレクトリの名前をてきとうに変更する。ディレクトリ間の移動はしない。
// 返り値は新しい名前。ディレクトリは含まない。
func Escape(path, suffix string) (newName string, err error) {
	name := filepath.Base(path)
	date := time.Now()
	tag := fmt.Sprintf("%04d%02d%02d%02d%02d%02d%09d", date.Year(), date.Month(), date.Day(), date.Hour(), date.Minute(), date.Second(), date.Nanosecond())
	newName = name + "." + tag + suffix
	newPath := filepath.Join(filepath.Dir(path), newName)

	if e := os.Rename(path, newPath); e != nil {
		return "", erro.Wrap(e)
	}

	log.Debug("mv ", path, " ", newPath)

	return newName, nil
}

// 引数の順番は io.Copy にならった。
func Copy(to, from string) error {
	buff, err := ioutil.ReadFile(from)
	if err != nil {
		return erro.Wrap(err)
	}

	// パーミッションも取得。
	fi, err := os.Stat(from)
	if err != nil {
		return erro.Wrap(err)
	}

	if e := os.MkdirAll(filepath.Dir(to), dirPerm); e != nil {
		return erro.Wrap(e)
	}

	if e := ioutil.WriteFile(to, buff, fi.Mode()); e != nil {
		return erro.Wrap(e)
	}

	log.Debug("cp ", from, " ", to)

	return nil
}

func RecursiveCopy(to, from string) error {
	if e := os.MkdirAll(filepath.Dir(to), dirPerm); e != nil {
		return erro.Wrap(e)
	}

	// ひどい手抜き。
	log.Debug("TE NU KI.")
	return erro.Wrap(run.Quiet("cp", "-r", from, to))
}

// ファイルの末尾に付け足す。
func Append(path string, data []byte) error {
	if e := os.MkdirAll(filepath.Dir(path), dirPerm); e != nil {
		return erro.Wrap(e)
	}

	writer, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, filePerm)
	if err != nil {
		return erro.Wrap(err)
	}
	defer writer.Close()

	stat, err := writer.Stat()
	if err != nil {
		return erro.Wrap(err)
	}

	if _, e := writer.WriteAt(data, stat.Size()); e != nil {
		return erro.Wrap(e)
	}

	return erro.Wrap(err)
}

// ファイルの末尾に行を付け足す。
func AppendLines(path string, lines []string) error {
	if e := os.MkdirAll(filepath.Dir(path), dirPerm); e != nil {
		return erro.Wrap(e)
	}

	writer, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, filePerm)
	if err != nil {
		return erro.Wrap(err)
	}
	defer writer.Close()

	stat, err := writer.Stat()
	if err != nil {
		return erro.Wrap(err)
	}

	buff := []byte{}
	if stat.Size() > 1 {
		// 改行で終わっていなければ改行する。
		tail := make([]byte, 1)
		if _, e := writer.ReadAt(tail, stat.Size()-1); e != nil {
			return erro.Wrap(e)
		} else if tail[0] != '\n' {
			buff = append(buff, '\n')
		}
	}
	for _, line := range lines {
		buff = append(buff, []byte(fmt.Sprintln(line))...)
	}

	if _, e := writer.WriteAt(buff, stat.Size()); e != nil {
		return erro.Wrap(e)
	}

	return erro.Wrap(err)
}

// path 以下を全部 owner のものにする。
func Chown(path, owner string) error {
	user, err := user.Lookup(owner)
	if err != nil {
		return erro.Wrap(err)
	}

	uid, err := strconv.ParseUint(user.Uid, 10, 32)
	if err != nil {
		return erro.Wrap(err)
	}

	return ChownById(path, int(uid))
}

func ChownById(path string, uid int) error {
	return ChownByIdGid(path, uid, uid)
}

func ChownByIdGid(path string, uid, gid int) error {
	if e := filepath.Walk(path, func(curPath string, info os.FileInfo, err error) error {
		if err != nil {
			return erro.Wrap(err)
		}

		if e := os.Chown(curPath, uid, gid); e != nil {
			return erro.Wrap(e)
		}

		return nil
	}); e != nil {
		return erro.Wrap(e)
	}

	return nil
}
<commit_msg>デバッグコードを追加<commit_after>package file

import (
	"bytes"
	"fmt"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"github.com/realglobe-Inc/go-lib-rg/rglog"
	"github.com/realglobe-Inc/go-lib-rg/run"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"time"
)

var log rglog.Logger

func init() {
	log = rglog.GetLogger("github.com/realglobe-Inc/go-lib-rg/file")
}

// ファイル・ディレクトリに関するユーティリティ。

const (
	dirPerm  = 0755
	filePerm = 0644
)

func IsExist(path string) (bool, error) {
	_, err := os.Lstat(path) // リンクも検知する。
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		} else {
			return false, erro.Wrap(err)
		}
	}
	return true, nil
}

func IsDir(path string) (bool, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, erro.Wrap(err)
	}
	return fi.IsDir(), nil
}

func Size(path string) (int64, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	return fi.Size(), nil
}

// /a/b/c から [ /a/b /a / ] をつくる
func DirList(path string) []string {
	list := []string{}
	for i := 0; i < 10; i++ {
		newPath := filepath.Dir(path)
		if newPath == path {
			break
		}
		path = newPath
		list = append(list, path)
	}
	return list
}

// ファイルの中身を丸ごと比較する。
func Compare(path1, path2 string) (int, error) {
	fi1, err := os.Stat(path1)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	fi2, err := os.Stat(path2)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	if fi1.Size() < fi2.Size() {
		return -1, nil
	} else if fi1.Size() > fi2.Size() {
		return 1, nil
	}

	// 読み込みつつ逐次比較するように改善できるが、めんどう。
	bytes1, err := ioutil.ReadFile(path1)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	bytes2, err := ioutil.ReadFile(path2)
	if err != nil {
		return 0, erro.Wrap(err)
	}
	return bytes.Compare(bytes1, bytes2), nil
}

// ファイルまたはディレクトリの名前をてきとうに変更する。ディレクトリ間の移動はしない。
// 返り値は新しい名前。ディレクトリは含まない。
func Escape(path, suffix string) (newName string, err error) {
	name := filepath.Base(path)
	date := time.Now()
	tag := fmt.Sprintf("%04d%02d%02d%02d%02d%02d%09d", date.Year(), date.Month(), date.Day(), date.Hour(), date.Minute(), date.Second(), date.Nanosecond())
	newName = name + "." + tag + suffix
	newPath := filepath.Join(filepath.Dir(path), newName)

	if e := os.Rename(path, newPath); e != nil {
		return "", erro.Wrap(e)
	}

	log.Debug("mv ", path, " ", newPath)

	return newName, nil
}

// 引数の順番は io.Copy にならった。
func Copy(to, from string) error {
	buff, err := ioutil.ReadFile(from)
	if err != nil {
		return erro.Wrap(err)
	}

	// パーミッションも取得。
	fi, err := os.Stat(from)
	if err != nil {
		return erro.Wrap(err)
	}

	if e := os.MkdirAll(filepath.Dir(to), dirPerm); e != nil {
		return erro.Wrap(e)
	}

	if e := ioutil.WriteFile(to, buff, fi.Mode()); e != nil {
		return erro.Wrap(e)
	}

	log.Debug("cp ", from, " ", to)

	return nil
}

func RecursiveCopy(to, from string) error {
	if e := os.MkdirAll(filepath.Dir(to), dirPerm); e != nil {
		return erro.Wrap(e)
	}

	// ひどい手抜き。
	log.Debug("TE NU KI.")
	return erro.Wrap(run.Quiet("cp", "-r", from, to))
}

// ファイルの末尾に付け足す。
func Append(path string, data []byte) error {
	if e := os.MkdirAll(filepath.Dir(path), dirPerm); e != nil {
		return erro.Wrap(e)
	}

	writer, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, filePerm)
	if err != nil {
		return erro.Wrap(err)
	}
	defer writer.Close()

	stat, err := writer.Stat()
	if err != nil {
		return erro.Wrap(err)
	}

	if _, e := writer.WriteAt(data, stat.Size()); e != nil {
		return erro.Wrap(e)
	}

	return erro.Wrap(err)
}

// ファイルの末尾に行を付け足す。
func AppendLines(path string, lines []string) error {
	if e := os.MkdirAll(filepath.Dir(path), dirPerm); e != nil {
		return erro.Wrap(e)
	}

	writer, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, filePerm)
	if err != nil {
		return erro.Wrap(err)
	}
	defer writer.Close()

	stat, err := writer.Stat()
	if err != nil {
		return erro.Wrap(err)
	}

	buff := []byte{}
	if stat.Size() > 1 {
		// 改行で終わっていなければ改行する。
		tail := make([]byte, 1)
		if _, e := writer.ReadAt(tail, stat.Size()-1); e != nil {
			return erro.Wrap(e)
		} else if tail[0] != '\n' {
			buff = append(buff, '\n')
		}
	}
	for _, line := range lines {
		buff = append(buff, []byte(fmt.Sprintln(line))...)
	}

	if _, e := writer.WriteAt(buff, stat.Size()); e != nil {
		return erro.Wrap(e)
	}

	return erro.Wrap(err)
}

// path 以下を全部 owner のものにする。
func Chown(path, owner string) error {
	user, err := user.Lookup(owner)
	if err != nil {
		return erro.Wrap(err)
	}

	uid, err := strconv.ParseUint(user.Uid, 10, 32)
	if err != nil {
		return erro.Wrap(err)
	}

	return ChownById(path, int(uid))
}

func ChownById(path string, uid int) error {
	return ChownByIdGid(path, uid, uid)
}

func ChownByIdGid(path string, uid, gid int) error {
	if e := filepath.Walk(path, func(curPath string, info os.FileInfo, err error) error {
		if err != nil {
			return erro.Wrap(err)
		}

		if e := os.Chown(curPath, uid, gid); e != nil {
			return erro.Wrap(e)
		}

		return nil
	}); e != nil {
		return erro.Wrap(e)
	}

	log.Debug("chown ", uid, ":", gid, " -R ", path)

	return nil
}
<|endoftext|>


<commit_before>package triegun

import (
	"errors"
	"fmt"
	"io"
	"os"
	"regexp"
)
import "text/template"

var ErrEmpty = errors.New("triegun can't construct trie with empty string set")

type Plant struct {
	PkgName        string
	TagName        string
	DisableMatcher bool
	DisablePrefix  bool
	words          []string
}

func New() *Plant {
	return &Plant{
		PkgName: "main",
		words:   make([]string, 0),
	}
}

func (p *Plant) AddBytes(bytes ...[]byte) {
	for _, b := range bytes {
		p.words = append(p.words, string(b))
	}
}

func (p *Plant) AddString(strs ...string) {
	for _, s := range strs {
		p.words = append(p.words, s)
	}
}

func (p *Plant) GenFile(file string) (err error) {
	out, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer func() {
		out.Close()
		if e := recover(); e != nil {
			var ok bool
			if err, ok = e.(error); !ok {
				panic(e)
			}
		}
		if err != nil {
			os.Remove(file)
		}
	}()
	err = p.Gen(out)
	return
}

var reId = regexp.MustCompile(`^[0-9a-zA-Z_]+$`)

func (p *Plant) Gen(w io.Writer) error {
	if len(p.words) < 1 {
		return ErrEmpty
	}
	if !reId.MatchString(p.PkgName) {
		return fmt.Errorf(`package name must be an identifier, but "%s" is not`, p.PkgName)
	}
	if p.TagName != "" && !reId.MatchString(p.TagName) {
		return fmt.Errorf(`tag name must be an identifier, but "%s" is not`, p.TagName)
	}
	dfa := newDFAFromStrings(p.words)
	if err := p.preamble(w); err != nil {
		return err
	}
	if !p.DisablePrefix {
		if err := p.genHasPrefix(w, dfa); err != nil {
			return err
		}
	}
	if !p.DisableMatcher {
		if err := p.genMatcher(w, dfa); err != nil {
			return err
		}
	}
	return nil
}

func (g *Plant) preamble(w io.Writer) error {
	return tmplPreabmle.Execute(w, g)
}

var tmplPreabmle = template.Must(template.New("preamble").Parse(`
// DO NOT EDIT!
// Code generated by go-triegun <https://github.com/Maki-Daisuke/go-triegun>
// DO NOT EDIT!
package {{ .PkgName }}

import "unsafe"
`))
<commit_msg>Change member names<commit_after>package triegun

import (
	"errors"
	"fmt"
	"io"
	"os"
	"regexp"
)
import "text/template"

var ErrEmpty = errors.New("triegun can't construct trie with empty string set")

type Plant struct {
	PkgName          string
	TagName          string
	DisableMatch     bool
	DisableHasPrefix bool
	words            []string
}

func New() *Plant {
	return &Plant{
		PkgName: "main",
		words:   make([]string, 0),
	}
}

func (p *Plant) AddBytes(bytes ...[]byte) {
	for _, b := range bytes {
		p.words = append(p.words, string(b))
	}
}

func (p *Plant) AddString(strs ...string) {
	for _, s := range strs {
		p.words = append(p.words, s)
	}
}

func (p *Plant) GenFile(file string) (err error) {
	out, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer func() {
		out.Close()
		if e := recover(); e != nil {
			var ok bool
			if err, ok = e.(error); !ok {
				panic(e)
			}
		}
		if err != nil {
			os.Remove(file)
		}
	}()
	err = p.Gen(out)
	return
}

var reId = regexp.MustCompile(`^[0-9a-zA-Z_]+$`)

func (p *Plant) Gen(w io.Writer) error {
	if len(p.words) < 1 {
		return ErrEmpty
	}
	if !reId.MatchString(p.PkgName) {
		return fmt.Errorf(`package name must be an identifier, but "%s" is not`, p.PkgName)
	}
	if p.TagName != "" && !reId.MatchString(p.TagName) {
		return fmt.Errorf(`tag name must be an identifier, but "%s" is not`, p.TagName)
	}
	dfa := newDFAFromStrings(p.words)
	if err := p.preamble(w); err != nil {
		return err
	}
	if !p.DisableHasPrefix {
		if err := p.genHasPrefix(w, dfa); err != nil {
			return err
		}
	}
	if !p.DisableMatch {
		if err := p.genMatcher(w, dfa); err != nil {
			return err
		}
	}
	return nil
}

func (g *Plant) preamble(w io.Writer) error {
	return tmplPreabmle.Execute(w, g)
}

var tmplPreabmle = template.Must(template.New("preamble").Parse(`
// DO NOT EDIT!
// Code generated by go-triegun <https://github.com/Maki-Daisuke/go-triegun>
// DO NOT EDIT!
package {{ .PkgName }}

import "unsafe"
`))
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ct

import (
	"crypto/sha256"
	"errors"
	"fmt"
	"time"

	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/tls"
	"github.com/google/certificate-transparency/go/x509"
	"github.com/google/trillian/crypto"
	"github.com/google/trillian/crypto/keys"
)

// signV1TreeHead signs a tree head for CT. The input STH should have been built from a
// backend response and already checked for validity.
func signV1TreeHead(signer *crypto.Signer, sth *ct.SignedTreeHead) error {
	sthBytes, err := ct.SerializeSTHSignatureInput(*sth)
	if err != nil {
		return err
	}

	signature, err := signer.Sign(sthBytes)
	if err != nil {
		return err
	}

	sth.TreeHeadSignature = ct.DigitallySigned{
		Algorithm: tls.SignatureAndHashAlgorithm{
			Hash: tls.SHA256,
			// This relies on the protobuf enum values matching the TLS-defined values.
			Signature: tls.SignatureAlgorithm(keys.SignatureAlgorithm(signer.Public())),
		},
		Signature: signature.Signature,
	}
	return nil
}

// signV1SCTForCertificate creates a MerkleTreeLeaf and builds and signs a V1 CT SCT for a certificate
// using the key held by a key manager.
func signV1SCTForCertificate(signer *crypto.Signer, cert, issuer *x509.Certificate, t time.Time) (*ct.MerkleTreeLeaf, *ct.SignedCertificateTimestamp, error) {
	// Temp SCT for input to the serializer
	sctInput := getSCTForSignatureInput(t)

	// Build up a MerkleTreeLeaf for the cert
	leaf := ct.MerkleTreeLeaf{
		Version:  ct.V1,
		LeafType: ct.TimestampedEntryLeafType,
		TimestampedEntry: &ct.TimestampedEntry{
			Timestamp: sctInput.Timestamp,
			EntryType: ct.X509LogEntryType,
			X509Entry: &ct.ASN1Cert{Data: cert.Raw},
		},
	}

	return serializeAndSignSCT(signer, leaf, sctInput, t)
}

// signV1SCTForPrecertificate builds and signs a V1 CT SCT for a pre-certificate using the key
// held by a key manager.
func signV1SCTForPrecertificate(signer *crypto.Signer, cert, issuer *x509.Certificate, t time.Time) (*ct.MerkleTreeLeaf, *ct.SignedCertificateTimestamp, error) {
	if issuer == nil {
		// Need issuer for the IssuerKeyHash
		return nil, nil, errors.New("no issuer available for pre-certificate")
	}
	// Temp SCT for input to the serializer
	sctInput := getSCTForSignatureInput(t)

	// Build up a LogEntry for the precert
	// For precerts we need to extract the relevant data from the Certificate container,
	// specifically the DER-encoded TBSCertificate, but with the CT poison extension removed.
	// (This is only possible using the CT specific modified version of the x509 library.)
	keyHash := sha256.Sum256(issuer.RawSubjectPublicKeyInfo)
	defangedTBS, err := x509.RemoveCTPoison(cert.RawTBSCertificate)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to remove poison extension: %v", err)
	}
	precert := ct.PreCert{
		IssuerKeyHash:  keyHash,
		TBSCertificate: defangedTBS,
	}

	timestampedEntry := ct.TimestampedEntry{
		Timestamp:    sctInput.Timestamp,
		EntryType:    ct.PrecertLogEntryType,
		PrecertEntry: &precert,
	}
	leaf := ct.MerkleTreeLeaf{
		Version:          ct.V1,
		LeafType:         ct.TimestampedEntryLeafType,
		TimestampedEntry: &timestampedEntry,
	}

	return serializeAndSignSCT(signer, leaf, sctInput, t)
}

func serializeAndSignSCT(signer *crypto.Signer, leaf ct.MerkleTreeLeaf, sctInput ct.SignedCertificateTimestamp, t time.Time) (*ct.MerkleTreeLeaf, *ct.SignedCertificateTimestamp, error) {
	// Serialize SCT signature input to get the bytes that need to be signed
	res, err := ct.SerializeSCTSignatureInput(sctInput, ct.LogEntry{Leaf: leaf})
	if err != nil {
		return nil, nil, fmt.Errorf("failed to serialize SCT data: %v", err)
	}

	// Create a complete SCT including signature
	sct, err := signSCT(signer, t, res)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to sign SCT data: %v", err)
	}

	return &leaf, sct, nil
}

func signSCT(signer *crypto.Signer, t time.Time, sctData []byte) (*ct.SignedCertificateTimestamp, error) {
	signature, err := signer.Sign(sctData)
	if err != nil {
		return nil, fmt.Errorf("failed to sign data: %v", err)
	}

	digitallySigned := ct.DigitallySigned{
		Algorithm: tls.SignatureAndHashAlgorithm{
			Hash: tls.SHA256,
			// This relies on the protobuf enum values matching the TLS-defined values.
			Signature: tls.SignatureAlgorithm(keys.SignatureAlgorithm(signer.Public())),
		},
		Signature: signature.Signature,
	}

	logID, err := GetCTLogID(signer.Public())
	if err != nil {
		return nil, fmt.Errorf("failed to get logID: %v", err)
	}

	return &ct.SignedCertificateTimestamp{
		SCTVersion: ct.V1,
		LogID:      ct.LogID{KeyID: logID},
		Timestamp:  uint64(t.UnixNano() / millisPerNano), // spec uses millisecond timestamps
		Extensions: ct.CTExtensions{},
		Signature:  digitallySigned,
	}, nil
}

func getSCTForSignatureInput(t time.Time) ct.SignedCertificateTimestamp {
	return ct.SignedCertificateTimestamp{
		SCTVersion: ct.V1,
		Timestamp:  uint64(t.UnixNano() / millisPerNano), // spec uses millisecond timestamps
		Extensions: ct.CTExtensions{}}
}
<commit_msg>examples/ct: drop getSCTForSignatureInput()<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ct

import (
	"crypto/sha256"
	"errors"
	"fmt"
	"time"

	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/tls"
	"github.com/google/certificate-transparency/go/x509"
	"github.com/google/trillian/crypto"
	"github.com/google/trillian/crypto/keys"
)

// signV1TreeHead signs a tree head for CT. The input STH should have been built from a
// backend response and already checked for validity.
func signV1TreeHead(signer *crypto.Signer, sth *ct.SignedTreeHead) error {
	sthBytes, err := ct.SerializeSTHSignatureInput(*sth)
	if err != nil {
		return err
	}

	signature, err := signer.Sign(sthBytes)
	if err != nil {
		return err
	}

	sth.TreeHeadSignature = ct.DigitallySigned{
		Algorithm: tls.SignatureAndHashAlgorithm{
			Hash: tls.SHA256,
			// This relies on the protobuf enum values matching the TLS-defined values.
			Signature: tls.SignatureAlgorithm(keys.SignatureAlgorithm(signer.Public())),
		},
		Signature: signature.Signature,
	}
	return nil
}

// signV1SCTForCertificate creates a MerkleTreeLeaf and builds and signs a V1 CT SCT for a certificate
// using the key held by a key manager.
func signV1SCTForCertificate(signer *crypto.Signer, cert, issuer *x509.Certificate, t time.Time) (*ct.MerkleTreeLeaf, *ct.SignedCertificateTimestamp, error) {
	sctInput := ct.SignedCertificateTimestamp{
		SCTVersion: ct.V1,
		Timestamp:  uint64(t.UnixNano() / millisPerNano),
		Extensions: ct.CTExtensions{},
	}

	// Build up a MerkleTreeLeaf for the cert
	leaf := ct.MerkleTreeLeaf{
		Version:  ct.V1,
		LeafType: ct.TimestampedEntryLeafType,
		TimestampedEntry: &ct.TimestampedEntry{
			Timestamp: sctInput.Timestamp,
			EntryType: ct.X509LogEntryType,
			X509Entry: &ct.ASN1Cert{Data: cert.Raw},
		},
	}

	return serializeAndSignSCT(signer, leaf, sctInput, t)
}

// signV1SCTForPrecertificate builds and signs a V1 CT SCT for a pre-certificate using the key
// held by a key manager.
func signV1SCTForPrecertificate(signer *crypto.Signer, cert, issuer *x509.Certificate, t time.Time) (*ct.MerkleTreeLeaf, *ct.SignedCertificateTimestamp, error) {
	if issuer == nil {
		// Need issuer for the IssuerKeyHash
		return nil, nil, errors.New("no issuer available for pre-certificate")
	}
	sctInput := ct.SignedCertificateTimestamp{
		SCTVersion: ct.V1,
		Timestamp:  uint64(t.UnixNano() / millisPerNano),
		Extensions: ct.CTExtensions{},
	}

	// Build up a LogEntry for the precert
	// For precerts we need to extract the relevant data from the Certificate container,
	// specifically the DER-encoded TBSCertificate, but with the CT poison extension removed.
	// (This is only possible using the CT specific modified version of the x509 library.)
	keyHash := sha256.Sum256(issuer.RawSubjectPublicKeyInfo)
	defangedTBS, err := x509.RemoveCTPoison(cert.RawTBSCertificate)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to remove poison extension: %v", err)
	}
	precert := ct.PreCert{
		IssuerKeyHash:  keyHash,
		TBSCertificate: defangedTBS,
	}

	timestampedEntry := ct.TimestampedEntry{
		Timestamp:    sctInput.Timestamp,
		EntryType:    ct.PrecertLogEntryType,
		PrecertEntry: &precert,
	}
	leaf := ct.MerkleTreeLeaf{
		Version:          ct.V1,
		LeafType:         ct.TimestampedEntryLeafType,
		TimestampedEntry: &timestampedEntry,
	}

	return serializeAndSignSCT(signer, leaf, sctInput, t)
}

func serializeAndSignSCT(signer *crypto.Signer, leaf ct.MerkleTreeLeaf, sctInput ct.SignedCertificateTimestamp, t time.Time) (*ct.MerkleTreeLeaf, *ct.SignedCertificateTimestamp, error) {
	// Serialize SCT signature input to get the bytes that need to be signed
	res, err := ct.SerializeSCTSignatureInput(sctInput, ct.LogEntry{Leaf: leaf})
	if err != nil {
		return nil, nil, fmt.Errorf("failed to serialize SCT data: %v", err)
	}

	// Create a complete SCT including signature
	sct, err := signSCT(signer, t, res)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to sign SCT data: %v", err)
	}

	return &leaf, sct, nil
}

func signSCT(signer *crypto.Signer, t time.Time, sctData []byte) (*ct.SignedCertificateTimestamp, error) {
	signature, err := signer.Sign(sctData)
	if err != nil {
		return nil, fmt.Errorf("failed to sign data: %v", err)
	}

	digitallySigned := ct.DigitallySigned{
		Algorithm: tls.SignatureAndHashAlgorithm{
			Hash: tls.SHA256,
			// This relies on the protobuf enum values matching the TLS-defined values.
			Signature: tls.SignatureAlgorithm(keys.SignatureAlgorithm(signer.Public())),
		},
		Signature: signature.Signature,
	}

	logID, err := GetCTLogID(signer.Public())
	if err != nil {
		return nil, fmt.Errorf("failed to get logID: %v", err)
	}

	return &ct.SignedCertificateTimestamp{
		SCTVersion: ct.V1,
		LogID:      ct.LogID{KeyID: logID},
		Timestamp:  uint64(t.UnixNano() / millisPerNano), // spec uses millisecond timestamps
		Extensions: ct.CTExtensions{},
		Signature:  digitallySigned,
	}, nil
}
<|endoftext|>