<commit_before>package main

import (
	"errors"
	"log"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"

	"github.com/stripe/sequins/blocks"
	"github.com/stripe/sequins/sharding"
)

const versionHeader = "X-Sequins-Version"

var (
	errNoAvailablePeers   = errors.New("no available peers")
	errProxiedIncorrectly = errors.New("this server doesn't have the requested partition")
	errMixedFiles         = errors.New("Mixed Sparkey and Sequencefile directory")
)

// A version represents a single version of a particular sequins db: in
// other words, a collection of files. In the sharding-enabled case, it
// understands distribution of partitions and can route requests.
type version struct {
	sequins *sequins
	db      *db

	path          string
	name          string
	blockStore    *blocks.BlockStore
	partitions    *sharding.Partitions
	numPartitions int
	files         []string

	state     versionState
	created   time.Time
	available time.Time
	stateLock sync.RWMutex

	ready     chan bool
	cancel    chan bool
	built     bool
	buildLock sync.Mutex

	stats *statsd.Client
}

func newVersion(sequins *sequins, db *db, path, name string) (*version, error) {
	files, err := sequins.backend.ListFiles(db.name, name)
	if err != nil {
		return nil, err
	}
	files, numPartitions, err := filterVersionFiles(files)
	if err != nil {
		return nil, err
	}

	vs := &version{
		sequins:       sequins,
		db:            db,
		path:          path,
		name:          name,
		files:         files,
		numPartitions: numPartitions,

		created: time.Now(),
		state:   versionBuilding,

		ready:  make(chan bool),
		cancel: make(chan bool),

		stats: sequins.stats,
	}

	minReplication := 1
	if sequins.config.Sharding.Enabled {
		minReplication = sequins.config.Sharding.MinReplication
	}

	vs.partitions = sharding.WatchPartitions(sequins.zkWatcher, sequins.peers,
		db.name, name, vs.numPartitions, sequins.config.Sharding.Replication, minReplication)

	err = vs.initBlockStore(path)
	if err != nil {
		return nil, err
	}

	// If we're running in non-distributed mode, ready gets closed once the block
	// store is built.
	if vs.partitions != nil {
		select {
		case <-vs.partitions.Ready:
			close(vs.ready)
		default:
			go func() {
				select {
				case <-vs.cancel:
				case <-vs.partitions.Ready:
				}

				close(vs.ready)
			}()
		}
	}

	return vs, nil
}

func (vs *version) initBlockStore(path string) error {
	// Try loading anything we have locally. If it doesn't work out, that's ok.
	blockStore, manifest, err := blocks.NewFromManifest(path)
	if err != nil && err != blocks.ErrNoManifest {
		log.Println("Error loading", vs.db.name, "version", vs.name, "from manifest:", err)
	}

	if blockStore == nil {
		blockStore = blocks.New(vs.path, vs.numPartitions,
			vs.sequins.config.Storage.Compression, vs.sequins.config.Storage.BlockSize)
	} else {
		have := make(map[int]bool)
		for _, partition := range manifest.SelectedPartitions {
			have[partition] = true
		}

		vs.partitions.UpdateLocal(have)
	}

	vs.blockStore = blockStore
	return nil
}

func (vs *version) close() {
	close(vs.cancel)

	// This happens once the building goroutine gets the cancel and exits.
	go func() {
		vs.buildLock.Lock()
		defer vs.buildLock.Unlock()

		vs.partitions.Close()
		vs.blockStore.Close()
	}()
}

func (vs *version) delete() error {
	return vs.blockStore.Delete()
}

var reSparkeyPart = regexp.MustCompile(`\d+`)

// Check if a file is a sparkey log file. If so, return (true, partition),
// where partition is the file's partition. See addSparkeyFile().
func isSparkeyFile(file string) (raw bool, partition int) {
	if !strings.HasSuffix(file, ".spl") {
		return false, 0
	}

	match := reSparkeyPart.FindString(file)
	if match == "" {
		return false, 0
	}
	part, err := strconv.Atoi(match)
	if err != nil {
		return false, 0
	}

	return true, part
}

// Identify auxiliary files, that should not be downloaded.
func isAuxiliaryFile(file string) bool {
	// Skip sparkey index files, we'll fetch them when we fetch the corresponding sparkey log file.
	return strings.HasSuffix(file, ".spi.sz")
}

// Filter files, yielding only non-auxiliary files. Also yield the number of partitions found.
func filterVersionFiles(files []string) (filtered []string, numPartitions int, err error) {
	filtered = []string{}
	partitions := map[int]bool{}
	sparkeyStatus := map[bool]bool{}

	for _, file := range files {
		if isAuxiliaryFile(file) {
			continue
		}
		filtered = append(filtered, file)

		isSparkey, partition := isSparkeyFile(file)
		sparkeyStatus[isSparkey] = true
		if len(sparkeyStatus) > 1 {
			return nil, 0, errMixedFiles
		}

		if isSparkey {
			partitions[partition] = true
		} else {
			partitions[len(partitions)] = true
		}
	}

	return filtered, len(partitions), nil
}
<commit_msg>If we have a manifest, don't recalculate what to fetch<commit_after>package main

import (
	"errors"
	"log"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"

	"github.com/stripe/sequins/blocks"
	"github.com/stripe/sequins/sharding"
)

const versionHeader = "X-Sequins-Version"

var (
	errNoAvailablePeers   = errors.New("no available peers")
	errProxiedIncorrectly = errors.New("this server doesn't have the requested partition")
	errMixedFiles         = errors.New("Mixed Sparkey and Sequencefile directory")
)

// A version represents a single version of a particular sequins db: in
// other words, a collection of files. In the sharding-enabled case, it
// understands distribution of partitions and can route requests.
type version struct {
	sequins *sequins
	db      *db

	path          string
	name          string
	blockStore    *blocks.BlockStore
	partitions    *sharding.Partitions
	numPartitions int
	files         []string

	state     versionState
	created   time.Time
	available time.Time
	stateLock sync.RWMutex

	ready     chan bool
	cancel    chan bool
	built     bool
	buildLock sync.Mutex

	stats *statsd.Client
}

func newVersion(sequins *sequins, db *db, path, name string) (*version, error) {
	files, err := sequins.backend.ListFiles(db.name, name)
	if err != nil {
		return nil, err
	}
	files, numPartitions, err := filterVersionFiles(files)
	if err != nil {
		return nil, err
	}

	vs := &version{
		sequins:       sequins,
		db:            db,
		path:          path,
		name:          name,
		files:         files,
		numPartitions: numPartitions,

		created: time.Now(),
		state:   versionBuilding,

		ready:  make(chan bool),
		cancel: make(chan bool),

		stats: sequins.stats,
	}

	minReplication := 1
	if sequins.config.Sharding.Enabled {
		minReplication = sequins.config.Sharding.MinReplication
	}

	vs.partitions = sharding.WatchPartitions(sequins.zkWatcher, sequins.peers,
		db.name, name, vs.numPartitions, sequins.config.Sharding.Replication, minReplication)

	err = vs.initBlockStore(path)
	if err != nil {
		return nil, err
	}

	// If we're running in non-distributed mode, ready gets closed once the block
	// store is built.
	if vs.partitions != nil {
		select {
		case <-vs.partitions.Ready:
			close(vs.ready)
		default:
			go func() {
				select {
				case <-vs.cancel:
				case <-vs.partitions.Ready:
				}

				close(vs.ready)
			}()
		}
	}

	return vs, nil
}

func (vs *version) initBlockStore(path string) error {
	// Try loading anything we have locally. If it doesn't work out, that's ok.
	blockStore, manifest, err := blocks.NewFromManifest(path)
	if err != nil && err != blocks.ErrNoManifest {
		log.Println("Error loading", vs.db.name, "version", vs.name, "from manifest:", err)
	}

	if blockStore == nil {
		blockStore = blocks.New(vs.path, vs.numPartitions,
			vs.sequins.config.Storage.Compression, vs.sequins.config.Storage.BlockSize)
	} else {
		have := make(map[int]bool)
		for _, partition := range manifest.SelectedPartitions {
			have[partition] = true
		}

		vs.partitions.UpdateLocal(have)

		// Assume that if we have a manifest, we have successfully fetched this version at some point.
		// We don't want to re-calculate partition assignment, but just use what the manifest told us.
		vs.built = true
	}

	vs.blockStore = blockStore
	return nil
}

func (vs *version) close() {
	close(vs.cancel)

	// This happens once the building goroutine gets the cancel and exits.
	go func() {
		vs.buildLock.Lock()
		defer vs.buildLock.Unlock()

		vs.partitions.Close()
		vs.blockStore.Close()
	}()
}

func (vs *version) delete() error {
	return vs.blockStore.Delete()
}

var reSparkeyPart = regexp.MustCompile(`\d+`)

// Check if a file is a sparkey log file. If so, return (true, partition),
// where partition is the file's partition. See addSparkeyFile().
func isSparkeyFile(file string) (raw bool, partition int) {
	if !strings.HasSuffix(file, ".spl") {
		return false, 0
	}

	match := reSparkeyPart.FindString(file)
	if match == "" {
		return false, 0
	}
	part, err := strconv.Atoi(match)
	if err != nil {
		return false, 0
	}

	return true, part
}

// Identify auxiliary files, that should not be downloaded.
func isAuxiliaryFile(file string) bool {
	// Skip sparkey index files, we'll fetch them when we fetch the corresponding sparkey log file.
	return strings.HasSuffix(file, ".spi.sz")
}

// Filter files, yielding only non-auxiliary files. Also yield the number of partitions found.
func filterVersionFiles(files []string) (filtered []string, numPartitions int, err error) {
	filtered = []string{}
	partitions := map[int]bool{}
	sparkeyStatus := map[bool]bool{}

	for _, file := range files {
		if isAuxiliaryFile(file) {
			continue
		}
		filtered = append(filtered, file)

		isSparkey, partition := isSparkeyFile(file)
		sparkeyStatus[isSparkey] = true
		if len(sparkeyStatus) > 1 {
			return nil, 0, errMixedFiles
		}

		if isSparkey {
			partitions[partition] = true
		} else {
			partitions[len(partitions)] = true
		}
	}

	return filtered, len(partitions), nil
}
<|endoftext|>


<commit_before>// Licensed to The Moov Authors under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. The Moov Authors licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package ach

// Version Number
const Version = "v1.12.1"
<commit_msg>release v1.12.2<commit_after>// Licensed to The Moov Authors under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. The Moov Authors licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package ach

// Version Number
const Version = "v1.12.2"
<|endoftext|>


<commit_before>package gf

const VERSION  = "v1.5.3"
const AUTHORS  = "john<john@goframe.org>"

<commit_msg>version updates<commit_after>package gf

const VERSION  = "v1.5.4"
const AUTHORS  = "john<john@goframe.org>"

<|endoftext|>


<commit_before>package main

// App version variables

// Version set in source code
const Version = "0.28"

// Build time filled by make on program build
var Build string

// Commit tag from git, filled in by the compiler.
var Commit string
<commit_msg>v0.29<commit_after>package main

// App version variables

// Version set in source code
const Version = "0.29"

// Build time filled by make on program build
var Build string

// Commit tag from git, filled in by the compiler.
var Commit string
<|endoftext|>


<commit_before>package main

const Version string = "0.2.0"
<commit_msg>v0.3.0<commit_after>package main

const Version string = "0.3.0"
<|endoftext|>


<commit_before>// watches the current directory for changes and runs the specificed program on change

package main

import (
	"fmt"
	"github.com/howeyc/fsnotify"
	"log"
	"os"
	"os/exec"
)

var help = `watcher [command to execute]`

func main() {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}
	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, help)
		os.Exit(1)
	}
	cmd, args := os.Args[1], os.Args[2:]

	done := make(chan bool)
	go func() {
		for {
			c := exec.Command(cmd, args...)
			c.Stdout = os.Stdout
			c.Stderr = os.Stderr
			select {
			case <-watcher.Event:
				fmt.Println("running", cmd, args)
				if err := c.Run(); err != nil {
					log.Println(err)
				}
			case err := <-watcher.Error:
				log.Println("fsnotify error:", err)
			}
		}
	}()

	wd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	err = watcher.Watch(wd)
	if err != nil {
		log.Fatal(err)
	}
	<-done
	watcher.Close()
}
<commit_msg>Add 200ms window<commit_after>// watches the current directory for changes and runs the specificed program on change

package main

import (
	"fmt"
	"github.com/howeyc/fsnotify"
	"log"
	"os"
	"os/exec"
	"time"
)

var help = `watcher [command to execute]`

func main() {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}
	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, help)
		os.Exit(1)
	}
	cmd, args := os.Args[1], os.Args[2:]

	done := make(chan bool)
	var event <-chan time.Time
	go func() {
		for {
			c := exec.Command(cmd, args...)
			c.Stdout = os.Stdout
			c.Stderr = os.Stderr
			select {
			case <-event:
				fmt.Println("running", cmd, args)
				if err := c.Run(); err != nil {
					log.Println(err)
				}
				event = nil
			case <-watcher.Event:
				if event == nil {
					event = time.After(200 * time.Millisecond)
				}
			case err := <-watcher.Error:
				log.Println("fsnotify error:", err)
			}
		}
	}()

	wd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	err = watcher.Watch(wd)
	if err != nil {
		log.Fatal(err)
	}
	<-done
	watcher.Close()
}
<|endoftext|>


<commit_before>package web

import (
	"fmt"

	"gopkg.in/macaron.v1"

	"github.com/prime/middleware"
	"github.com/prime/router"
	"github.com/wrench/db"
	"github.com/wrench/setting"
)

func SetPrimeMacaron(m *macaron.Macaron) {
	//Setting Database
	if err := db.InitDB(setting.DBURI, setting.DBPasswd, setting.DBDB); err != nil {
		fmt.Printf("Connect Database error %s", err.Error())
	}

	if err := middleware.Initfunc(); err != nil {
		fmt.Printf("Init middleware error %s", err.Error())
	}

	//Setting Middleware
	middleware.SetMiddlewares(m)

	/*	//Start Object Storage Service if sets in conf
		if strings.EqualFold(setting.OssSwitch, "enable") {
			ossobj := oss.Instance()
			ossobj.StartOSS()
		}
	*/
	//Setting Router
	fmt.Println("##### SetPrimeMacaron #####")
	fmt.Println("##### SetPrimeMacaron #####")
	router.SetRouters(m)
}
<commit_msg>delte test for<commit_after>package web

import (
	"fmt"

	"gopkg.in/macaron.v1"

	"github.com/prime/middleware"
	"github.com/prime/router"
	"github.com/wrench/db"
	"github.com/wrench/setting"
)

func SetPrimeMacaron(m *macaron.Macaron) {
	//Setting Database
	if err := db.InitDB(setting.DBURI, setting.DBPasswd, setting.DBDB); err != nil {
		fmt.Printf("Connect Database error %s", err.Error())
	}

	if err := middleware.Initfunc(); err != nil {
		fmt.Printf("Init middleware error %s", err.Error())
	}

	//Setting Middleware
	middleware.SetMiddlewares(m)

	/*	//Start Object Storage Service if sets in conf
		if strings.EqualFold(setting.OssSwitch, "enable") {
			ossobj := oss.Instance()
			ossobj.StartOSS()
		}
	*/
	//Setting Router
	fmt.Println("##### SetPrimeMacaron #####")
	router.SetRouters(m)
}
<|endoftext|>


<commit_before>/*
 Copyright 2017 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

// Package cmd provides the command line functions of the crunchy CLI
package cmd

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/crunchydata/postgres-operator/tpr"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"strconv"
	"strings"
)

const MAJOR_UPGRADE = "major"
const MINOR_UPGRADE = "minor"
const SEP = "-"

var UpgradeType string

var upgradeCmd = &cobra.Command{
	Use:   "upgrade",
	Short: "perform an upgrade",
	Long: `UPGRADE performs an upgrade, for example:
		pgo upgrade mycluster`,
	Run: func(cmd *cobra.Command, args []string) {
		log.Debug("upgrade called")
		if len(args) == 0 {
			fmt.Println(`You must specify the cluster to upgrade.`)
		} else {
			err := validateCreateUpdate(args)
			if err != nil {
				log.Error(err.Error())
			} else {

				createUpgrade(args)
			}
		}

	},
}

func init() {
	RootCmd.AddCommand(upgradeCmd)
	upgradeCmd.Flags().StringVarP(&UpgradeType, "upgrade-type", "t", "minor", "The upgrade type to perform either minor or major, default is minor ")
	upgradeCmd.Flags().StringVarP(&CCP_IMAGE_TAG, "ccp-image-tag", "c", "", "The CCP_IMAGE_TAG to use for the upgrade target")

}

func validateCreateUpdate(args []string) error {
	var err error

	if UpgradeType == MAJOR_UPGRADE || UpgradeType == MINOR_UPGRADE {
	} else {
		return errors.New("upgrade-type requires either a value of major or minor, if not specified, minor is the default value")
	}
	return err
}

func showUpgrade(args []string) {
	var err error
	log.Debugf("showUpgrade called %v\n", args)

	//show pod information for job
	for _, arg := range args {
		log.Debug("show upgrade called for " + arg)
		if arg == "all" {
			tprs := tpr.PgUpgradeList{}
			err = Tprclient.Get().
				Resource(tpr.UPGRADE_RESOURCE).
				Namespace(Namespace).
				Do().Into(&tprs)
			if err != nil {
				log.Error("error getting list of pgupgrades " + err.Error())
				return
			}
			for _, u := range tprs.Items {
				showUpgradeItem(&u)
			}

		} else {
			var upgrade tpr.PgUpgrade

			err = Tprclient.Get().
				Resource(tpr.UPGRADE_RESOURCE).
				Namespace(Namespace).
				Name(arg).
				Do().Into(&upgrade)
			if kerrors.IsNotFound(err) {
				fmt.Println("pgupgrade " + arg + " not found ")
			} else {
				showUpgradeItem(&upgrade)
			}

		}

	}

}

func showUpgradeItem(upgrade *tpr.PgUpgrade) {

	//print the TPR
	fmt.Printf("%s%s\n", "", "")
	fmt.Printf("%s%s\n", "", "pgupgrade : "+upgrade.Spec.Name)
	fmt.Printf("%s%s\n", TREE_BRANCH, "upgrade_status : "+upgrade.Spec.UPGRADE_STATUS)
	fmt.Printf("%s%s\n", TREE_BRANCH, "resource_type : "+upgrade.Spec.RESOURCE_TYPE)
	fmt.Printf("%s%s\n", TREE_BRANCH, "upgrade_type : "+upgrade.Spec.UPGRADE_TYPE)
	fmt.Printf("%s%s\n", TREE_BRANCH, "pvc_access_mode : "+upgrade.Spec.StorageSpec.PvcAccessMode)
	fmt.Printf("%s%s\n", TREE_BRANCH, "pvc_size : "+upgrade.Spec.StorageSpec.PvcSize)
	fmt.Printf("%s%s\n", TREE_BRANCH, "ccp_image_tag : "+upgrade.Spec.CCP_IMAGE_TAG)
	fmt.Printf("%s%s\n", TREE_BRANCH, "old_database_name : "+upgrade.Spec.OLD_DATABASE_NAME)
	fmt.Printf("%s%s\n", TREE_BRANCH, "new_database_name : "+upgrade.Spec.NEW_DATABASE_NAME)
	fmt.Printf("%s%s\n", TREE_BRANCH, "old_version : "+upgrade.Spec.OLD_VERSION)
	fmt.Printf("%s%s\n", TREE_BRANCH, "new_version : "+upgrade.Spec.NEW_VERSION)
	fmt.Printf("%s%s\n", TREE_BRANCH, "old_pvc_name : "+upgrade.Spec.OLD_PVC_NAME)
	fmt.Printf("%s%s\n", TREE_TRUNK, "new_pvc_name : "+upgrade.Spec.NEW_PVC_NAME)

	//print the upgrade jobs if any exists
	lo := meta_v1.ListOptions{
		LabelSelector: "pg-database=" + upgrade.Spec.Name + ",pgupgrade=true",
	}
	log.Debug("label selector is " + lo.LabelSelector)
	pods, err2 := Clientset.CoreV1().Pods(Namespace).List(lo)
	if err2 != nil {
		log.Error(err2.Error())
	}

	if len(pods.Items) == 0 {
		fmt.Printf("\nno upgrade job pods for %s\n", upgrade.Spec.Name+" were found")
	} else {
		fmt.Printf("\nupgrade job pods for %s\n", upgrade.Spec.Name+"...")
		for _, p := range pods.Items {
			fmt.Printf("%s pod : %s (%s)\n", TREE_TRUNK, p.Name, p.Status.Phase)
		}
	}

	fmt.Println("")

}

func createUpgrade(args []string) {
	log.Debugf("createUpgrade called %v\n", args)

	var err error
	var newInstance *tpr.PgUpgrade

	for _, arg := range args {
		log.Debug("create upgrade called for " + arg)
		result := tpr.PgUpgrade{}

		// error if it already exists
		err = Tprclient.Get().
			Resource(tpr.UPGRADE_RESOURCE).
			Namespace(Namespace).
			Name(arg).
			Do().
			Into(&result)
		if err == nil {
			log.Warn("previous pgupgrade " + arg + " was found so we will remove it.")
			forDeletion := make([]string, 1)
			forDeletion[0] = arg
			deleteUpgrade(forDeletion)
		} else if kerrors.IsNotFound(err) {
			log.Debug("pgupgrade " + arg + " not found so we will create it")
		} else {
			log.Error("error getting pgupgrade " + arg)
			log.Error(err.Error())
			break
		}

		cl := tpr.PgCluster{}

		err = Tprclient.Get().
			Resource(tpr.CLUSTER_RESOURCE).
			Namespace(Namespace).
			Name(arg).
			Do().
			Into(&cl)
		if kerrors.IsNotFound(err) {
			log.Error("error getting pgupgrade " + arg)
			break
		}

		if cl.Spec.MasterStorage.StorageType == "emptydir" {
			fmt.Println("cluster " + arg + " uses emptydir storage and can not be upgraded")
			break
		}

		// Create an instance of our TPR
		newInstance, err = getUpgradeParams(arg)
		if err == nil {
			err = Tprclient.Post().
				Resource(tpr.UPGRADE_RESOURCE).
				Namespace(Namespace).
				Body(newInstance).
				Do().Into(&result)
			if err != nil {
				log.Error("error in creating PgUpgrade TPR instance", err.Error())
			} else {
				fmt.Println("created PgUpgrade " + arg)
			}
		}

	}

}

func deleteUpgrade(args []string) {
	log.Debugf("deleteUpgrade called %v\n", args)
	var err error
	upgradeList := tpr.PgUpgradeList{}
	err = Tprclient.Get().Resource(tpr.UPGRADE_RESOURCE).Do().Into(&upgradeList)
	if err != nil {
		log.Error("error getting upgrade list")
		log.Error(err.Error())
		return
	}
	// delete the pgupgrade resource instance
	// which will cause the operator to remove the related Job
	for _, arg := range args {
		upgradeFound := false
		for _, upgrade := range upgradeList.Items {
			if arg == "all" || upgrade.Spec.Name == arg {
				upgradeFound = true
				err = Tprclient.Delete().
					Resource(tpr.UPGRADE_RESOURCE).
					Namespace(Namespace).
					Name(upgrade.Spec.Name).
					Do().
					Error()
				if err != nil {
					log.Error("error deleting pgupgrade " + arg)
					log.Error(err.Error())
				}
				fmt.Println("deleted pgupgrade " + upgrade.Spec.Name)
			}
		}
		if !upgradeFound {
			fmt.Println("upgrade " + arg + " not found")
		}

	}

}

func getUpgradeParams(name string) (*tpr.PgUpgrade, error) {

	var err error
	var existingImage string
	var existingMajorVersion float64

	spec := tpr.PgUpgradeSpec{
		Name:              name,
		RESOURCE_TYPE:     "cluster",
		UPGRADE_TYPE:      UpgradeType,
		CCP_IMAGE_TAG:     viper.GetString("CLUSTER.CCP_IMAGE_TAG"),
		StorageSpec:       tpr.PgStorageSpec{},
		OLD_DATABASE_NAME: "basic",
		NEW_DATABASE_NAME: "master",
		OLD_VERSION:       "9.5",
		NEW_VERSION:       "9.6",
		OLD_PVC_NAME:      viper.GetString("MASTER_STORAGE.PVC_NAME"),
		NEW_PVC_NAME:      viper.GetString("MASTER_STORAGE.PVC_NAME"),
	}

	spec.StorageSpec.PvcAccessMode = viper.GetString("MASTER_STORAGE.PVC_ACCESS_MODE")
	spec.StorageSpec.PvcSize = viper.GetString("MASTER_STORAGE.PVC_SIZE")

	if CCP_IMAGE_TAG != "" {
		log.Debug("using CCP_IMAGE_TAG from command line " + CCP_IMAGE_TAG)
		spec.CCP_IMAGE_TAG = CCP_IMAGE_TAG
	}

	cluster := tpr.PgCluster{}
	err = Tprclient.Get().
		Resource(tpr.CLUSTER_RESOURCE).
		Namespace(Namespace).
		Name(name).
		Do().
		Into(&cluster)
	if err == nil {
		spec.RESOURCE_TYPE = "cluster"
		spec.OLD_DATABASE_NAME = cluster.Spec.Name
		spec.NEW_DATABASE_NAME = cluster.Spec.Name + "-upgrade"
		spec.OLD_PVC_NAME = cluster.Spec.MasterStorage.PvcName
		spec.NEW_PVC_NAME = cluster.Spec.MasterStorage.PvcName + "-upgrade"
		spec.BACKUP_PVC_NAME = cluster.Spec.BACKUP_PVC_NAME
		existingImage = cluster.Spec.CCP_IMAGE_TAG
		existingMajorVersion = parseMajorVersion(cluster.Spec.CCP_IMAGE_TAG)
	} else if kerrors.IsNotFound(err) {
		log.Debug(name + " is not a cluster")
		return nil, err
	} else {
		log.Error("error getting pgcluster " + name)
		log.Error(err.Error())
		return nil, err
	}

	var requestedMajorVersion float64

	if CCP_IMAGE_TAG != "" {
		if CCP_IMAGE_TAG == existingImage {
			log.Error("CCP_IMAGE_TAG is the same as the cluster")
			log.Error("can't upgrade to the same image version")

			return nil, errors.New("invalid image tag")
		}
		requestedMajorVersion = parseMajorVersion(CCP_IMAGE_TAG)
	} else if viper.GetString("CLUSTER.CCP_IMAGE_TAG") == existingImage {
		log.Error("CCP_IMAGE_TAG is the same as the cluster")
		log.Error("can't upgrade to the same image version")

		return nil, errors.New("invalid image tag")
	} else {
		requestedMajorVersion = parseMajorVersion(viper.GetString("CLUSTER.CCP_IMAGE_TAG"))
	}

	if UpgradeType == MAJOR_UPGRADE {
		if requestedMajorVersion == existingMajorVersion {
			log.Error("can't upgrade to the same major version")
			return nil, errors.New("requested upgrade major version can not equal existing upgrade major version")
		} else if requestedMajorVersion < existingMajorVersion {
			log.Error("can't upgrade to a previous major version")
			return nil, errors.New("requested upgrade major version can not be older than existing upgrade major version")
		}
	} else {
		//minor upgrade
		if requestedMajorVersion > existingMajorVersion {
			log.Error("can't do minor upgrade to a newer major version")
			return nil, errors.New("requested minor upgrade to major version is not allowed")
		}
	}

	newInstance := &tpr.PgUpgrade{
		Metadata: meta_v1.ObjectMeta{
			Name: name,
		},
		Spec: spec,
	}
	return newInstance, err
}

func parseMajorVersion(st string) float64 {
	parts := strings.Split(st, SEP)
	//OS = parts[0]
	//PGVERSION = parts[1]
	//CVERSION = parts[2]

	f, err := strconv.ParseFloat(parts[1], 64)
	if err != nil {
		fmt.Println(err.Error())
	}
	return f

}
<commit_msg>add upgrade stub<commit_after>/*
 Copyright 2017 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

// Package cmd provides the command line functions of the crunchy CLI
package cmd

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/crunchydata/postgres-operator/tpr"
	"github.com/crunchydata/postgres-operator/upgradeservice"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"net/http"
	"strconv"
	"strings"
)

const MAJOR_UPGRADE = "major"
const MINOR_UPGRADE = "minor"
const SEP = "-"

var UpgradeType string

var upgradeCmd = &cobra.Command{
	Use:   "upgrade",
	Short: "perform an upgrade",
	Long: `UPGRADE performs an upgrade, for example:
		pgo upgrade mycluster`,
	Run: func(cmd *cobra.Command, args []string) {
		log.Debug("upgrade called")
		if len(args) == 0 {
			fmt.Println(`You must specify the cluster to upgrade.`)
		} else {
			err := validateCreateUpdate(args)
			if err != nil {
				log.Error(err.Error())
			} else {

				createUpgrade(args)
			}
		}

	},
}

func init() {
	RootCmd.AddCommand(upgradeCmd)
	upgradeCmd.Flags().StringVarP(&UpgradeType, "upgrade-type", "t", "minor", "The upgrade type to perform either minor or major, default is minor ")
	upgradeCmd.Flags().StringVarP(&CCP_IMAGE_TAG, "ccp-image-tag", "c", "", "The CCP_IMAGE_TAG to use for the upgrade target")

}

func validateCreateUpdate(args []string) error {
	var err error

	if UpgradeType == MAJOR_UPGRADE || UpgradeType == MINOR_UPGRADE {
	} else {
		return errors.New("upgrade-type requires either a value of major or minor, if not specified, minor is the default value")
	}
	return err
}

func showUpgrade(args []string) {
	var err error
	log.Debugf("showUpgrade called %v\n", args)

	url := "http://localhost:8080/upgrades/somename?showsecrets=true&other=thing"

	action := "GET"
	req, err := http.NewRequest(action, url, nil)
	if err != nil {
		log.Fatal("NewRequest: ", err)
		return
	}

	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		log.Fatal("Do: ", err)
		return
	}

	defer resp.Body.Close()

	var response upgradeservice.ShowUpgradeResponse

	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		log.Println(err)
	}

	fmt.Println("Name = ", response.Items[0].Name)

}

func showUpgradeItem(upgrade *tpr.PgUpgrade) {

	//print the TPR
	fmt.Printf("%s%s\n", "", "")
	fmt.Printf("%s%s\n", "", "pgupgrade : "+upgrade.Spec.Name)
	fmt.Printf("%s%s\n", TREE_BRANCH, "upgrade_status : "+upgrade.Spec.UPGRADE_STATUS)
	fmt.Printf("%s%s\n", TREE_BRANCH, "resource_type : "+upgrade.Spec.RESOURCE_TYPE)
	fmt.Printf("%s%s\n", TREE_BRANCH, "upgrade_type : "+upgrade.Spec.UPGRADE_TYPE)
	fmt.Printf("%s%s\n", TREE_BRANCH, "pvc_access_mode : "+upgrade.Spec.StorageSpec.PvcAccessMode)
	fmt.Printf("%s%s\n", TREE_BRANCH, "pvc_size : "+upgrade.Spec.StorageSpec.PvcSize)
	fmt.Printf("%s%s\n", TREE_BRANCH, "ccp_image_tag : "+upgrade.Spec.CCP_IMAGE_TAG)
	fmt.Printf("%s%s\n", TREE_BRANCH, "old_database_name : "+upgrade.Spec.OLD_DATABASE_NAME)
	fmt.Printf("%s%s\n", TREE_BRANCH, "new_database_name : "+upgrade.Spec.NEW_DATABASE_NAME)
	fmt.Printf("%s%s\n", TREE_BRANCH, "old_version : "+upgrade.Spec.OLD_VERSION)
	fmt.Printf("%s%s\n", TREE_BRANCH, "new_version : "+upgrade.Spec.NEW_VERSION)
	fmt.Printf("%s%s\n", TREE_BRANCH, "old_pvc_name : "+upgrade.Spec.OLD_PVC_NAME)
	fmt.Printf("%s%s\n", TREE_TRUNK, "new_pvc_name : "+upgrade.Spec.NEW_PVC_NAME)

	//print the upgrade jobs if any exists
	lo := meta_v1.ListOptions{
		LabelSelector: "pg-database=" + upgrade.Spec.Name + ",pgupgrade=true",
	}
	log.Debug("label selector is " + lo.LabelSelector)
	pods, err2 := Clientset.CoreV1().Pods(Namespace).List(lo)
	if err2 != nil {
		log.Error(err2.Error())
	}

	if len(pods.Items) == 0 {
		fmt.Printf("\nno upgrade job pods for %s\n", upgrade.Spec.Name+" were found")
	} else {
		fmt.Printf("\nupgrade job pods for %s\n", upgrade.Spec.Name+"...")
		for _, p := range pods.Items {
			fmt.Printf("%s pod : %s (%s)\n", TREE_TRUNK, p.Name, p.Status.Phase)
		}
	}

	fmt.Println("")

}

func createUpgrade(args []string) {
	log.Debugf("createUpgrade called %v\n", args)

	//var err error
	//var newInstance *tpr.PgUpgrade

	for _, arg := range args {
		log.Debug("create upgrade called for " + arg)
		url := "http://localhost:8080/upgrades"

		cl := new(upgradeservice.CreateUpgradeRequest)
		cl.Name = "newupgrae"
		jsonValue, _ := json.Marshal(cl)
		req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonValue))
		if err != nil {
			log.Fatal("NewRequest: ", err)
			return
		}

		req.Header.Set("Content-Type", "application/json")
		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			log.Fatal("Do: ", err)
			return
		}
		fmt.Printf("%v\n", resp)

	}

}

func deleteUpgrade(args []string) {
	log.Debugf("deleteUpgrade called %v\n", args)
	//var err error
	for _, arg := range args {
		fmt.Println("deleting upgrade " + arg)
		url := "http://localhost:8080/upgrades/somename?showsecrets=true&other=thing"

		action := "DELETE"
		req, err := http.NewRequest(action, url, nil)
		if err != nil {
			log.Fatal("NewRequest: ", err)
			return
		}

		client := &http.Client{}

		resp, err := client.Do(req)
		if err != nil {
			log.Fatal("Do: ", err)
			return
		}

		defer resp.Body.Close()

		var response upgradeservice.ShowUpgradeResponse

		if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
			log.Println(err)
		}

		fmt.Println("Name = ", response.Items[0].Name)

	}

}

func getUpgradeParams(name string) (*tpr.PgUpgrade, error) {

	var err error
	var existingImage string
	var existingMajorVersion float64

	spec := tpr.PgUpgradeSpec{
		Name:              name,
		RESOURCE_TYPE:     "cluster",
		UPGRADE_TYPE:      UpgradeType,
		CCP_IMAGE_TAG:     viper.GetString("CLUSTER.CCP_IMAGE_TAG"),
		StorageSpec:       tpr.PgStorageSpec{},
		OLD_DATABASE_NAME: "basic",
		NEW_DATABASE_NAME: "master",
		OLD_VERSION:       "9.5",
		NEW_VERSION:       "9.6",
		OLD_PVC_NAME:      viper.GetString("MASTER_STORAGE.PVC_NAME"),
		NEW_PVC_NAME:      viper.GetString("MASTER_STORAGE.PVC_NAME"),
	}

	spec.StorageSpec.PvcAccessMode = viper.GetString("MASTER_STORAGE.PVC_ACCESS_MODE")
	spec.StorageSpec.PvcSize = viper.GetString("MASTER_STORAGE.PVC_SIZE")

	if CCP_IMAGE_TAG != "" {
		log.Debug("using CCP_IMAGE_TAG from command line " + CCP_IMAGE_TAG)
		spec.CCP_IMAGE_TAG = CCP_IMAGE_TAG
	}

	cluster := tpr.PgCluster{}
	err = Tprclient.Get().
		Resource(tpr.CLUSTER_RESOURCE).
		Namespace(Namespace).
		Name(name).
		Do().
		Into(&cluster)
	if err == nil {
		spec.RESOURCE_TYPE = "cluster"
		spec.OLD_DATABASE_NAME = cluster.Spec.Name
		spec.NEW_DATABASE_NAME = cluster.Spec.Name + "-upgrade"
		spec.OLD_PVC_NAME = cluster.Spec.MasterStorage.PvcName
		spec.NEW_PVC_NAME = cluster.Spec.MasterStorage.PvcName + "-upgrade"
		spec.BACKUP_PVC_NAME = cluster.Spec.BACKUP_PVC_NAME
		existingImage = cluster.Spec.CCP_IMAGE_TAG
		existingMajorVersion = parseMajorVersion(cluster.Spec.CCP_IMAGE_TAG)
	} else if kerrors.IsNotFound(err) {
		log.Debug(name + " is not a cluster")
		return nil, err
	} else {
		log.Error("error getting pgcluster " + name)
		log.Error(err.Error())
		return nil, err
	}

	var requestedMajorVersion float64

	if CCP_IMAGE_TAG != "" {
		if CCP_IMAGE_TAG == existingImage {
			log.Error("CCP_IMAGE_TAG is the same as the cluster")
			log.Error("can't upgrade to the same image version")

			return nil, errors.New("invalid image tag")
		}
		requestedMajorVersion = parseMajorVersion(CCP_IMAGE_TAG)
	} else if viper.GetString("CLUSTER.CCP_IMAGE_TAG") == existingImage {
		log.Error("CCP_IMAGE_TAG is the same as the cluster")
		log.Error("can't upgrade to the same image version")

		return nil, errors.New("invalid image tag")
	} else {
		requestedMajorVersion = parseMajorVersion(viper.GetString("CLUSTER.CCP_IMAGE_TAG"))
	}

	if UpgradeType == MAJOR_UPGRADE {
		if requestedMajorVersion == existingMajorVersion {
			log.Error("can't upgrade to the same major version")
			return nil, errors.New("requested upgrade major version can not equal existing upgrade major version")
		} else if requestedMajorVersion < existingMajorVersion {
			log.Error("can't upgrade to a previous major version")
			return nil, errors.New("requested upgrade major version can not be older than existing upgrade major version")
		}
	} else {
		//minor upgrade
		if requestedMajorVersion > existingMajorVersion {
			log.Error("can't do minor upgrade to a newer major version")
			return nil, errors.New("requested minor upgrade to major version is not allowed")
		}
	}

	newInstance := &tpr.PgUpgrade{
		Metadata: meta_v1.ObjectMeta{
			Name: name,
		},
		Spec: spec,
	}
	return newInstance, err
}

func parseMajorVersion(st string) float64 {
	parts := strings.Split(st, SEP)
	//OS = parts[0]
	//PGVERSION = parts[1]
	//CVERSION = parts[2]

	f, err := strconv.ParseFloat(parts[1], 64)
	if err != nil {
		fmt.Println(err.Error())
	}
	return f

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"time"

	"github.com/adnanh/webhook/hooks"

	"github.com/go-martini/martini"

	"crypto/hmac"
	"crypto/sha256"

	l4g "code.google.com/p/log4go"
)

const (
	version string = "1.0.1"
)

var (
	webhooks      *hooks.Hooks
	appStart      time.Time
	ip            = flag.String("ip", "", "ip the webhook server should listen on")
	port          = flag.Int("port", 9000, "port the webhook server should listen on")
	hooksFilename = flag.String("hooks", "hooks.json", "path to the json file containing defined hooks the webhook should serve")
	logFilename   = flag.String("log", "webhook.log", "path to the log file")
)

func init() {
	flag.Parse()

	fileLogWriter := l4g.NewFileLogWriter(*logFilename, false)
	fileLogWriter.SetRotateDaily(false)

	martini.Env = "production"

	l4g.AddFilter("file", l4g.FINE, fileLogWriter)
}

func main() {
	appStart = time.Now()
	var e error

	webhooks, e = hooks.New(*hooksFilename)

	if e != nil {
		l4g.Warn("Error occurred while loading hooks from %s: %s", *hooksFilename, e)
	}

	web := martini.Classic()

	web.Get("/", rootHandler)
	web.Get("/hook/:id", hookHandler)
	web.Post("/hook/:id", hookHandler)

	l4g.Info("Starting webhook %s with %d hook(s) on %s:%d", version, webhooks.Count(), *ip, *port)

	web.RunOnAddr(fmt.Sprintf("%s:%d", *ip, *port))
}

func rootHandler() string {
	return fmt.Sprintf("webhook %s running for %s serving %d hook(s)\n", version, time.Since(appStart).String(), webhooks.Count())
}

func hookHandler(req *http.Request, params martini.Params) string {
	p := make(map[string]interface{})

	if req.Header.Get("Content-Type") == "application/json" {
		decoder := json.NewDecoder(req.Body)
		decoder.UseNumber()

		err := decoder.Decode(&p)

		if err != nil {
			l4g.Warn("Error occurred while trying to parse the payload as JSON: %s", err)
		}
	}

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		l4g.Warn("Error occurred while trying to read the request body: %s", err)
	}

	go func(id string, body []byte, signature string, params interface{}) {
		if hook := webhooks.Match(id, params); hook != nil {
			if hook.Secret != "" {
				if signature == "" {
					l4g.Error("Hook %s got matched, but the request contained invalid signature.", hook.ID)
					return
				}

				mac := hmac.New(sha256.New, []byte(hook.Secret))
				mac.Write(body)
				expectedMAC := mac.Sum(nil)

				if !hmac.Equal([]byte(signature), expectedMAC) {
					l4g.Error("Hook %s got matched, but the request contained invalid signature. Expected %s, got %s.", hook.ID, signature, expectedMAC)
					return
				}
			}

			cmd := exec.Command(hook.Command, "", "", hook.Cwd)
			out, _ := cmd.Output()
			l4g.Info("Hook %s triggered successfully! Command output:\n%s", hook.ID, out)
		}
	}(params["id"], body, req.Header.Get("X-Hub-Signature"), p)

	return "Got it, thanks. :-)"
}
<commit_msg>debugging...<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"time"

	"github.com/adnanh/webhook/hooks"

	"github.com/go-martini/martini"

	"crypto/hmac"
	"crypto/sha256"

	l4g "code.google.com/p/log4go"
)

const (
	version string = "1.0.1"
)

var (
	webhooks      *hooks.Hooks
	appStart      time.Time
	ip            = flag.String("ip", "", "ip the webhook server should listen on")
	port          = flag.Int("port", 9000, "port the webhook server should listen on")
	hooksFilename = flag.String("hooks", "hooks.json", "path to the json file containing defined hooks the webhook should serve")
	logFilename   = flag.String("log", "webhook.log", "path to the log file")
)

func init() {
	flag.Parse()

	fileLogWriter := l4g.NewFileLogWriter(*logFilename, false)
	fileLogWriter.SetRotateDaily(false)

	martini.Env = "production"

	l4g.AddFilter("file", l4g.FINE, fileLogWriter)
}

func main() {
	appStart = time.Now()
	var e error

	webhooks, e = hooks.New(*hooksFilename)

	if e != nil {
		l4g.Warn("Error occurred while loading hooks from %s: %s", *hooksFilename, e)
	}

	web := martini.Classic()

	web.Get("/", rootHandler)
	web.Get("/hook/:id", hookHandler)
	web.Post("/hook/:id", hookHandler)

	l4g.Info("Starting webhook %s with %d hook(s) on %s:%d", version, webhooks.Count(), *ip, *port)

	web.RunOnAddr(fmt.Sprintf("%s:%d", *ip, *port))
}

func rootHandler() string {
	return fmt.Sprintf("webhook %s running for %s serving %d hook(s)\n", version, time.Since(appStart).String(), webhooks.Count())
}

func hookHandler(req *http.Request, params martini.Params) string {
	p := make(map[string]interface{})

	if req.Header.Get("Content-Type") == "application/json" {
		decoder := json.NewDecoder(req.Body)
		decoder.UseNumber()

		err := decoder.Decode(&p)

		if err != nil {
			l4g.Warn("Error occurred while trying to parse the payload as JSON: %s", err)
		}
	}

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		l4g.Warn("Error occurred while trying to read the request body: %s", err)
	}

	go func(id string, body []byte, signature string, params interface{}) {
		if hook := webhooks.Match(id, params); hook != nil {
			if hook.Secret != "" {
				if signature == "" {
					l4g.Error("Hook %s got matched, but the request contained invalid signature.", hook.ID)
					return
				}

				mac := hmac.New(sha256.New, []byte(hook.Secret))
				mac.Write(body)
				expectedMAC := mac.Sum(nil)

				l4g.Info("Expected %s, got %s.", expectedMAC, signature)

				if !hmac.Equal([]byte(signature), expectedMAC) {
					l4g.Error("Hook %s got matched, but the request contained invalid signature. Expected %s, got %s.", hook.ID, expectedMAC, signature)
					return
				}
			}

			cmd := exec.Command(hook.Command, "", "", hook.Cwd)
			out, _ := cmd.Output()
			l4g.Info("Hook %s triggered successfully! Command output:\n%s", hook.ID, out)
		}
	}(params["id"], body, req.Header.Get("X-Hub-Signature"), p)

	return "Got it, thanks. :-)"
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Mute Communications Ltd.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package log

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"

	"github.com/cihub/seelog"
)

var logger seelog.LoggerInterface

func init() {
	// disable logger by default
	logger = seelog.Disabled
}

// Init initializes the Mute logging framework to the given logging level.
// If logDir is not nil logging is done to a logfile in the directory.
// If logToConsole is true the console logging is activated.
// cmdPrefix must be a 5 character long command prefix.
// If the given level is invalid or the initialization fails, an
// error is returned.
func Init(logLevel, cmdPrefix, logDir string, logToConsole bool) error {
	// check level string
	_, found := seelog.LogLevelFromString(logLevel)
	if !found {
		return fmt.Errorf("log: level '%s' is invalid", logLevel)
	}
	// check cmdPrefix
	if len(cmdPrefix) != 5 {
		return fmt.Errorf("len(cmdPrefix) must be 5: \"%s\"", cmdPrefix)
	}
	// create logger
	console := "<console />"
	if !logToConsole {
		console = ""
	}
	var file string
	if logDir != "" {
		file = fmt.Sprintf("<rollingfile type=\"size\" filename=\"%s\" maxsize=\"10485760\" maxrolls=\"3\" />",
			filepath.Join(logDir, os.Args[0]+".log"))
	}
	config := `
<seelog type="adaptive" mininterval="2000000" maxinterval="100000000"
	critmsgcount="500" minlevel="%s">
	<outputs formatid="all">
		%s
		%s
	</outputs>
	<formats>
		<format id="all" format="%%UTCDate %%UTCTime [%s] [%%LEV] %%Msg%%n" />
	</formats>
</seelog>`
	config = fmt.Sprintf(config, logLevel, console, file, cmdPrefix)
	logger, err := seelog.LoggerFromConfigAsString(config)
	if err != nil {
		return err
	}
	logger.SetAdditionalStackDepth(1)
	// replace logger
	UseLogger(logger)
	// log info about running binary
	Infof("%s started (built with %s %s for %s/%s)", os.Args[0], runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH)
	return nil
}

// Flush flushes all the messages in the logger.
func Flush() {
	Infof("%s stopping", os.Args[0])
	logger.Flush()
}

// Critical formats message using the default formats for its operands and
// writes to default logger with log level = Critical.
func Critical(v ...interface{}) error {
	if len(v) == 1 {
		err, ok := v[0].(error)
		if ok {
			logger.Critical(err)
			return err
		}
	}
	return logger.Critical(v...)
}

// Criticalf formats message according to format specifier and writes to
// default logger with log level = Critical.
func Criticalf(format string, params ...interface{}) error {
	return logger.Criticalf(format, params...)
}

// Error formats message using the default formats for its operands and writes
// to default logger with log level = Error.
func Error(v ...interface{}) error {
	if len(v) == 1 {
		err, ok := v[0].(error)
		if ok {
			logger.Error(err)
			return err
		}
	}
	return logger.Error(v...)
}

// Errorf formats message according to format specifier and writes to default
// logger with log level = Error.
func Errorf(format string, params ...interface{}) error {
	return logger.Errorf(format, params...)
}

// Warn formats message using the default formats for its operands and writes
// to default logger with log level = Warn.
func Warn(v ...interface{}) error {
	if len(v) == 1 {
		err, ok := v[0].(error)
		if ok {
			logger.Warn(err)
			return err
		}
	}
	return logger.Warn(v...)
}

// Warnf formats message according to format specifier and writes to default
// logger with log level = Warn.
func Warnf(format string, params ...interface{}) error {
	return logger.Warnf(format, params...)
}

// Info formats message using the default formats for its operands and writes
// to default logger with log level = Info.
func Info(v ...interface{}) {
	logger.Info(v...)
}

// Infof formats message according to format specifier and writes to default
// logger with log level = Info.
func Infof(format string, params ...interface{}) {
	logger.Infof(format, params...)
}

// Debug formats message using the default formats for its operands and writes
// to default logger with log level = Debug.
func Debug(v ...interface{}) {
	logger.Debug(v...)
}

// Debugf formats message according to format specifier and writes to default
// logger with log level = Debug.
func Debugf(format string, params ...interface{}) {
	logger.Debugf(format, params...)
}

// Trace formats message using the default formats for its operands and writes
// to default logger with log level = Trace.
func Trace(v ...interface{}) {
	logger.Trace(v...)
}

// Tracef formats message according to format specifier and writes to default
// logger with log level = Trace.
func Tracef(format string, params ...interface{}) {
	logger.Tracef(format, params...)
}

// UseLogger uses a specified seelog.LoggerInterface to output library log.
// Use this func if you are using Seelog logging system in your app.
func UseLogger(newLogger seelog.LoggerInterface) {
	logger = newLogger
}

// SetLogWriter uses a specified io.Writer to output library log.
// Use this func if you are not using Seelog logging system in your app.
func SetLogWriter(writer io.Writer) error {
	if writer == nil {
		return errors.New("Nil writer")
	}

	newLogger, err := seelog.LoggerFromWriterWithMinLevel(writer, seelog.TraceLvl)
	if err != nil {
		return err
	}

	UseLogger(newLogger)
	return nil
}
<commit_msg>log: fix problem with log file naming<commit_after>// Copyright (c) 2015 Mute Communications Ltd.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package log

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/cihub/seelog"
)

var logger seelog.LoggerInterface

func init() {
	// disable logger by default
	logger = seelog.Disabled
}

// Init initializes the Mute logging framework to the given logging level.
// If logDir is not nil logging is done to a logfile in the directory.
// If logToConsole is true the console logging is activated.
// cmdPrefix must be a 5 character long command prefix.
// If the given level is invalid or the initialization fails, an
// error is returned.
func Init(logLevel, cmdPrefix, logDir string, logToConsole bool) error {
	// check level string
	_, found := seelog.LogLevelFromString(logLevel)
	if !found {
		return fmt.Errorf("log: level '%s' is invalid", logLevel)
	}
	// check cmdPrefix
	if len(cmdPrefix) != 5 {
		return fmt.Errorf("len(cmdPrefix) must be 5: \"%s\"", cmdPrefix)
	}
	// create logger
	console := "<console />"
	if !logToConsole {
		console = ""
	}
	var file string
	if logDir != "" {
		execBase := strings.TrimSuffix(filepath.Base(os.Args[0]), ".exe")
		file = fmt.Sprintf("<rollingfile type=\"size\" filename=\"%s\" maxsize=\"10485760\" maxrolls=\"3\" />",
			filepath.Join(logDir, execBase+".log"))
	}
	config := `
<seelog type="adaptive" mininterval="2000000" maxinterval="100000000"
	critmsgcount="500" minlevel="%s">
	<outputs formatid="all">
		%s
		%s
	</outputs>
	<formats>
		<format id="all" format="%%UTCDate %%UTCTime [%s] [%%LEV] %%Msg%%n" />
	</formats>
</seelog>`
	config = fmt.Sprintf(config, logLevel, console, file, cmdPrefix)
	logger, err := seelog.LoggerFromConfigAsString(config)
	if err != nil {
		return err
	}
	logger.SetAdditionalStackDepth(1)
	// replace logger
	UseLogger(logger)
	// log info about running binary
	Infof("%s started (built with %s %s for %s/%s)", os.Args[0], runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH)
	return nil
}

// Flush flushes all the messages in the logger.
func Flush() {
	Infof("%s stopping", os.Args[0])
	logger.Flush()
}

// Critical formats message using the default formats for its operands and
// writes to default logger with log level = Critical.
func Critical(v ...interface{}) error {
	if len(v) == 1 {
		err, ok := v[0].(error)
		if ok {
			logger.Critical(err)
			return err
		}
	}
	return logger.Critical(v...)
}

// Criticalf formats message according to format specifier and writes to
// default logger with log level = Critical.
func Criticalf(format string, params ...interface{}) error {
	return logger.Criticalf(format, params...)
}

// Error formats message using the default formats for its operands and writes
// to default logger with log level = Error.
func Error(v ...interface{}) error {
	if len(v) == 1 {
		err, ok := v[0].(error)
		if ok {
			logger.Error(err)
			return err
		}
	}
	return logger.Error(v...)
}

// Errorf formats message according to format specifier and writes to default
// logger with log level = Error.
func Errorf(format string, params ...interface{}) error {
	return logger.Errorf(format, params...)
}

// Warn formats message using the default formats for its operands and writes
// to default logger with log level = Warn.
func Warn(v ...interface{}) error {
	if len(v) == 1 {
		err, ok := v[0].(error)
		if ok {
			logger.Warn(err)
			return err
		}
	}
	return logger.Warn(v...)
}

// Warnf formats message according to format specifier and writes to default
// logger with log level = Warn.
func Warnf(format string, params ...interface{}) error {
	return logger.Warnf(format, params...)
}

// Info formats message using the default formats for its operands and writes
// to default logger with log level = Info.
func Info(v ...interface{}) {
	logger.Info(v...)
}

// Infof formats message according to format specifier and writes to default
// logger with log level = Info.
func Infof(format string, params ...interface{}) {
	logger.Infof(format, params...)
}

// Debug formats message using the default formats for its operands and writes
// to default logger with log level = Debug.
func Debug(v ...interface{}) {
	logger.Debug(v...)
}

// Debugf formats message according to format specifier and writes to default
// logger with log level = Debug.
func Debugf(format string, params ...interface{}) {
	logger.Debugf(format, params...)
}

// Trace formats message using the default formats for its operands and writes
// to default logger with log level = Trace.
func Trace(v ...interface{}) {
	logger.Trace(v...)
}

// Tracef formats message according to format specifier and writes to default
// logger with log level = Trace.
func Tracef(format string, params ...interface{}) {
	logger.Tracef(format, params...)
}

// UseLogger uses a specified seelog.LoggerInterface to output library log.
// Use this func if you are using Seelog logging system in your app.
func UseLogger(newLogger seelog.LoggerInterface) {
	logger = newLogger
}

// SetLogWriter uses a specified io.Writer to output library log.
// Use this func if you are not using Seelog logging system in your app.
func SetLogWriter(writer io.Writer) error {
	if writer == nil {
		return errors.New("Nil writer")
	}

	newLogger, err := seelog.LoggerFromWriterWithMinLevel(writer, seelog.TraceLvl)
	if err != nil {
		return err
	}

	UseLogger(newLogger)
	return nil
}
<|endoftext|>


<commit_before>package zlog

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path"
	"runtime"
	"sync"
	"time"
)

const (
	DebugLevel = iota
	TraceLevel
	InfoLevel
	ErrorLevel
	FatalLevel
)

// return string base of log level
var severityName = []string{
	DebugLevel: "Debug",
	TraceLevel: "Trace",
	InfoLevel:  " Info",
	ErrorLevel: "Error",
	FatalLevel: "Fatal",
}

type Logger struct {
	logLevel int
	depth    int
	buffers  [2]bytes.Buffer
	writebuf buffer
	readbuf  buffer
	mu       sync.Mutex
}

type buffer struct {
	ptr   *bytes.Buffer
	index int
}

var logger Logger

const defaultCallDepth int = 2

var message = make(chan bool)

//50M roll back the file
var rollFileSize int64 = 1024 * 1024 * 50

func init() {
	logger.depth = defaultCallDepth
	logger.logLevel = TraceLevel
	logger.writebuf.ptr = &logger.buffers[0]
	logger.writebuf.index = 0
	logger.readbuf.ptr = &logger.buffers[0]
	logger.readbuf.index = 0
}

func InitLogger(rootPath string, level int) {

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(err)
		}
	}()

	if level < DebugLevel || level > FatalLevel {
		panic("Logger level is not supported")
	}

	logFileProperty.rootPath = rootPath
	err := logFileProperty.getLogFile()
	if err != nil {
		panic(err)
	}

	go WriteMsg()
}

func SetOutput(out io.Writer) {

}

func (logger *Logger) switchBuf() {
	if logger.writebuf.index == 0 {
		logger.writebuf.ptr = &logger.buffers[1]
		logger.writebuf.index = 1
		logger.readbuf.ptr = &logger.buffers[0]
		logger.readbuf.index = 0
	} else {
		logger.writebuf.ptr = &logger.buffers[0]
		logger.writebuf.index = 0
		logger.readbuf.ptr = &logger.buffers[1]
		logger.readbuf.index = 1
	}
}

// call after InitLogger function
// generally, you needn't change it
func SetCallDepth(depth int) {
	if depth > 0 {
		logger.depth = depth
	}
}

func (logger *Logger) logFormat(level int, log string) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println(err)
		}
	}()

	fileTime, filename, line := makeLogHead()
	logger.mu.Lock()
	logger.writebuf.ptr.WriteString(fmt.Sprintf("%s [%s]: %s (%s:%d) \n", fileTime, severityName[level], log, filename, line))

	if logger.writebuf.ptr.Len() > 1024*1024*10 {
		message <- true
	}
	logger.mu.Unlock()
}

func makeLogHead() (headTime, fileName string, line int) {
	now := time.Now()
	fileTime := now.Format("20060102 15:04:05")
	fileTime = fmt.Sprintf("%s.%09d", fileTime, now.Nanosecond())
	_, filePath, line, ok := runtime.Caller(logger.depth)
	if ok == false {
		fileName = "xxx"
		line = 0
		//panic(errors.New("get the line failed"))
	}
	//tmp := strings.Split(file, "/")
	//file = tmp[len(tmp)-1]
	_, fileName = path.Split(filePath)

	return fileTime, fileName, line
}

func Debug(format string, args ...interface{}) {
	if DebugLevel < logger.logLevel {
		return
	}
	logger.logFormat(DebugLevel, fmt.Sprintf(format, args...))
}

func Info(format string, args ...interface{}) {
	if InfoLevel < logger.logLevel {
		return
	}

	logger.logFormat(InfoLevel, fmt.Sprintf(format, args...))
}

func Error(format string, args ...interface{}) {
	if ErrorLevel < logger.logLevel {
		return
	}

	logger.logFormat(ErrorLevel, fmt.Sprintf(format, args...))
}

func Trace(format string, args ...interface{}) {
	if TraceLevel < logger.logLevel {
		return
	}

	logger.logFormat(TraceLevel, fmt.Sprintf(format, args...))
}

func Fatal(format string, args ...interface{}) {
	if FatalLevel < logger.logLevel {
		return
	}

	logger.logFormat(FatalLevel, fmt.Sprintf(format, args...))
	os.Exit(-1)
}
<commit_msg>change default call depth<commit_after>package zlog

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path"
	"runtime"
	"sync"
	"time"
)

const (
	DebugLevel = iota
	TraceLevel
	InfoLevel
	ErrorLevel
	FatalLevel
)

// return string base of log level
var severityName = []string{
	DebugLevel: "Debug",
	TraceLevel: "Trace",
	InfoLevel:  " Info",
	ErrorLevel: "Error",
	FatalLevel: "Fatal",
}

type Logger struct {
	logLevel int
	depth    int
	buffers  [2]bytes.Buffer
	writebuf buffer
	readbuf  buffer
	mu       sync.Mutex
}

type buffer struct {
	ptr   *bytes.Buffer
	index int
}

var logger Logger

const defaultCallDepth int = 3

var message = make(chan bool)

//50M roll back the file
var rollFileSize int64 = 1024 * 1024 * 50

func init() {
	logger.depth = defaultCallDepth
	logger.logLevel = TraceLevel
	logger.writebuf.ptr = &logger.buffers[0]
	logger.writebuf.index = 0
	logger.readbuf.ptr = &logger.buffers[0]
	logger.readbuf.index = 0
}

func InitLogger(rootPath string, level int) {

	defer func() {
		if err := recover(); err != nil {
			fmt.Println(err)
		}
	}()

	if level < DebugLevel || level > FatalLevel {
		panic("Logger level is not supported")
	}

	logFileProperty.rootPath = rootPath
	err := logFileProperty.getLogFile()
	if err != nil {
		panic(err)
	}

	go WriteMsg()
}

func SetOutput(out io.Writer) {

}

func (logger *Logger) switchBuf() {
	if logger.writebuf.index == 0 {
		logger.writebuf.ptr = &logger.buffers[1]
		logger.writebuf.index = 1
		logger.readbuf.ptr = &logger.buffers[0]
		logger.readbuf.index = 0
	} else {
		logger.writebuf.ptr = &logger.buffers[0]
		logger.writebuf.index = 0
		logger.readbuf.ptr = &logger.buffers[1]
		logger.readbuf.index = 1
	}
}

// call after InitLogger function
// generally, you needn't change it
func SetCallDepth(depth int) {
	if depth > 0 {
		logger.depth = depth
	}
}

func (logger *Logger) logFormat(level int, log string) {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println(err)
		}
	}()

	fileTime, filename, line := makeLogHead()
	logger.mu.Lock()
	logger.writebuf.ptr.WriteString(fmt.Sprintf("%s [%s]: %s (%s:%d) \n", fileTime, severityName[level], log, filename, line))

	if logger.writebuf.ptr.Len() > 1024*1024*10 {
		message <- true
	}
	logger.mu.Unlock()
}

func makeLogHead() (headTime, fileName string, line int) {
	now := time.Now()
	fileTime := now.Format("20060102 15:04:05")
	fileTime = fmt.Sprintf("%s.%09d", fileTime, now.Nanosecond())
	_, filePath, line, ok := runtime.Caller(logger.depth)
	if ok == false {
		fileName = "xxx"
		line = 0
		//panic(errors.New("get the line failed"))
	}
	//tmp := strings.Split(file, "/")
	//file = tmp[len(tmp)-1]
	_, fileName = path.Split(filePath)

	return fileTime, fileName, line
}

func Debug(format string, args ...interface{}) {
	if DebugLevel < logger.logLevel {
		return
	}
	logger.logFormat(DebugLevel, fmt.Sprintf(format, args...))
}

func Info(format string, args ...interface{}) {
	if InfoLevel < logger.logLevel {
		return
	}

	logger.logFormat(InfoLevel, fmt.Sprintf(format, args...))
}

func Error(format string, args ...interface{}) {
	if ErrorLevel < logger.logLevel {
		return
	}

	logger.logFormat(ErrorLevel, fmt.Sprintf(format, args...))
}

func Trace(format string, args ...interface{}) {
	if TraceLevel < logger.logLevel {
		return
	}

	logger.logFormat(TraceLevel, fmt.Sprintf(format, args...))
}

func Fatal(format string, args ...interface{}) {
	if FatalLevel < logger.logLevel {
		return
	}

	logger.logFormat(FatalLevel, fmt.Sprintf(format, args...))
	os.Exit(-1)
}
<|endoftext|>


<commit_before>// The watcher package provides an interface for observing changes
// to arbitrary MongoDB documents that are maintained via the
// mgo/txn transaction package.
package watcher

import (
	"fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"launchpad.net/juju-core/log"
	"launchpad.net/tomb"
	"time"
)

// Debug specifies whether the package will log debug
// messages.
// TODO(rog) allow debug level setting in the log package.
var Debug = false

// A Watcher can watch any number of collections and documents for changes.
type Watcher struct {
	tomb tomb.Tomb
	log  *mgo.Collection

	// watches holds the observers managed by Watch/Unwatch.
	watches map[watchKey][]watchInfo

	// current holds the current txn-revno values for all the observed
	// documents known to exist. Documents not observed or deleted are
	// omitted from this map and are considered to have revno -1.
	current map[watchKey]int64

	// syncEvents and requestEvents contain the events to be
	// dispatched to the watcher channels. They're queued during
	// processing and flushed at the end to simplify the algorithm.
	// The two queues are separated because events from sync are
	// handled in reverse order due to the way the algorithm works.
	syncEvents, requestEvents []event

	// request is used to deliver requests from the public API into
	// the the goroutine loop.
	request chan interface{}

	// syncDone contains pending done channels from sync requests.
	syncDone []chan bool

	// lastId is the most recent transaction id observed by a sync.
	lastId interface{}

	// next will dispatch when it's time to sync the database
	// knowledge. It's maintained here so that Sync and StartSync
	// can manipulate it to force a sync sooner.
	next <-chan time.Time
}

// A Change holds information about a document change.
type Change struct {
	// C and Id hold the collection name and document _id field value.
	C  string
	Id interface{}

	// Revno is the latest known value for the document's txn-revno
	// field, or -1 if the document was deleted.
	Revno int64
}

type watchKey struct {
	c  string
	id interface{} // nil when watching collection
}

func (k watchKey) String() string {
	coll := "collection " + k.c
	if k.id == nil {
		return coll
	}
	return fmt.Sprintf("document %v in %s", k.id, coll)
}

// match returns whether the receiving watch key,
// which may refer to a particular item or
// an entire collection, matches k1, which refers
// to a particular item.
func (k watchKey) match(k1 watchKey) bool {
	if k.c != k1.c {
		return false
	}
	if k.id == nil {
		// k refers to entire collection
		return true
	}
	return k.id == k1.id
}

type watchInfo struct {
	ch    chan<- Change
	revno int64
}

type event struct {
	ch    chan<- Change
	key   watchKey
	revno int64
}

// New returns a new Watcher observing the changelog collection,
// which must be a capped collection maintained by mgo/txn.
func New(changelog *mgo.Collection) *Watcher {
	w := &Watcher{
		log:     changelog,
		watches: make(map[watchKey][]watchInfo),
		current: make(map[watchKey]int64),
		request: make(chan interface{}),
	}
	go func() {
		w.tomb.Kill(w.loop())
		w.tomb.Done()
	}()
	return w
}

// Stop stops all the watcher activities.
func (w *Watcher) Stop() error {
	w.tomb.Kill(nil)
	return w.tomb.Wait()
}

// Dead returns a channel that is closed when the watcher has stopped.
func (w *Watcher) Dead() <-chan struct{} {
	return w.tomb.Dead()
}

// Err returns the error with which the watcher stopped.
// It returns nil if the watcher stopped cleanly, tomb.ErrStillAlive
// if the watcher is still running properly, or the respective error
// if the watcher is terminating or has terminated with an error.
func (w *Watcher) Err() error {
	return w.tomb.Err()
}

type reqWatch struct {
	key  watchKey
	info watchInfo
}

type reqUnwatch struct {
	key watchKey
	ch  chan<- Change
}

type reqSync struct {
	done chan bool
}

func (w *Watcher) sendReq(req interface{}) {
	select {
	case w.request <- req:
	case <-w.tomb.Dying():
	}
}

// Watch starts watching the given collection and document id.
// An event will be sent onto ch whenever a matching document's txn-revno
// field is observed to change after a transaction is applied. The revno
// parameter holds the currently known revision number for the document.
// Non-existent documents are represented by a -1 revno.
func (w *Watcher) Watch(collection string, id interface{}, revno int64, ch chan<- Change) {
	if id == nil {
		panic("watcher: cannot watch a document with nil id")
	}
	w.sendReq(reqWatch{watchKey{collection, id}, watchInfo{ch, revno}})
}

// WatchCollection starts watching the given collection.
// An event will be sent onto ch whenever the txn-revno field is observed
// to change after a transaction is applied for any document in the collection.
func (w *Watcher) WatchCollection(collection string, ch chan<- Change) {
	w.sendReq(reqWatch{watchKey{collection, nil}, watchInfo{ch, 0}})
}

// Unwatch stops watching the given collection and document id via ch.
func (w *Watcher) Unwatch(collection string, id interface{}, ch chan<- Change) {
	if id == nil {
		panic("watcher: cannot unwatch a document with nil id")
	}
	w.sendReq(reqUnwatch{watchKey{collection, id}, ch})
}

// UnwatchCollection stops watching the given collection via ch.
func (w *Watcher) UnwatchCollection(collection string, ch chan<- Change) {
	w.sendReq(reqUnwatch{watchKey{collection, nil}, ch})
}

// StartSync forces the watcher to load new events from the database.
func (w *Watcher) StartSync() {
	w.sendReq(reqSync{nil})
}

// Sync forces the watcher to load new events from the database and blocks
// until all events have been dispatched.
func (w *Watcher) Sync() {
	done := make(chan bool)
	w.sendReq(reqSync{done})
	select {
	case <-done:
	case <-w.tomb.Dying():
	}
}

// Period is the delay between each sync.
// It must not be changed when any watchers are active.
var Period time.Duration = 5 * time.Second

// loop implements the main watcher loop.
func (w *Watcher) loop() error {
	w.next = time.After(0)
	if err := w.initLastId(); err != nil {
		return err
	}
	for {
		select {
		case <-w.tomb.Dying():
			return tomb.ErrDying
		case <-w.next:
			w.next = time.After(Period)
			syncDone := w.syncDone
			w.syncDone = nil
			if err := w.sync(); err != nil {
				return err
			}
			w.flush()
			for _, done := range syncDone {
				close(done)
			}
		case req := <-w.request:
			w.handle(req)
			w.flush()
		}
	}
	panic("not reached")
}

// flush sends all pending events to their respective channels.
func (w *Watcher) flush() {
	// refreshEvents are stored newest first.
	for i := len(w.syncEvents) - 1; i >= 0; i-- {
		e := &w.syncEvents[i]
		for e.ch != nil {
			select {
			case <-w.tomb.Dying():
				return
			case req := <-w.request:
				w.handle(req)
				continue
			case e.ch <- Change{e.key.c, e.key.id, e.revno}:
			}
			break
		}
	}
	// requestEvents are stored oldest first, and
	// may grow during the loop.
	for i := 0; i < len(w.requestEvents); i++ {
		e := &w.requestEvents[i]
		for e.ch != nil {
			select {
			case <-w.tomb.Dying():
				return
			case req := <-w.request:
				w.handle(req)
				continue
			case e.ch <- Change{e.key.c, e.key.id, e.revno}:
			}
			break
		}
	}
	w.syncEvents = w.syncEvents[:0]
	w.requestEvents = w.requestEvents[:0]
}

// handle deals with requests delivered by the public API
// onto the background watcher goroutine.
func (w *Watcher) handle(req interface{}) {
	switch r := req.(type) {
	case reqSync:
		w.next = time.After(0)
		if r.done != nil {
			w.syncDone = append(w.syncDone, r.done)
		}
	case reqWatch:
		for _, info := range w.watches[r.key] {
			if info.ch == r.info.ch {
				panic(fmt.Errorf("tried to re-add channel %v for %s", info.ch, r.key))
			}
		}
		if revno, ok := w.current[r.key]; ok && (revno > r.info.revno || revno == -1 && r.info.revno >= 0) {
			r.info.revno = revno
			w.requestEvents = append(w.requestEvents, event{r.info.ch, r.key, revno})
		}
		w.watches[r.key] = append(w.watches[r.key], r.info)
	case reqUnwatch:
		watches := w.watches[r.key]
		removed := false
		for i, info := range watches {
			if info.ch == r.ch {
				watches[i] = watches[len(watches)-1]
				w.watches[r.key] = watches[:len(watches)-1]
				removed = true
				break
			}
		}
		if !removed {
			panic(fmt.Errorf("tried to remove missing channel %v for %s", r.ch, r.key))
		}
		for i := range w.requestEvents {
			e := &w.requestEvents[i]
			if r.key.match(e.key) && e.ch == r.ch {
				e.ch = nil
			}
		}
		for i := range w.syncEvents {
			e := &w.syncEvents[i]
			if r.key.match(e.key) && e.ch == r.ch {
				e.ch = nil
			}
		}
	default:
		panic(fmt.Errorf("unknown request: %T", req))
	}
}

type logInfo struct {
	Docs   []interface{} `bson:"d"`
	Revnos []int64       `bson:"r"`
}

// initLastId reads the most recent changelog document and initializes
// lastId with it. This causes all history that precedes the creation
// of the watcher to be ignored.
func (w *Watcher) initLastId() error {
	var entry struct {
		Id interface{} "_id"
	}
	err := w.log.Find(nil).Sort("-$natural").One(&entry)
	if err != nil && err != mgo.ErrNotFound {
		return err
	}
	w.lastId = entry.Id
	return nil
}

// sync updates the watcher knowledge from the database, and
// queues events to observing channels.
func (w *Watcher) sync() error {
	// Iterate through log events in reverse insertion order (newest first).
	iter := w.log.Find(nil).Batch(10).Sort("-$natural").Iter()
	seen := make(map[watchKey]bool)
	first := true
	lastId := w.lastId
	var entry bson.D
	for iter.Next(&entry) {
		if len(entry) == 0 {
			debugf("state/watcher: got empty changelog document")
		}
		id := entry[0]
		if id.Name != "_id" {
			panic("watcher: _id field isn't first entry")
		}
		if first {
			w.lastId = id.Value
			first = false
		}
		if id.Value == lastId {
			break
		}
		if Debug {
			debugf("state/watcher: got changelog document: %#v", entry)
		}
		for _, c := range entry[1:] {
			// See txn's Runner.ChangeLog for the structure of log entries.
			var d, r []interface{}
			dr, _ := c.Value.(bson.D)
			for _, item := range dr {
				switch item.Name {
				case "d":
					d, _ = item.Value.([]interface{})
				case "r":
					r, _ = item.Value.([]interface{})
				}
			}
			if len(d) == 0 || len(d) != len(r) {
				log.Warningf("state/watcher: changelog has invalid collection document: %#v", c)
				continue
			}
			for i := len(d) - 1; i >= 0; i-- {
				key := watchKey{c.Name, d[i]}
				if seen[key] {
					continue
				}
				seen[key] = true
				revno, ok := r[i].(int64)
				if !ok {
					log.Warningf("state/watcher: changelog has revno with type %T: %#v", r[i], r[i])
					continue
				}
				if revno < 0 {
					revno = -1
				}
				if w.current[key] == revno {
					continue
				}
				w.current[key] = revno
				// Queue notifications for per-collection watches.
				for _, info := range w.watches[watchKey{c.Name, nil}] {
					w.syncEvents = append(w.syncEvents, event{info.ch, key, revno})
				}
				// Queue notifications for per-document watches.
				infos := w.watches[key]
				for i, info := range infos {
					if revno > info.revno || revno < 0 && info.revno >= 0 {
						infos[i].revno = revno
						w.syncEvents = append(w.syncEvents, event{info.ch, key, revno})
					}
				}
			}
		}
	}
	if iter.Err() != nil {
		return fmt.Errorf("watcher iteration error: %v", iter.Err())
	}
	return nil
}

func debugf(f string, a ...interface{}) {
	if Debug {
		log.Debugf(f, a...)
	}
}
<commit_msg>state/watcher: remove redundant test<commit_after>// The watcher package provides an interface for observing changes
// to arbitrary MongoDB documents that are maintained via the
// mgo/txn transaction package.
package watcher

import (
	"fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"launchpad.net/juju-core/log"
	"launchpad.net/tomb"
	"time"
)

// Debug specifies whether the package will log debug
// messages.
// TODO(rog) allow debug level setting in the log package.
var Debug = false

// A Watcher can watch any number of collections and documents for changes.
type Watcher struct {
	tomb tomb.Tomb
	log  *mgo.Collection

	// watches holds the observers managed by Watch/Unwatch.
	watches map[watchKey][]watchInfo

	// current holds the current txn-revno values for all the observed
	// documents known to exist. Documents not observed or deleted are
	// omitted from this map and are considered to have revno -1.
	current map[watchKey]int64

	// syncEvents and requestEvents contain the events to be
	// dispatched to the watcher channels. They're queued during
	// processing and flushed at the end to simplify the algorithm.
	// The two queues are separated because events from sync are
	// handled in reverse order due to the way the algorithm works.
	syncEvents, requestEvents []event

	// request is used to deliver requests from the public API into
	// the the goroutine loop.
	request chan interface{}

	// syncDone contains pending done channels from sync requests.
	syncDone []chan bool

	// lastId is the most recent transaction id observed by a sync.
	lastId interface{}

	// next will dispatch when it's time to sync the database
	// knowledge. It's maintained here so that Sync and StartSync
	// can manipulate it to force a sync sooner.
	next <-chan time.Time
}

// A Change holds information about a document change.
type Change struct {
	// C and Id hold the collection name and document _id field value.
	C  string
	Id interface{}

	// Revno is the latest known value for the document's txn-revno
	// field, or -1 if the document was deleted.
	Revno int64
}

type watchKey struct {
	c  string
	id interface{} // nil when watching collection
}

func (k watchKey) String() string {
	coll := "collection " + k.c
	if k.id == nil {
		return coll
	}
	return fmt.Sprintf("document %v in %s", k.id, coll)
}

// match returns whether the receiving watch key,
// which may refer to a particular item or
// an entire collection, matches k1, which refers
// to a particular item.
func (k watchKey) match(k1 watchKey) bool {
	if k.c != k1.c {
		return false
	}
	if k.id == nil {
		// k refers to entire collection
		return true
	}
	return k.id == k1.id
}

type watchInfo struct {
	ch    chan<- Change
	revno int64
}

type event struct {
	ch    chan<- Change
	key   watchKey
	revno int64
}

// New returns a new Watcher observing the changelog collection,
// which must be a capped collection maintained by mgo/txn.
func New(changelog *mgo.Collection) *Watcher {
	w := &Watcher{
		log:     changelog,
		watches: make(map[watchKey][]watchInfo),
		current: make(map[watchKey]int64),
		request: make(chan interface{}),
	}
	go func() {
		w.tomb.Kill(w.loop())
		w.tomb.Done()
	}()
	return w
}

// Stop stops all the watcher activities.
func (w *Watcher) Stop() error {
	w.tomb.Kill(nil)
	return w.tomb.Wait()
}

// Dead returns a channel that is closed when the watcher has stopped.
func (w *Watcher) Dead() <-chan struct{} {
	return w.tomb.Dead()
}

// Err returns the error with which the watcher stopped.
// It returns nil if the watcher stopped cleanly, tomb.ErrStillAlive
// if the watcher is still running properly, or the respective error
// if the watcher is terminating or has terminated with an error.
func (w *Watcher) Err() error {
	return w.tomb.Err()
}

type reqWatch struct {
	key  watchKey
	info watchInfo
}

type reqUnwatch struct {
	key watchKey
	ch  chan<- Change
}

type reqSync struct {
	done chan bool
}

func (w *Watcher) sendReq(req interface{}) {
	select {
	case w.request <- req:
	case <-w.tomb.Dying():
	}
}

// Watch starts watching the given collection and document id.
// An event will be sent onto ch whenever a matching document's txn-revno
// field is observed to change after a transaction is applied. The revno
// parameter holds the currently known revision number for the document.
// Non-existent documents are represented by a -1 revno.
func (w *Watcher) Watch(collection string, id interface{}, revno int64, ch chan<- Change) {
	if id == nil {
		panic("watcher: cannot watch a document with nil id")
	}
	w.sendReq(reqWatch{watchKey{collection, id}, watchInfo{ch, revno}})
}

// WatchCollection starts watching the given collection.
// An event will be sent onto ch whenever the txn-revno field is observed
// to change after a transaction is applied for any document in the collection.
func (w *Watcher) WatchCollection(collection string, ch chan<- Change) {
	w.sendReq(reqWatch{watchKey{collection, nil}, watchInfo{ch, 0}})
}

// Unwatch stops watching the given collection and document id via ch.
func (w *Watcher) Unwatch(collection string, id interface{}, ch chan<- Change) {
	if id == nil {
		panic("watcher: cannot unwatch a document with nil id")
	}
	w.sendReq(reqUnwatch{watchKey{collection, id}, ch})
}

// UnwatchCollection stops watching the given collection via ch.
func (w *Watcher) UnwatchCollection(collection string, ch chan<- Change) {
	w.sendReq(reqUnwatch{watchKey{collection, nil}, ch})
}

// StartSync forces the watcher to load new events from the database.
func (w *Watcher) StartSync() {
	w.sendReq(reqSync{nil})
}

// Sync forces the watcher to load new events from the database and blocks
// until all events have been dispatched.
func (w *Watcher) Sync() {
	done := make(chan bool)
	w.sendReq(reqSync{done})
	select {
	case <-done:
	case <-w.tomb.Dying():
	}
}

// Period is the delay between each sync.
// It must not be changed when any watchers are active.
var Period time.Duration = 5 * time.Second

// loop implements the main watcher loop.
func (w *Watcher) loop() error {
	w.next = time.After(0)
	if err := w.initLastId(); err != nil {
		return err
	}
	for {
		select {
		case <-w.tomb.Dying():
			return tomb.ErrDying
		case <-w.next:
			w.next = time.After(Period)
			syncDone := w.syncDone
			w.syncDone = nil
			if err := w.sync(); err != nil {
				return err
			}
			w.flush()
			for _, done := range syncDone {
				close(done)
			}
		case req := <-w.request:
			w.handle(req)
			w.flush()
		}
	}
	panic("not reached")
}

// flush sends all pending events to their respective channels.
func (w *Watcher) flush() {
	// refreshEvents are stored newest first.
	for i := len(w.syncEvents) - 1; i >= 0; i-- {
		e := &w.syncEvents[i]
		for e.ch != nil {
			select {
			case <-w.tomb.Dying():
				return
			case req := <-w.request:
				w.handle(req)
				continue
			case e.ch <- Change{e.key.c, e.key.id, e.revno}:
			}
			break
		}
	}
	// requestEvents are stored oldest first, and
	// may grow during the loop.
	for i := 0; i < len(w.requestEvents); i++ {
		e := &w.requestEvents[i]
		for e.ch != nil {
			select {
			case <-w.tomb.Dying():
				return
			case req := <-w.request:
				w.handle(req)
				continue
			case e.ch <- Change{e.key.c, e.key.id, e.revno}:
			}
			break
		}
	}
	w.syncEvents = w.syncEvents[:0]
	w.requestEvents = w.requestEvents[:0]
}

// handle deals with requests delivered by the public API
// onto the background watcher goroutine.
func (w *Watcher) handle(req interface{}) {
	switch r := req.(type) {
	case reqSync:
		w.next = time.After(0)
		if r.done != nil {
			w.syncDone = append(w.syncDone, r.done)
		}
	case reqWatch:
		for _, info := range w.watches[r.key] {
			if info.ch == r.info.ch {
				panic(fmt.Errorf("tried to re-add channel %v for %s", info.ch, r.key))
			}
		}
		if revno, ok := w.current[r.key]; ok && (revno > r.info.revno || revno == -1 && r.info.revno >= 0) {
			r.info.revno = revno
			w.requestEvents = append(w.requestEvents, event{r.info.ch, r.key, revno})
		}
		w.watches[r.key] = append(w.watches[r.key], r.info)
	case reqUnwatch:
		watches := w.watches[r.key]
		removed := false
		for i, info := range watches {
			if info.ch == r.ch {
				watches[i] = watches[len(watches)-1]
				w.watches[r.key] = watches[:len(watches)-1]
				removed = true
				break
			}
		}
		if !removed {
			panic(fmt.Errorf("tried to remove missing channel %v for %s", r.ch, r.key))
		}
		for i := range w.requestEvents {
			e := &w.requestEvents[i]
			if r.key.match(e.key) && e.ch == r.ch {
				e.ch = nil
			}
		}
		for i := range w.syncEvents {
			e := &w.syncEvents[i]
			if r.key.match(e.key) && e.ch == r.ch {
				e.ch = nil
			}
		}
	default:
		panic(fmt.Errorf("unknown request: %T", req))
	}
}

type logInfo struct {
	Docs   []interface{} `bson:"d"`
	Revnos []int64       `bson:"r"`
}

// initLastId reads the most recent changelog document and initializes
// lastId with it. This causes all history that precedes the creation
// of the watcher to be ignored.
func (w *Watcher) initLastId() error {
	var entry struct {
		Id interface{} "_id"
	}
	err := w.log.Find(nil).Sort("-$natural").One(&entry)
	if err != nil && err != mgo.ErrNotFound {
		return err
	}
	w.lastId = entry.Id
	return nil
}

// sync updates the watcher knowledge from the database, and
// queues events to observing channels.
func (w *Watcher) sync() error {
	// Iterate through log events in reverse insertion order (newest first).
	iter := w.log.Find(nil).Batch(10).Sort("-$natural").Iter()
	seen := make(map[watchKey]bool)
	first := true
	lastId := w.lastId
	var entry bson.D
	for iter.Next(&entry) {
		if len(entry) == 0 {
			debugf("state/watcher: got empty changelog document")
		}
		id := entry[0]
		if id.Name != "_id" {
			panic("watcher: _id field isn't first entry")
		}
		if first {
			w.lastId = id.Value
			first = false
		}
		if id.Value == lastId {
			break
		}
		debugf("state/watcher: got changelog document: %#v", entry)
		for _, c := range entry[1:] {
			// See txn's Runner.ChangeLog for the structure of log entries.
			var d, r []interface{}
			dr, _ := c.Value.(bson.D)
			for _, item := range dr {
				switch item.Name {
				case "d":
					d, _ = item.Value.([]interface{})
				case "r":
					r, _ = item.Value.([]interface{})
				}
			}
			if len(d) == 0 || len(d) != len(r) {
				log.Warningf("state/watcher: changelog has invalid collection document: %#v", c)
				continue
			}
			for i := len(d) - 1; i >= 0; i-- {
				key := watchKey{c.Name, d[i]}
				if seen[key] {
					continue
				}
				seen[key] = true
				revno, ok := r[i].(int64)
				if !ok {
					log.Warningf("state/watcher: changelog has revno with type %T: %#v", r[i], r[i])
					continue
				}
				if revno < 0 {
					revno = -1
				}
				if w.current[key] == revno {
					continue
				}
				w.current[key] = revno
				// Queue notifications for per-collection watches.
				for _, info := range w.watches[watchKey{c.Name, nil}] {
					w.syncEvents = append(w.syncEvents, event{info.ch, key, revno})
				}
				// Queue notifications for per-document watches.
				infos := w.watches[key]
				for i, info := range infos {
					if revno > info.revno || revno < 0 && info.revno >= 0 {
						infos[i].revno = revno
						w.syncEvents = append(w.syncEvents, event{info.ch, key, revno})
					}
				}
			}
		}
	}
	if iter.Err() != nil {
		return fmt.Errorf("watcher iteration error: %v", iter.Err())
	}
	return nil
}

func debugf(f string, a ...interface{}) {
	if Debug {
		log.Debugf(f, a...)
	}
}
<|endoftext|>