<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

const logo = `
 8888888           .d888 888                   8888888b.  888888b.
   888            d88P"  888                   888  "Y88b 888  "88b
   888            888    888                   888    888 888  .88P
   888   88888b.  888888 888 888  888 888  888 888    888 8888888K.
   888   888 "88b 888    888 888  888  Y8bd8P' 888    888 888  "Y88b
   888   888  888 888    888 888  888   X88K   888    888 888    888
   888   888  888 888    888 Y88b 888 .d8""8b. 888  .d88P 888   d88P
 8888888 888  888 888    888  "Y88888 888  888 8888888P"  8888888P"

`

// These variables are populated via the Go linker.
var (
	version string = "0.9"
	commit  string
)

// Various constants used by the main package.
const (
	messagingClientFile string = "messaging"
)

func main() {
	log.SetFlags(0)
	log.SetPrefix(`[srvr] `)
	log.SetFlags(log.LstdFlags)
	rand.Seed(time.Now().UnixNano())

	// If commit not set, make that clear.
	if commit == "" {
		commit = "unknown"
	}

	// Set parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())
	log.Printf("GOMAXPROCS set to %d", runtime.GOMAXPROCS(0))

	// Shift binary name off argument list.
	args := os.Args[1:]

	// Retrieve command name as first argument.
	var cmd string
	if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
		cmd = args[0]
	}

	// Special case -h immediately following binary name
	if len(args) > 0 && args[0] == "-h" {
		cmd = "help"
	}

	// If command is "help" and has an argument then rewrite args to use "-h".
	if cmd == "help" && len(args) > 1 {
		args[0], args[1] = args[1], "-h"
		cmd = args[0]
	}

	// Extract name from args.
	switch cmd {
	case "run":
		execRun(args[1:])
	case "":
		execRun(args)
	case "backup":
		cmd := NewBackupCommand()
		if err := cmd.Run(args[1:]...); err != nil {
			log.Fatalf("backup: %s", err)
		}
	case "restore":
		cmd := NewRestoreCommand()
		if err := cmd.Run(args[1:]...); err != nil {
			log.Fatalf("restore: %s", err)
		}
	case "version":
		execVersion(args[1:])
	case "config":
		execConfig(args[1:])
	case "help":
		execHelp(args[1:])
	default:
		log.Fatalf(`influxd: unknown command "%s"`+"\n"+`Run 'influxd help' for usage`+"\n\n", cmd)
	}
}

// execRun runs the "run" command.
func execRun(args []string) {
	// Parse command flags.
	fs := flag.NewFlagSet("", flag.ExitOnError)
	var (
		configPath = fs.String("config", "", "")
		pidPath    = fs.String("pidfile", "", "")
		hostname   = fs.String("hostname", "", "")
		join       = fs.String("join", "", "")
		cpuprofile = fs.String("cpuprofile", "", "")
		memprofile = fs.String("memprofile", "", "")
	)
	fs.Usage = printRunUsage
	fs.Parse(args)

	// Start profiling, if set.
	startProfiling(*cpuprofile, *memprofile)
	defer stopProfiling()

	// Print sweet InfluxDB logo and write the process id to file.
	fmt.Print(logo)
	writePIDFile(*pidPath)

	// Parse configuration file from disk.
	config, err := parseConfig(*configPath, *hostname)
	if err != nil {
		log.Fatal(err)
	} else if *configPath == "" {
		log.Println("No config provided, using default settings")
	}

	Run(config, *join, version)

	// Wait indefinitely.
	<-(chan struct{})(nil)
}

// execVersion runs the "version" command.
// Prints the commit SHA1 if set by the build process.
func execVersion(args []string) {
	fs := flag.NewFlagSet("", flag.ExitOnError)
	fs.Usage = func() {
		log.Println(`usage: version

	version displays the InfluxDB version and build git commit hash
	`)
	}
	fs.Parse(args)

	s := fmt.Sprintf("InfluxDB v%s", version)
	if commit != "" {
		s += fmt.Sprintf(" (git: %s)", commit)
	}
	log.Print(s)
}

// execConfig parses and prints the current config loaded.
func execConfig(args []string) {
	// Parse command flags.
	fs := flag.NewFlagSet("", flag.ExitOnError)
	fs.Usage = func() {
		log.Println(`usage: config

	config displays the default configiguration
						    `)
	}

	var (
		configPath = fs.String("config", "", "")
		hostname   = fs.String("hostname", "", "")
	)
	fs.Parse(args)

	config, err := parseConfig(*configPath, *hostname)
	if err != nil {
		log.Fatalf("parse config: %s", err)
	}

	config.Write(os.Stdout)
}

// execHelp runs the "help" command.
func execHelp(args []string) {
	fmt.Println(`
Configure and start an InfluxDB server.

Usage:

	influxd [[command] [arguments]]

The commands are:

    config               display the default configuration
    join-cluster         create a new node that will join an existing cluster
    run                  run node with existing configuration
    version              displays the InfluxDB version

"run" is the default command.

Use "influxd help [command]" for more information about a command.
`)
}

type Stopper interface {
	Stop()
}

type State struct {
	Mode string `json:"mode"`
}

var prof struct {
	cpu *os.File
	mem *os.File
}

func startProfiling(cpuprofile, memprofile string) {
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatalf("cpuprofile: %v", err)
		}
		prof.cpu = f
		pprof.StartCPUProfile(prof.cpu)
	}

	if memprofile != "" {
		f, err := os.Create(memprofile)
		if err != nil {
			log.Fatalf("memprofile: %v", err)
		}
		prof.mem = f
		runtime.MemProfileRate = 4096
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		<-c
		stopProfiling()
		os.Exit(0)
	}()
}

func stopProfiling() {
	if prof.cpu != nil {
		pprof.StopCPUProfile()
		prof.cpu.Close()
	}
	if prof.mem != nil {
		pprof.Lookup("heap").WriteTo(prof.mem, 0)
		prof.mem.Close()
	}
}

func warn(v ...interface{})              { fmt.Fprintln(os.Stderr, v...) }
func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) }
<commit_msg>Set Go Max procs in a better location<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

const logo = `
 8888888           .d888 888                   8888888b.  888888b.
   888            d88P"  888                   888  "Y88b 888  "88b
   888            888    888                   888    888 888  .88P
   888   88888b.  888888 888 888  888 888  888 888    888 8888888K.
   888   888 "88b 888    888 888  888  Y8bd8P' 888    888 888  "Y88b
   888   888  888 888    888 888  888   X88K   888    888 888    888
   888   888  888 888    888 Y88b 888 .d8""8b. 888  .d88P 888   d88P
 8888888 888  888 888    888  "Y88888 888  888 8888888P"  8888888P"

`

// These variables are populated via the Go linker.
var (
	version string = "0.9"
	commit  string
)

// Various constants used by the main package.
const (
	messagingClientFile string = "messaging"
)

func main() {
	log.SetFlags(0)
	log.SetPrefix(`[srvr] `)
	log.SetFlags(log.LstdFlags)
	rand.Seed(time.Now().UnixNano())

	// If commit not set, make that clear.
	if commit == "" {
		commit = "unknown"
	}

	// Shift binary name off argument list.
	args := os.Args[1:]

	// Retrieve command name as first argument.
	var cmd string
	if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
		cmd = args[0]
	}

	// Special case -h immediately following binary name
	if len(args) > 0 && args[0] == "-h" {
		cmd = "help"
	}

	// If command is "help" and has an argument then rewrite args to use "-h".
	if cmd == "help" && len(args) > 1 {
		args[0], args[1] = args[1], "-h"
		cmd = args[0]
	}

	// Extract name from args.
	switch cmd {
	case "run":
		execRun(args[1:])
	case "":
		execRun(args)
	case "backup":
		setGoMaxProcs()
		cmd := NewBackupCommand()
		if err := cmd.Run(args[1:]...); err != nil {
			log.Fatalf("backup: %s", err)
		}
	case "restore":
		setGoMaxProcs()
		cmd := NewRestoreCommand()
		if err := cmd.Run(args[1:]...); err != nil {
			log.Fatalf("restore: %s", err)
		}
	case "version":
		execVersion(args[1:])
	case "config":
		execConfig(args[1:])
	case "help":
		execHelp(args[1:])
	default:
		log.Fatalf(`influxd: unknown command "%s"`+"\n"+`Run 'influxd help' for usage`+"\n\n", cmd)
	}
}

func setGoMaxProcs() {
	// Set parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())
	log.Printf("GOMAXPROCS set to %d", runtime.GOMAXPROCS(0))
}

// execRun runs the "run" command.
func execRun(args []string) {
	// Parse command flags.
	fs := flag.NewFlagSet("", flag.ExitOnError)
	var (
		configPath = fs.String("config", "", "")
		pidPath    = fs.String("pidfile", "", "")
		hostname   = fs.String("hostname", "", "")
		join       = fs.String("join", "", "")
		cpuprofile = fs.String("cpuprofile", "", "")
		memprofile = fs.String("memprofile", "", "")
	)
	fs.Usage = printRunUsage
	fs.Parse(args)

	// Start profiling, if set.
	startProfiling(*cpuprofile, *memprofile)
	defer stopProfiling()

	// Print sweet InfluxDB logo and write the process id to file.
	fmt.Print(logo)
	writePIDFile(*pidPath)

	setGoMaxProcs()

	// Parse configuration file from disk.
	config, err := parseConfig(*configPath, *hostname)
	if err != nil {
		log.Fatal(err)
	} else if *configPath == "" {
		log.Println("No config provided, using default settings")
	}

	Run(config, *join, version)

	// Wait indefinitely.
	<-(chan struct{})(nil)
}

// execVersion runs the "version" command.
// Prints the commit SHA1 if set by the build process.
func execVersion(args []string) {
	fs := flag.NewFlagSet("", flag.ExitOnError)
	fs.Usage = func() {
		log.Println(`usage: version

	version displays the InfluxDB version and build git commit hash
	`)
	}
	fs.Parse(args)

	s := fmt.Sprintf("InfluxDB v%s", version)
	if commit != "" {
		s += fmt.Sprintf(" (git: %s)", commit)
	}
	log.Print(s)
}

// execConfig parses and prints the current config loaded.
func execConfig(args []string) {
	// Parse command flags.
	fs := flag.NewFlagSet("", flag.ExitOnError)
	fs.Usage = func() {
		log.Println(`usage: config

	config displays the default configiguration
						    `)
	}

	var (
		configPath = fs.String("config", "", "")
		hostname   = fs.String("hostname", "", "")
	)
	fs.Parse(args)

	config, err := parseConfig(*configPath, *hostname)
	if err != nil {
		log.Fatalf("parse config: %s", err)
	}

	config.Write(os.Stdout)
}

// execHelp runs the "help" command.
func execHelp(args []string) {
	fmt.Println(`
Configure and start an InfluxDB server.

Usage:

	influxd [[command] [arguments]]

The commands are:

    config               display the default configuration
    join-cluster         create a new node that will join an existing cluster
    run                  run node with existing configuration
    version              displays the InfluxDB version

"run" is the default command.

Use "influxd help [command]" for more information about a command.
`)
}

type Stopper interface {
	Stop()
}

type State struct {
	Mode string `json:"mode"`
}

var prof struct {
	cpu *os.File
	mem *os.File
}

func startProfiling(cpuprofile, memprofile string) {
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatalf("cpuprofile: %v", err)
		}
		prof.cpu = f
		pprof.StartCPUProfile(prof.cpu)
	}

	if memprofile != "" {
		f, err := os.Create(memprofile)
		if err != nil {
			log.Fatalf("memprofile: %v", err)
		}
		prof.mem = f
		runtime.MemProfileRate = 4096
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		<-c
		stopProfiling()
		os.Exit(0)
	}()
}

func stopProfiling() {
	if prof.cpu != nil {
		pprof.StopCPUProfile()
		prof.cpu.Close()
	}
	if prof.mem != nil {
		pprof.Lookup("heap").WriteTo(prof.mem, 0)
		prof.mem.Close()
	}
}

func warn(v ...interface{})              { fmt.Fprintln(os.Stderr, v...) }
func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) }
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Bram Gruneir

// Monitor is a tool designed to occasionally poll an active cluster and save
// the status to disk. The monitor program will exit if the status of the
// cluster can not be determined.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/cockroachdb/cockroach/base"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/retry"
)

var outputDir = flag.String("dir", ".", "Directory in which to output the status logs to.")
var interval = flag.Duration("interval", 10*time.Second, "Interval in which to poll the cluster's status.")
var addr = flag.String("addr", ":26257", "The host:port of the cockroach cluster.")
var insecure = flag.Bool("insecure", false, "True if using an insecure connection.")
var user = flag.String("user", security.RootUser, "User used to connect to the cluster.")
var certs = flag.String("certs", "certs", "Directory containing RSA key and x509 certs. This flag is required if --insecure=false.")
var endpoint = flag.String("endpoint", "_status/nodes", "Status endpoint to monitor and log.")

var retryOptions = retry.Options{
	InitialBackoff: 100 * time.Millisecond,
	MaxRetries:     10,
	Multiplier:     2,
}

// request returns the result of performing a http get request.
func request(url string, httpClient *http.Client) ([]byte, bool) {
	for r := retry.Start(retryOptions); r.Next(); {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			log.Fatal(err)
			return nil, false
		}
		req.Header.Set(util.AcceptHeader, util.JSONContentType)
		resp, err := httpClient.Do(req)
		if err != nil {
			log.Infof("could not GET %s - %s", url, err)
			continue
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Infof("could not ready body for %s - %s", url, err)
			continue
		}
		if resp.StatusCode != http.StatusOK {
			log.Infof("could not GET %s - statuscode: %d - body: %s", url, resp.StatusCode, body)
			continue
		}
		returnedContentType := resp.Header.Get(util.ContentTypeHeader)
		if returnedContentType != util.JSONContentType {
			log.Infof("unexpected content type: %v", returnedContentType)
			continue
		}
		log.Infof("OK response from %s", url)
		return body, true
	}
	log.Warningf("There was an error retrieving %s", url)
	return nil, false
}

func main() {
	flag.Parse()

	ctx := base.Context{Insecure: *insecure, Certs: *certs, User: *user}
	httpClient, err := ctx.GetHTTPClient()
	if err != nil {
		panic(err)
	}

	startTime := time.Now()
	file := filepath.Join(*outputDir, fmt.Sprintf("monitor.%s", strings.Replace(
		startTime.Format(time.RFC3339), ":", "_", -1)))
	log.Infof("Logging cluster status to: %s.\n", file)
	w, err := os.Create(file)
	if err != nil {
		panic(err)
	}
	defer w.Close()

	url := fmt.Sprintf("%s://%s/%s", ctx.HTTPRequestScheme(), *addr, *endpoint)
	log.Infof("Cluster Status URL: %s\n", url)

	for range time.Tick(*interval) {
		resp, found := request(url, httpClient)
		if !found {
			log.Warningf("Could not get cluster status. Time since monitor started %s.", time.Since(startTime))
			break
		}
		log.Infof("Got cluster status.")
		fmt.Fprintf(w, "%s\n", resp)
	}
}
<commit_msg>Status Monitor can monitor Multiple Servers<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Bram Gruneir

// Monitor is a tool designed to occasionally poll an active cluster and save
// the status to disk. The monitor program will exit if the status of the
// cluster can not be determined.

package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/cockroachdb/cockroach/base"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/retry"
	"github.com/cockroachdb/cockroach/util/stop"
)

const (
	// urlPath is the http path of the status server on each node.
	urlPath = "_status/details/local"
)

var interval = flag.Duration("interval", 10*time.Second, "Interval in which to poll each services's status.")
var addrs = flag.String("addrs", ":26257", "Comma-separated list of host:port addressess to monitor.")
var insecure = flag.Bool("insecure", false, "True if using an insecure connection.")
var user = flag.String("user", security.RootUser, "User used to connect to the cluster.")
var certs = flag.String("certs", "certs", "Directory containing RSA key and x509 certs. This flag is required if --insecure=false.")

var retryOptions = retry.Options{
	InitialBackoff: 100 * time.Millisecond,
	MaxRetries:     10,
	Multiplier:     2,
}

type statusMonitor struct {
	addr       string
	url        string
	httpClient *http.Client
	file       *io.Writer
}

func newStatusMonitor(context *base.Context, addr string) (*statusMonitor, error) {
	monitor := &statusMonitor{
		addr: addr,
	}
	var err error
	monitor.httpClient, err = context.GetHTTPClient()
	if err != nil {
		return nil, err
	}
	monitor.url = fmt.Sprintf("%s://%s/%s", context.HTTPRequestScheme(), monitor.addr, urlPath)
	return monitor, nil
}

func (m *statusMonitor) queryStatus() error {
	var queryErr error
	for r := retry.Start(retryOptions); r.Next(); {
		if log.V(1) && queryErr != nil {
			log.Infof("retrying after error: %s", queryErr)
		}

		// Construct a new HTTP GET Request.
		req, err := http.NewRequest("GET", m.url, nil)
		if err != nil {
			queryErr = fmt.Errorf("could not create http request for %s: %s", m.url, err)
			// Break immediately, this is not recoverable.
			break
		}
		req.Header.Set(util.AcceptHeader, util.JSONContentType)

		// Execute request.
		resp, err := m.httpClient.Do(req)
		if err != nil {
			queryErr = fmt.Errorf("could not GET %s - %s", m.url, err)
			continue
		}
		defer resp.Body.Close()

		// Read and verify body of response.
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			queryErr = fmt.Errorf("could not ready body for %s - %s", m.url, err)
			continue
		}
		if resp.StatusCode != http.StatusOK {
			queryErr = fmt.Errorf("could not GET %s - statuscode: %d - body: %s", m.url, resp.StatusCode, body)
			continue
		}
		returnedContentType := resp.Header.Get(util.ContentTypeHeader)
		if returnedContentType != util.JSONContentType {
			queryErr = fmt.Errorf("unexpected content type: %v", returnedContentType)
			continue
		}
		return nil
	}
	return queryErr
}

func main() {
	flag.Parse()
	parsedAddrs := strings.Split(*addrs, ",")

	ctx := base.Context{Insecure: *insecure, Certs: *certs, User: *user}

	startTime := time.Now()
	stopper := stop.NewStopper()
	for _, addr := range parsedAddrs {
		client, err := newStatusMonitor(&ctx, addr)
		if err != nil {
			log.Errorf("error creating client: %s", err)
			return
		}
		log.Infof("Monitoring Status URL: %s", client.url)
		stopper.RunWorker(func() {
			timer := time.Tick(*interval)
			for {
				select {
				case <-stopper.ShouldStop():
					return
				case <-timer:
					elapsed := time.Since(startTime)
					if err := client.queryStatus(); err != nil {
						log.Warningf("Could not get status from url %s. Time since monitor started %s.", client.url, elapsed)
						stopper.Stop()
						continue
					}
					log.Infof("Got status from url %s. Time since start: %s", client.url, elapsed)
				}
			}
		})
	}

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt, os.Kill, syscall.SIGTERM)
	// Block until a termination signal is received, or the stopper is closed by
	// an error in one of the client routines.
	select {
	case <-stopper.ShouldStop():
		log.Infof("Monitor stopped by error...")
	case <-signalCh:
		log.Infof("Stopping status monitor...")
		stopper.Stop()
	}
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"socialapi/models"

	// "fmt"
	"time"

	"github.com/koding/bongo"
)

type NotificationContent struct {
	// unique identifier of NotificationContent
	Id int64 `json:"id"`

	// target of the activity (replied messageId, followed accountId etc.)
	TargetId int64 `json:"targetId,string"   sql:"NOT NULL"`

	// Type of the NotificationContent
	TypeConstant string `json:"typeConstant" sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// Creation date of the NotificationContent
	CreatedAt time.Time `json:"createdAt"`
}

const (
	// NotificationContent Types
	NotificationContent_TYPE_LIKE    = "like"
	NotificationContent_TYPE_COMMENT = "comment"
	NotificationContent_TYPE_MENTION = "mention"
	NotificationContent_TYPE_PM      = "chat"
)

func (n *NotificationContent) FindByTarget() error {
	s := map[string]interface{}{
		"type_constant": n.TypeConstant,
		"target_id":     n.TargetId,
	}
	q := bongo.NewQS(s)

	return n.One(q)
}

// CreateNotification validates notifier instance and creates a new notification
// with actor activity.
func CreateNotificationContent(i Notifier) (*NotificationContent, error) {
	// first check for type constant and target id
	if i.GetType() == "" {
		return nil, errors.New("Type must be set")
	}

	if i.GetTargetId() == 0 {
		return nil, errors.New("TargetId must be set")
	}

	if i.GetActorId() == 0 {
		return nil, errors.New("ActorId must be set")
	}

	nc, err := ensureNotificationContent(i)
	if err != nil {
		return nil, err
	}

	a := NewNotificationActivity()
	a.NotificationContentId = nc.Id
	a.ActorId = i.GetActorId()
	a.MessageId = i.GetMessageId()

	if err := a.Create(); err != nil {
		return nil, err
	}

	return nc, nil
}

// ensureNotificationContent adds caching layer on top of notification content fetching
func ensureNotificationContent(i Notifier) (*NotificationContent, error) {
	// check for previous NotificationContent create if it does not exist (type:comment targetId:messageId)
	nc, err := Cache.NotificationContent.ByTypeConstantAndTargetID(i.GetType(), i.GetTargetId())
	if err == nil {
		return nc, nil
	}

	nc = NewNotificationContent()
	nc.TypeConstant = i.GetType()
	nc.TargetId = i.GetTargetId()
	if err := nc.Create(); err != nil {
		return nil, err
	}

	// after creating the notificationcontent we can set it to cache for future
	// usage
	if err := Cache.NotificationContent.SetToCache(nc); err != nil {
		return nil, err
	}

	return nc, nil
}

// FetchByIds fetches notification contents with given ids
func (n *NotificationContent) FetchByIds(ids []int64) ([]NotificationContent, error) {
	notificationContents := make([]NotificationContent, 0)
	if err := bongo.B.FetchByIds(n, &notificationContents, ids); err != nil {
		return nil, err
	}
	return notificationContents, nil
}

// FetchMapByIds returns NotificationContent map with given ids
func (n *NotificationContent) FetchMapByIds(ids []int64) (map[int64]NotificationContent, error) {
	ncList, err := n.FetchByIds(ids)
	if err != nil {
		return nil, err
	}

	ncMap := make(map[int64]NotificationContent, 0)
	for _, nc := range ncList {
		ncMap[nc.Id] = nc
	}

	return ncMap, nil
}

func (n *NotificationContent) FetchIdsByTargetId(targetId int64) ([]int64, error) {
	var ids []int64

	query := &bongo.Query{
		Selector: map[string]interface{}{
			"target_id": targetId,
		},
		Pluck: "id",
	}

	return ids, n.Some(&ids, query)
}

// CreateNotificationType creates an instance of notifier subclasses
func CreateNotificationContentType(notificationType string) (Notifier, error) {
	switch notificationType {
	case NotificationContent_TYPE_LIKE:
		return NewInteractionNotification(notificationType), nil
	case NotificationContent_TYPE_COMMENT:
		return NewReplyNotification(), nil
	case NotificationContent_TYPE_MENTION:
		return NewMentionNotification(), nil
	default:
		return nil, errors.New("undefined notification type")
	}

}

func (n *NotificationContent) GetContentType() (Notifier, error) {
	return CreateNotificationContentType(n.TypeConstant)
}

func (n *NotificationContent) GetDefinition() string {
	nt, err := CreateNotificationContentType(n.TypeConstant)
	if err != nil {
		return ""
	}

	return nt.GetDefinition()
}

// DeleteByIds deletes the given id of NotificationContent (same with content id)
func (n *NotificationContent) DeleteByIds(ids ...int64) error {
	if len(ids) == 0 {
		return models.ErrIdIsNotSet
	}

	for _, id := range ids {
		nc := NewNotificationContent()
		if err := nc.ById(id); err != nil {
			// our aim is removing data from DB
			// so if record is not found in database
			// we can ignore this RecordNotFound error
			if err != bongo.RecordNotFound {
				return err
			}
		}

		if err := nc.Delete(); err != nil {
			if err != bongo.RecordNotFound {
				return err
			}
		}

	}

	return nil
}
<commit_msg>socialapi: add multierror support<commit_after>package models

import (
	"errors"
	"socialapi/models"

	// "fmt"
	"time"

	"github.com/hashicorp/go-multierror"
	"github.com/koding/bongo"
)

type NotificationContent struct {
	// unique identifier of NotificationContent
	Id int64 `json:"id"`

	// target of the activity (replied messageId, followed accountId etc.)
	TargetId int64 `json:"targetId,string"   sql:"NOT NULL"`

	// Type of the NotificationContent
	TypeConstant string `json:"typeConstant" sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// Creation date of the NotificationContent
	CreatedAt time.Time `json:"createdAt"`
}

const (
	// NotificationContent Types
	NotificationContent_TYPE_LIKE    = "like"
	NotificationContent_TYPE_COMMENT = "comment"
	NotificationContent_TYPE_MENTION = "mention"
	NotificationContent_TYPE_PM      = "chat"
)

func (n *NotificationContent) FindByTarget() error {
	s := map[string]interface{}{
		"type_constant": n.TypeConstant,
		"target_id":     n.TargetId,
	}
	q := bongo.NewQS(s)

	return n.One(q)
}

// CreateNotification validates notifier instance and creates a new notification
// with actor activity.
func CreateNotificationContent(i Notifier) (*NotificationContent, error) {
	// first check for type constant and target id
	if i.GetType() == "" {
		return nil, errors.New("Type must be set")
	}

	if i.GetTargetId() == 0 {
		return nil, errors.New("TargetId must be set")
	}

	if i.GetActorId() == 0 {
		return nil, errors.New("ActorId must be set")
	}

	nc, err := ensureNotificationContent(i)
	if err != nil {
		return nil, err
	}

	a := NewNotificationActivity()
	a.NotificationContentId = nc.Id
	a.ActorId = i.GetActorId()
	a.MessageId = i.GetMessageId()

	if err := a.Create(); err != nil {
		return nil, err
	}

	return nc, nil
}

// ensureNotificationContent adds caching layer on top of notification content fetching
func ensureNotificationContent(i Notifier) (*NotificationContent, error) {
	// check for previous NotificationContent create if it does not exist (type:comment targetId:messageId)
	nc, err := Cache.NotificationContent.ByTypeConstantAndTargetID(i.GetType(), i.GetTargetId())
	if err == nil {
		return nc, nil
	}

	nc = NewNotificationContent()
	nc.TypeConstant = i.GetType()
	nc.TargetId = i.GetTargetId()
	if err := nc.Create(); err != nil {
		return nil, err
	}

	// after creating the notificationcontent we can set it to cache for future
	// usage
	if err := Cache.NotificationContent.SetToCache(nc); err != nil {
		return nil, err
	}

	return nc, nil
}

// FetchByIds fetches notification contents with given ids
func (n *NotificationContent) FetchByIds(ids []int64) ([]NotificationContent, error) {
	notificationContents := make([]NotificationContent, 0)
	if err := bongo.B.FetchByIds(n, &notificationContents, ids); err != nil {
		return nil, err
	}
	return notificationContents, nil
}

// FetchMapByIds returns NotificationContent map with given ids
func (n *NotificationContent) FetchMapByIds(ids []int64) (map[int64]NotificationContent, error) {
	ncList, err := n.FetchByIds(ids)
	if err != nil {
		return nil, err
	}

	ncMap := make(map[int64]NotificationContent, 0)
	for _, nc := range ncList {
		ncMap[nc.Id] = nc
	}

	return ncMap, nil
}

func (n *NotificationContent) FetchIdsByTargetId(targetId int64) ([]int64, error) {
	var ids []int64

	query := &bongo.Query{
		Selector: map[string]interface{}{
			"target_id": targetId,
		},
		Pluck: "id",
	}

	return ids, n.Some(&ids, query)
}

// CreateNotificationType creates an instance of notifier subclasses
func CreateNotificationContentType(notificationType string) (Notifier, error) {
	switch notificationType {
	case NotificationContent_TYPE_LIKE:
		return NewInteractionNotification(notificationType), nil
	case NotificationContent_TYPE_COMMENT:
		return NewReplyNotification(), nil
	case NotificationContent_TYPE_MENTION:
		return NewMentionNotification(), nil
	default:
		return nil, errors.New("undefined notification type")
	}

}

func (n *NotificationContent) GetContentType() (Notifier, error) {
	return CreateNotificationContentType(n.TypeConstant)
}

func (n *NotificationContent) GetDefinition() string {
	nt, err := CreateNotificationContentType(n.TypeConstant)
	if err != nil {
		return ""
	}

	return nt.GetDefinition()
}

// DeleteByIds deletes the given id of NotificationContent (same with content id)
func (n *NotificationContent) DeleteByIds(ids ...int64) error {
	// we use error struct for this function because of iterating over all elements
	// and we'r gonna try to delete given ids at least one time..
	var errs *multierror.Error

	if len(ids) == 0 {
		return models.ErrIdIsNotSet
	}

	for _, id := range ids {
		nc := NewNotificationContent()
		if err := nc.ById(id); err != nil {
			// our aim is removing data from DB
			// so if record is not found in database
			// we can ignore this RecordNotFound error
			if err != bongo.RecordNotFound {
				// return err
				errs = multierror.Append(errs, err)
			}
		}

		if err := nc.Delete(); err != nil {
			if err != bongo.RecordNotFound {
				// return err
				errs = multierror.Append(errs, err)
			}
		}

	}

	return errs.ErrorOrNil()
	// return nil
}
<|endoftext|>


<commit_before>package shell

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"runtime"
	"syscall"
	//"time"

	"github.com/kr/pty"

	"github.com/driusan/de/actions"
	"github.com/driusan/de/demodel"
	"github.com/driusan/de/kbmap"
	"golang.org/x/mobile/event/key"
)

type shellKbmap struct {
	process *os.Process
	stdin   io.WriteCloser
}

// Keymap that sends everything to the Shell command, except Escape (quit the shell and return to
// normal mode), arrow keys (scroll the viewport)
func (s shellKbmap) HandleKey(e key.Event, buff *demodel.CharBuffer, v demodel.Viewport) (demodel.Map, demodel.ScrollDirection, error) {
	switch e.Code {
	case key.CodeEscape:
		if e.Direction != key.DirPress {
			//	s.stdin.Close()
			s.process.Kill()
			return s, demodel.DirectionNone, nil
		}
		return s, demodel.DirectionDown, nil
	// Still honour the viewport manipulation keys
	case key.CodeRightArrow:
		// Arrow keys indicate their scroll direction via the error return value,
		// they return demodel.DirectionNone to make sure both code paths don't accidentally
		// get triggered
		if e.Direction == key.DirPress {
			return s, demodel.DirectionNone, kbmap.ErrScrollRight
		}
		return s, demodel.DirectionNone, nil
	case key.CodeLeftArrow:
		if e.Direction == key.DirPress {
			return s, demodel.DirectionNone, kbmap.ErrScrollLeft
		}
		return s, demodel.DirectionNone, nil
	case key.CodeDownArrow:
		if e.Direction == key.DirPress {
			return s, demodel.DirectionNone, kbmap.ErrScrollDown
		}
		return s, demodel.DirectionNone, nil
	case key.CodeUpArrow:
		if e.Direction == key.DirPress {
			return s, demodel.DirectionNone, kbmap.ErrScrollUp
		}
		return s, demodel.DirectionNone, nil
	// Special cases for control characters.
	case key.CodeTab:
		if e.Direction != key.DirPress {
			//buff.Buffer = append(buff.Buffer, '\t')
			fmt.Fprintf(s.stdin, "%c", '\t')
			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
		}
		return s, demodel.DirectionDown, nil
		//	fmt.Printf("Pressed key %s. Rune is %x", e, e.Rune
	case key.CodeDeleteBackspace:
		if e.Direction != key.DirPress {
			//buff.Buffer = buff.Buffer[:len(buff.Buffer)-1] //append(buff.Buffer, "\t")
			fmt.Fprintf(s.stdin, "%c", '\b')
			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
		}
		return s, demodel.DirectionDown, nil
	case key.CodeReturnEnter:
		if e.Direction != key.DirRelease {
			buff.Buffer = append(buff.Buffer, '\n')
			fmt.Fprintf(s.stdin, "%c", '\n')
			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
			return s, demodel.DirectionDown, nil
		}
		return s, demodel.DirectionDown, nil
	default:
		if e.Direction != key.DirPress && e.Rune > 0 {
			// send the rune to the buffer and to the shell
			//rbytes := make([]byte, 4)
			//n := utf8.EncodeRune(rbytes, e.Rune)
			//	fmt.Printf("Sent to stdin: %c %d", e.Rune, e.Rune)
			// bash and zsh echo the character typed back when invoked with $SHELL -i
			// and it's not a tty.
			// dash doesn't.
			// Don't append the rune to the buffer, because odds are high it'll
			// get echoed back, though there's no way to know for sure.
			//buff.Buffer = append(buff.Buffer, rbytes[:n]...)

			fmt.Fprintf(s.stdin, "%c", e.Rune)
			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
		} else {
			/*
					for debugging only. This otherwise triggers errors on things like
					the user pressing a control key.

				if e.Rune <= 0 {
					fmt.Printf("Invalid rune %d from %s\n", e.Rune, e)
				}*/
		}
	}
	return s, demodel.DirectionDown, nil
}
func init() {
	actions.RegisterAction("Shell", Shell)
}

// Shell invokes an interactive shell terminal similarly to "win" in ACME
func Shell(args string, buff *demodel.CharBuffer, viewport demodel.Viewport) {
	go func() {
		shell := os.Getenv("SHELL")
		if shell == "" {
			switch runtime.GOOS {
			case "plan9":
				shell = "rc"
			default:
				shell = "sh"
			}
		}

		c := exec.Command(shell)
		master, slave, err := pty.Open()
		if err != nil {
			// FIXME: add better error handling.
			panic(err)
		}
		c.SysProcAttr = &syscall.SysProcAttr{
			Setsid:  true,
			Setctty: true,
			Ctty:    int(master.Fd()),
		}

		buff.Filename = ""

		c.Stdout = master
		c.Stderr = master
		c.Stdin = master
		c.Start()
		kbMap := &shellKbmap{c.Process, slave}
		viewport.LockKeyboardMode(kbMap)
		defer func() {
			println("Unlocking keyboard")
			viewport.UnlockKeyboardMode(kbMap)
			viewport.SetKeyboardMode(kbmap.NormalMode)
			println("waiting")
			c.Wait()
			println("shell exited")
		}()

		viewport.SetRenderer(&TerminalRenderer{})
		buf := make([]byte, 1024)

		mouseChan := make(chan interface{})
		viewport.RegisterMouseListener(mouseChan)
		go func() {
			for {
				select {
				case <-mouseChan:
					if buff.Filename != "" {
						viewport.DeregisterMouseListener(mouseChan)
						// The user must have clicked on a filename and
						// opened it. Stop the Shell.
						c.Process.Kill()
					}
				}
			}
		}()
		for {
			if buff.Filename != "" {
				master.Close()
				slave.Close()
				break
			}

			n, err := slave.Read(buf)
			if n > 0 {
				buff.Buffer = append(buff.Buffer, buf[:n]...)

			}
			if err != nil {
				master.Close()
				slave.Close()
				break
			}

			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
			viewport.Rerender()
			buf = make([]byte, 1024)

		}
	}()
}
<commit_msg>Shell: Handle keys on key down, rather than key release and add arrow key support<commit_after>package shell

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"runtime"
	"syscall"
	//"time"

	"github.com/kr/pty"

	"github.com/driusan/de/actions"
	"github.com/driusan/de/demodel"
	"github.com/driusan/de/kbmap"
	"golang.org/x/mobile/event/key"
)

type shellKbmap struct {
	process *os.Process
	stdin   io.WriteCloser
}

// Keymap that sends everything to the Shell command, except Escape (quit the shell and return to
// normal mode), arrow keys (scroll the viewport)
func (s shellKbmap) HandleKey(e key.Event, buff *demodel.CharBuffer, v demodel.Viewport) (demodel.Map, demodel.ScrollDirection, error) {
	switch e.Code {
	case key.CodeEscape:
		if e.Direction != key.DirPress {
			s.process.Kill()
			return s, demodel.DirectionNone, nil
		}
		return s, demodel.DirectionDown, nil
	// Still honour the viewport manipulation keys
	case key.CodeRightArrow:
		// Arrow keys indicate their scroll direction via the error return value,
		// they return demodel.DirectionNone to make sure both code paths don't accidentally
		// get triggered
		if e.Direction == key.DirPress {
			return s, demodel.DirectionNone, kbmap.ErrScrollRight
		}
		return s, demodel.DirectionNone, nil
	case key.CodeLeftArrow:
		if e.Direction == key.DirPress {
			return s, demodel.DirectionNone, kbmap.ErrScrollLeft
		}
		return s, demodel.DirectionNone, nil
	case key.CodeDownArrow:
		if e.Modifiers&key.ModControl != 0 {
			if e.Direction != key.DirRelease {
				fmt.Fprintf(s.stdin, "%c[B", '\033')

				println("send up arrow")
			}
			return s, demodel.DirectionNone, nil
		}

		if e.Direction == key.DirPress {
			return s, demodel.DirectionNone, kbmap.ErrScrollDown
		}
		return s, demodel.DirectionNone, nil
	case key.CodeUpArrow:
		if e.Modifiers&key.ModControl != 0 {
			if e.Direction != key.DirRelease {
				fmt.Fprintf(s.stdin, "%c[A", '\033')

				println("send up arrow")
			}
			return s, demodel.DirectionNone, nil
		}
		if e.Direction == key.DirPress {
			return s, demodel.DirectionNone, kbmap.ErrScrollUp
		}
		return s, demodel.DirectionNone, nil
	// Special cases for control characters.
	case key.CodeTab:
		if e.Direction != key.DirRelease {
			//buff.Buffer = append(buff.Buffer, '\t')
			fmt.Fprintf(s.stdin, "%c", '\t')
			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
		}
		return s, demodel.DirectionDown, nil
		//	fmt.Printf("Pressed key %s. Rune is %x", e, e.Rune
	case key.CodeDeleteBackspace:
		if e.Direction != key.DirRelease {
			fmt.Fprintf(s.stdin, "%c", '\b')
			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
		}
		return s, demodel.DirectionDown, nil
	case key.CodeReturnEnter:
		if e.Direction != key.DirRelease {
			buff.Buffer = append(buff.Buffer, '\n')
			fmt.Fprintf(s.stdin, "%c", '\n')
			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
			return s, demodel.DirectionDown, nil
		}
		return s, demodel.DirectionDown, nil
	default:
		if e.Direction != key.DirRelease && e.Rune > 0 {
			// send the rune to the buffer and to the shell
			//rbytes := make([]byte, 4)
			//n := utf8.EncodeRune(rbytes, e.Rune)
			//	fmt.Printf("Sent to stdin: %c %d", e.Rune, e.Rune)
			// bash and zsh echo the character typed back when invoked with $SHELL -i
			// and it's not a tty.
			// dash doesn't.
			// Don't append the rune to the buffer, because odds are high it'll
			// get echoed back, though there's no way to know for sure.
			//buff.Buffer = append(buff.Buffer, rbytes[:n]...)

			fmt.Fprintf(s.stdin, "%c", e.Rune)
			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
		} else {
			/*
					for debugging only. This otherwise triggers errors on things like
					the user pressing a control key.

				if e.Rune <= 0 {
					fmt.Printf("Invalid rune %d from %s\n", e.Rune, e)
				}*/
		}
	}
	return s, demodel.DirectionDown, nil
}
func init() {
	actions.RegisterAction("Shell", Shell)
}

// Shell invokes an interactive shell terminal similarly to "win" in ACME
func Shell(args string, buff *demodel.CharBuffer, viewport demodel.Viewport) {
	go func() {
		shell := os.Getenv("SHELL")
		if shell == "" {
			switch runtime.GOOS {
			case "plan9":
				shell = "rc"
			default:
				shell = "sh"
			}
		}

		c := exec.Command(shell)
		master, slave, err := pty.Open()
		if err != nil {
			// FIXME: add better error handling.
			panic(err)
		}
		c.SysProcAttr = &syscall.SysProcAttr{
			Setsid:  true,
			Setctty: true,
			Ctty:    int(master.Fd()),
		}

		buff.Filename = ""

		c.Stdout = master
		c.Stderr = master
		c.Stdin = master
		c.Start()
		kbMap := &shellKbmap{c.Process, slave}
		viewport.LockKeyboardMode(kbMap)
		defer func() {
			println("Unlocking keyboard")
			viewport.UnlockKeyboardMode(kbMap)
			viewport.SetKeyboardMode(kbmap.NormalMode)
			println("waiting")
			c.Wait()
			println("shell exited")
		}()

		viewport.SetRenderer(&TerminalRenderer{})
		buf := make([]byte, 1024)

		mouseChan := make(chan interface{})
		viewport.RegisterMouseListener(mouseChan)
		go func() {
			for {
				select {
				case <-mouseChan:
					if buff.Filename != "" {
						viewport.DeregisterMouseListener(mouseChan)
						// The user must have clicked on a filename and
						// opened it. Stop the Shell.
						c.Process.Kill()
					}
				}
			}
		}()
		for {
			if buff.Filename != "" {
				master.Close()
				slave.Close()
				break
			}

			n, err := slave.Read(buf)
			if n > 0 {
				buff.Buffer = append(buff.Buffer, buf[:n]...)

			}
			if err != nil {
				master.Close()
				slave.Close()
				break
			}

			buff.Dot.End = uint(len(buff.Buffer)) - 1
			buff.Dot.Start = buff.Dot.End
			viewport.Rerender()
			buf = make([]byte, 1024)

		}
	}()
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"sync"
	"time"
)

const (
	// DefaultLockSessionName is the Session Name we assign if none is provided
	DefaultLockSessionName = "Consul API Lock"

	// DefaultLockSessionTTL is the default session TTL if no Session is provided
	// when creating a new Lock. This is used because we do not have another
	// other check to depend upon.
	DefaultLockSessionTTL = "15s"

	// DefaultLockWaitTime is how long we block for at a time to check if lock
	// acquisition is possible. This affects the minimum time it takes to cancel
	// a Lock acquisition.
	DefaultLockWaitTime = 15 * time.Second

	// DefaultLockRetryTime is how long we wait after a failed lock acquisition
	// before attempting to do the lock again. This is so that once a lock-delay
	// is in affect, we do not hot loop retrying the acquisition.
	DefaultLockRetryTime = 5 * time.Second
)

var (
	// ErrLockHeld is returned if we attempt to double lock
	ErrLockHeld = fmt.Errorf("Lock already held")

	// ErrLockNotHeld is returned if we attempt to unlock a lock
	// that we do not hold.
	ErrLockNotHeld = fmt.Errorf("Lock not held")
)

// Lock is used to implement client-side leader election. It is follows the
// algorithm as described here: https://consul.io/docs/guides/leader-election.html.
type Lock struct {
	c    *Client
	opts *LockOptions

	isHeld       bool
	sessionRenew chan struct{}
	lockSession  string
	l            sync.Mutex
}

// LockOptions is used to parameterize the Lock behavior.
type LockOptions struct {
	Key         string // Must be set and have write permissions
	Value       []byte // Optional, value to associate with the lock
	Session     string // Optional, created if not specified
	SessionName string // Optional, defaults to DefaultLockSessionName
	SessionTTL  string // Optional, defaults to DefaultLockSessionTTL
}

// LockKey returns a handle to a lock struct which can be used
// to acquire and release the mutex. The key used must have
// write permissions.
func (c *Client) LockKey(key string) (*Lock, error) {
	opts := &LockOptions{
		Key: key,
	}
	return c.LockOpts(opts)
}

// LockOpts returns a handle to a lock struct which can be used
// to acquire and release the mutex. The key used must have
// write permissions.
func (c *Client) LockOpts(opts *LockOptions) (*Lock, error) {
	if opts.SessionName == "" {
		opts.SessionName = DefaultLockSessionName
	}
	if opts.SessionTTL == "" {
		opts.SessionTTL = DefaultLockSessionTTL
	} else {
		if _, err := time.ParseDuration(opts.SessionTTL); err != nil {
			return nil, fmt.Errorf("invalid SessionTTL: %v", err)
		}
	}
	l := &Lock{
		c:    c,
		opts: opts,
	}
	return l, nil
}

// Lock attempts to acquire the lock and blocks while doing so.
// Providing a non-nil stopCh can be used to abort the lock attempt.
// Returns a channel that is closed if our lock is lost or an error.
// This channel could be closed at any time due to session invalidation,
// communication errors, operator intervention, etc. It is NOT safe to
// assume that the lock is held until Unlock() unless the Session is specifically
// created without any associated health checks. By default Consul sessions
// prefer liveness over safety and an application must be able to handle
// the lock being lost.
func (l *Lock) Lock(stopCh chan struct{}) (chan struct{}, error) {
	// Hold the lock as we try to acquire
	l.l.Lock()
	defer l.l.Unlock()

	// Check if we already hold the lock
	if l.isHeld {
		return nil, ErrLockHeld
	}

	// Check if we need to create a session first
	l.lockSession = l.opts.Session
	if l.lockSession == "" {
		if s, err := l.createSession(); err != nil {
			return nil, fmt.Errorf("failed to create session: %v", err)
		} else {
			l.sessionRenew = make(chan struct{})
			l.lockSession = s
			go l.renewSession(s, l.sessionRenew)

			// If we fail to acquire the lock, cleanup the session
			defer func() {
				if !l.isHeld {
					close(l.sessionRenew)
					l.sessionRenew = nil
				}
			}()
		}
	}

	// Setup the query options
	kv := l.c.KV()
	qOpts := &QueryOptions{
		WaitTime: DefaultLockWaitTime,
	}

WAIT:
	// Check if we should quit
	select {
	case <-stopCh:
		return nil, nil
	default:
	}

	// Look for an existing lock, blocking until not taken
	pair, meta, err := kv.Get(l.opts.Key, qOpts)
	if err != nil {
		return nil, fmt.Errorf("failed to read lock: %v", err)
	}
	if pair != nil && pair.Session != "" {
		qOpts.WaitIndex = meta.LastIndex
		goto WAIT
	}

	// Try to acquire the lock
	lockEnt := l.lockEntry(l.lockSession)
	locked, _, err := kv.Acquire(lockEnt, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to acquire lock: %v", err)
	}

	// Handle the case of not getting the lock
	if !locked {
		select {
		case <-time.After(DefaultLockRetryTime):
			goto WAIT
		case <-stopCh:
			return nil, nil
		}
	}

	// Watch to ensure we maintain leadership
	leaderCh := make(chan struct{})
	go l.monitorLock(l.lockSession, leaderCh)

	// Set that we own the lock
	l.isHeld = true

	// Locked! All done
	return leaderCh, nil
}

// Unlock released the lock. It is an error to call this
// if the lock is not currently held.
func (l *Lock) Unlock() error {
	// Hold the lock as we try to release
	l.l.Lock()
	defer l.l.Unlock()

	// Ensure the lock is actually held
	if !l.isHeld {
		return ErrLockNotHeld
	}

	// Set that we no longer own the lock
	l.isHeld = false

	// Stop the session renew
	if l.sessionRenew != nil {
		defer func() {
			close(l.sessionRenew)
			l.sessionRenew = nil
		}()
	}

	// Get the lock entry, and clear the lock session
	lockEnt := l.lockEntry(l.lockSession)
	l.lockSession = ""

	// Release the lock explicitly
	kv := l.c.KV()
	_, _, err := kv.Release(lockEnt, nil)
	if err != nil {
		return fmt.Errorf("failed to release lock: %v", err)
	}
	return nil
}

// createSession is used to create a new managed session
func (l *Lock) createSession() (string, error) {
	session := l.c.Session()
	se := &SessionEntry{
		Name: l.opts.SessionName,
		TTL:  l.opts.SessionTTL,
	}
	id, _, err := session.Create(se, nil)
	if err != nil {
		return "", err
	}
	return id, nil
}

// lockEntry returns a formatted KVPair for the lock
func (l *Lock) lockEntry(session string) *KVPair {
	return &KVPair{
		Key:     l.opts.Key,
		Value:   l.opts.Value,
		Session: session,
	}
}

// renewSession is a long running routine that maintians a session
// by doing a periodic Session renewal.
func (l *Lock) renewSession(id string, doneCh chan struct{}) {
	session := l.c.Session()
	ttl, _ := time.ParseDuration(l.opts.SessionTTL)
	for {
		select {
		case <-time.After(ttl / 2):
			entry, _, err := session.Renew(id, nil)
			if err != nil || entry == nil {
				return
			}

			// Handle the server updating the TTL
			ttl, _ = time.ParseDuration(entry.TTL)

		case <-doneCh:
			// Attempt a session destroy
			session.Destroy(id, nil)
			return
		}
	}
}

// monitorLock is a long running routine to monitor a lock ownership
// It closes the stopCh if we lose our leadership.
func (l *Lock) monitorLock(session string, stopCh chan struct{}) {
	kv := l.c.KV()
	opts := &QueryOptions{RequireConsistent: true}
WAIT:
	pair, meta, err := kv.Get(l.opts.Key, opts)
	if err != nil {
		close(stopCh)
		return
	}
	if pair != nil && pair.Session == session {
		opts.WaitIndex = meta.LastIndex
		goto WAIT
	}
	close(stopCh)
}
<commit_msg>api: Minor cleanups in lock<commit_after>package api

import (
	"fmt"
	"sync"
	"time"
)

const (
	// DefaultLockSessionName is the Session Name we assign if none is provided
	DefaultLockSessionName = "Consul API Lock"

	// DefaultLockSessionTTL is the default session TTL if no Session is provided
	// when creating a new Lock. This is used because we do not have another
	// other check to depend upon.
	DefaultLockSessionTTL = "15s"

	// DefaultLockWaitTime is how long we block for at a time to check if lock
	// acquisition is possible. This affects the minimum time it takes to cancel
	// a Lock acquisition.
	DefaultLockWaitTime = 15 * time.Second

	// DefaultLockRetryTime is how long we wait after a failed lock acquisition
	// before attempting to do the lock again. This is so that once a lock-delay
	// is in affect, we do not hot loop retrying the acquisition.
	DefaultLockRetryTime = 5 * time.Second
)

var (
	// ErrLockHeld is returned if we attempt to double lock
	ErrLockHeld = fmt.Errorf("Lock already held")

	// ErrLockNotHeld is returned if we attempt to unlock a lock
	// that we do not hold.
	ErrLockNotHeld = fmt.Errorf("Lock not held")
)

// Lock is used to implement client-side leader election. It is follows the
// algorithm as described here: https://consul.io/docs/guides/leader-election.html.
type Lock struct {
	c    *Client
	opts *LockOptions

	isHeld       bool
	sessionRenew chan struct{}
	lockSession  string
	l            sync.Mutex
}

// LockOptions is used to parameterize the Lock behavior.
type LockOptions struct {
	Key         string // Must be set and have write permissions
	Value       []byte // Optional, value to associate with the lock
	Session     string // Optional, created if not specified
	SessionName string // Optional, defaults to DefaultLockSessionName
	SessionTTL  string // Optional, defaults to DefaultLockSessionTTL
}

// LockKey returns a handle to a lock struct which can be used
// to acquire and release the mutex. The key used must have
// write permissions.
func (c *Client) LockKey(key string) (*Lock, error) {
	opts := &LockOptions{
		Key: key,
	}
	return c.LockOpts(opts)
}

// LockOpts returns a handle to a lock struct which can be used
// to acquire and release the mutex. The key used must have
// write permissions.
func (c *Client) LockOpts(opts *LockOptions) (*Lock, error) {
	if opts.Key == "" {
		return nil, fmt.Errorf("missing key")
	}
	if opts.SessionName == "" {
		opts.SessionName = DefaultLockSessionName
	}
	if opts.SessionTTL == "" {
		opts.SessionTTL = DefaultLockSessionTTL
	} else {
		if _, err := time.ParseDuration(opts.SessionTTL); err != nil {
			return nil, fmt.Errorf("invalid SessionTTL: %v", err)
		}
	}
	l := &Lock{
		c:    c,
		opts: opts,
	}
	return l, nil
}

// Lock attempts to acquire the lock and blocks while doing so.
// Providing a non-nil stopCh can be used to abort the lock attempt.
// Returns a channel that is closed if our lock is lost or an error.
// This channel could be closed at any time due to session invalidation,
// communication errors, operator intervention, etc. It is NOT safe to
// assume that the lock is held until Unlock() unless the Session is specifically
// created without any associated health checks. By default Consul sessions
// prefer liveness over safety and an application must be able to handle
// the lock being lost.
func (l *Lock) Lock(stopCh chan struct{}) (chan struct{}, error) {
	// Hold the lock as we try to acquire
	l.l.Lock()
	defer l.l.Unlock()

	// Check if we already hold the lock
	if l.isHeld {
		return nil, ErrLockHeld
	}

	// Check if we need to create a session first
	l.lockSession = l.opts.Session
	if l.lockSession == "" {
		if s, err := l.createSession(); err != nil {
			return nil, fmt.Errorf("failed to create session: %v", err)
		} else {
			l.sessionRenew = make(chan struct{})
			l.lockSession = s
			go l.renewSession(s, l.sessionRenew)

			// If we fail to acquire the lock, cleanup the session
			defer func() {
				if !l.isHeld {
					close(l.sessionRenew)
					l.sessionRenew = nil
				}
			}()
		}
	}

	// Setup the query options
	kv := l.c.KV()
	qOpts := &QueryOptions{
		WaitTime: DefaultLockWaitTime,
	}

WAIT:
	// Check if we should quit
	select {
	case <-stopCh:
		return nil, nil
	default:
	}

	// Look for an existing lock, blocking until not taken
	pair, meta, err := kv.Get(l.opts.Key, qOpts)
	if err != nil {
		return nil, fmt.Errorf("failed to read lock: %v", err)
	}
	if pair != nil && pair.Session != "" {
		qOpts.WaitIndex = meta.LastIndex
		goto WAIT
	}

	// Try to acquire the lock
	lockEnt := l.lockEntry(l.lockSession)
	locked, _, err := kv.Acquire(lockEnt, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to acquire lock: %v", err)
	}

	// Handle the case of not getting the lock
	if !locked {
		select {
		case <-time.After(DefaultLockRetryTime):
			goto WAIT
		case <-stopCh:
			return nil, nil
		}
	}

	// Watch to ensure we maintain leadership
	leaderCh := make(chan struct{})
	go l.monitorLock(l.lockSession, leaderCh)

	// Set that we own the lock
	l.isHeld = true

	// Locked! All done
	return leaderCh, nil
}

// Unlock released the lock. It is an error to call this
// if the lock is not currently held.
func (l *Lock) Unlock() error {
	// Hold the lock as we try to release
	l.l.Lock()
	defer l.l.Unlock()

	// Ensure the lock is actually held
	if !l.isHeld {
		return ErrLockNotHeld
	}

	// Set that we no longer own the lock
	l.isHeld = false

	// Stop the session renew
	if l.sessionRenew != nil {
		defer func() {
			close(l.sessionRenew)
			l.sessionRenew = nil
		}()
	}

	// Get the lock entry, and clear the lock session
	lockEnt := l.lockEntry(l.lockSession)
	l.lockSession = ""

	// Release the lock explicitly
	kv := l.c.KV()
	_, _, err := kv.Release(lockEnt, nil)
	if err != nil {
		return fmt.Errorf("failed to release lock: %v", err)
	}
	return nil
}

// createSession is used to create a new managed session
func (l *Lock) createSession() (string, error) {
	session := l.c.Session()
	se := &SessionEntry{
		Name: l.opts.SessionName,
		TTL:  l.opts.SessionTTL,
	}
	id, _, err := session.Create(se, nil)
	if err != nil {
		return "", err
	}
	return id, nil
}

// lockEntry returns a formatted KVPair for the lock
func (l *Lock) lockEntry(session string) *KVPair {
	return &KVPair{
		Key:     l.opts.Key,
		Value:   l.opts.Value,
		Session: session,
	}
}

// renewSession is a long running routine that maintians a session
// by doing a periodic Session renewal.
func (l *Lock) renewSession(id string, doneCh chan struct{}) {
	session := l.c.Session()
	ttl, _ := time.ParseDuration(l.opts.SessionTTL)
	for {
		select {
		case <-time.After(ttl / 2):
			entry, _, err := session.Renew(id, nil)
			if err != nil || entry == nil {
				return
			}

			// Handle the server updating the TTL
			ttl, _ = time.ParseDuration(entry.TTL)

		case <-doneCh:
			// Attempt a session destroy
			session.Destroy(id, nil)
			return
		}
	}
}

// monitorLock is a long running routine to monitor a lock ownership
// It closes the stopCh if we lose our leadership.
func (l *Lock) monitorLock(session string, stopCh chan struct{}) {
	defer close(stopCh)
	kv := l.c.KV()
	opts := &QueryOptions{RequireConsistent: true}
WAIT:
	pair, meta, err := kv.Get(l.opts.Key, opts)
	if err != nil {
		return
	}
	if pair != nil && pair.Session == session {
		opts.WaitIndex = meta.LastIndex
		goto WAIT
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"

	"github.com/muesli/polly/api/config"
	"github.com/muesli/polly/api/db"
	"github.com/muesli/polly/api/mailman"
	"github.com/muesli/polly/api/utils"

	"github.com/muesli/polly/api/resources/budgets"
	"github.com/muesli/polly/api/resources/proposals"
	"github.com/muesli/polly/api/resources/sessions"
	"github.com/muesli/polly/api/resources/users"
	"github.com/muesli/polly/api/resources/votes"

	log "github.com/Sirupsen/logrus"
	"github.com/emicklei/go-restful/swagger"
	"github.com/muesli/smolder"
)

func handleSignals() (chan int, bool) {
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, os.Kill)

	shutdownGracefully := false
	requestIncChan := make(chan int)

	go func() {
		boldGreen := string(byte(27)) + "[1;32m"
		boldRed := string(byte(27)) + "[1;31m"
		boldEnd := string(byte(27)) + "[0m"

		pendingRequests := 0
		for {
			select {
			case sig := <-sigChan:
				if !shutdownGracefully {
					shutdownGracefully = true
					fmt.Printf(boldGreen+"\nGot %s signal, shutting down gracefully. Press Ctrl-C again to stop now.\n\n"+boldEnd, sig.String())
					if pendingRequests == 0 {
						os.Exit(0)
					}
				} else {
					fmt.Printf(boldRed+"\nGot %s signal, shutting down now!\n\n"+boldEnd, sig.String())
					os.Exit(0)
				}

			case inc := <-requestIncChan:
				pendingRequests += inc
				if shutdownGracefully {
					log.Infoln("Pending requests:", pendingRequests)
					if pendingRequests == 0 {
						os.Exit(0)
					}
				}
			}
		}
	}()

	return requestIncChan, shutdownGracefully
}

func main() {
	ch, shutdownGracefully := handleSignals()

	config.ParseSettings()
	db.SetupPostgres(config.Settings.Connections.PostgreSQLConnection)

	context := &db.PollyContext{
		Config: *config.Settings,
	}

	utils.SetupEmailTemplates(*config.Settings)
	mailman.SetupMailmanContext(context.NewAPIContext().(*db.PollyContext))
	go mailman.RunLoop()

	// Setup web-service
	smolderConfig := smolder.APIConfig{
		BaseURL:    config.Settings.API.BaseURL,
		PathPrefix: config.Settings.API.PathPrefix,
	}

	wsContainer := smolder.NewSmolderContainer(smolderConfig, &shutdownGracefully, ch)
	func(resources ...smolder.APIResource) {
		for _, r := range resources {
			r.Register(wsContainer, smolderConfig, context)
		}
	}(
		&sessions.SessionResource{},
		&users.UserResource{},
		&proposals.ProposalResource{},
		&votes.VoteResource{},
		&budgets.BudgetResource{},
	)

	if config.Settings.API.SwaggerFilePath != "" {
		wsConfig := swagger.Config{
			WebServices:     wsContainer.RegisteredWebServices(),
			WebServicesUrl:  config.Settings.API.BaseURL,
			ApiPath:         config.Settings.API.SwaggerAPIPath,
			SwaggerPath:     config.Settings.API.SwaggerPath,
			SwaggerFilePath: config.Settings.API.SwaggerFilePath,
		}
		swagger.RegisterSwaggerService(wsConfig, wsContainer)
	}

	// GlobalLog("Starting polly web-api...")
	server := &http.Server{Addr: config.Settings.API.Bind, Handler: wsContainer}
	log.Fatal(server.ListenAndServe())
}
<commit_msg>Updated go-restful-swagger import path<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"

	"github.com/muesli/polly/api/config"
	"github.com/muesli/polly/api/db"
	"github.com/muesli/polly/api/mailman"
	"github.com/muesli/polly/api/utils"

	"github.com/muesli/polly/api/resources/budgets"
	"github.com/muesli/polly/api/resources/proposals"
	"github.com/muesli/polly/api/resources/sessions"
	"github.com/muesli/polly/api/resources/users"
	"github.com/muesli/polly/api/resources/votes"

	log "github.com/Sirupsen/logrus"
	"github.com/emicklei/go-restful-swagger12"
	"github.com/muesli/smolder"
)

func handleSignals() (chan int, bool) {
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, os.Kill)

	shutdownGracefully := false
	requestIncChan := make(chan int)

	go func() {
		boldGreen := string(byte(27)) + "[1;32m"
		boldRed := string(byte(27)) + "[1;31m"
		boldEnd := string(byte(27)) + "[0m"

		pendingRequests := 0
		for {
			select {
			case sig := <-sigChan:
				if !shutdownGracefully {
					shutdownGracefully = true
					fmt.Printf(boldGreen+"\nGot %s signal, shutting down gracefully. Press Ctrl-C again to stop now.\n\n"+boldEnd, sig.String())
					if pendingRequests == 0 {
						os.Exit(0)
					}
				} else {
					fmt.Printf(boldRed+"\nGot %s signal, shutting down now!\n\n"+boldEnd, sig.String())
					os.Exit(0)
				}

			case inc := <-requestIncChan:
				pendingRequests += inc
				if shutdownGracefully {
					log.Infoln("Pending requests:", pendingRequests)
					if pendingRequests == 0 {
						os.Exit(0)
					}
				}
			}
		}
	}()

	return requestIncChan, shutdownGracefully
}

func main() {
	ch, shutdownGracefully := handleSignals()

	config.ParseSettings()
	db.SetupPostgres(config.Settings.Connections.PostgreSQLConnection)

	context := &db.PollyContext{
		Config: *config.Settings,
	}

	utils.SetupEmailTemplates(*config.Settings)
	mailman.SetupMailmanContext(context.NewAPIContext().(*db.PollyContext))
	go mailman.RunLoop()

	// Setup web-service
	smolderConfig := smolder.APIConfig{
		BaseURL:    config.Settings.API.BaseURL,
		PathPrefix: config.Settings.API.PathPrefix,
	}

	wsContainer := smolder.NewSmolderContainer(smolderConfig, &shutdownGracefully, ch)
	func(resources ...smolder.APIResource) {
		for _, r := range resources {
			r.Register(wsContainer, smolderConfig, context)
		}
	}(
		&sessions.SessionResource{},
		&users.UserResource{},
		&proposals.ProposalResource{},
		&votes.VoteResource{},
		&budgets.BudgetResource{},
	)

	if config.Settings.API.SwaggerFilePath != "" {
		wsConfig := swagger.Config{
			WebServices:     wsContainer.RegisteredWebServices(),
			WebServicesUrl:  config.Settings.API.BaseURL,
			ApiPath:         config.Settings.API.SwaggerAPIPath,
			SwaggerPath:     config.Settings.API.SwaggerPath,
			SwaggerFilePath: config.Settings.API.SwaggerFilePath,
		}
		swagger.RegisterSwaggerService(wsConfig, wsContainer)
	}

	// GlobalLog("Starting polly web-api...")
	server := &http.Server{Addr: config.Settings.API.Bind, Handler: wsContainer}
	log.Fatal(server.ListenAndServe())
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	_ "github.com/globocom/tsuru/provision/juju"
	stdlog "log"
	"log/syslog"
	"net/http"
	"os"
)

func fatal(err error) {
	fmt.Fprintln(os.Stderr, err)
	log.Fatal(err)
}

func main() {
	logger, err := syslog.NewLogger(syslog.LOG_INFO, stdlog.LstdFlags)
	if err != nil {
		stdlog.Fatal(err)
	}
	log.SetLogger(logger)
	configFile := flag.String("config", "/etc/tsuru/tsuru.conf", "tsuru config file")
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	flag.Parse()
	err = config.ReadAndWatchConfigFile(*configFile)
	if err != nil {
		fatal(err)
	}
	connString, err := config.GetString("database:url")
	if err != nil {
		fatal(err)
	}
	dbName, err := config.GetString("database:name")
	if err != nil {
		fatal(err)
	}
	fmt.Printf("Using the database %q from the server %q.\n\n", dbName, connString)

	m := pat.New()

	m.Get("/services/instances", AuthorizationRequiredHandler(ServicesInstancesHandler))
	m.Post("/services/instances", AuthorizationRequiredHandler(CreateInstanceHandler))
	m.Put("/services/instances/:instance/:app", AuthorizationRequiredHandler(BindHandler))
	m.Del("/services/instances/:instance/:app", AuthorizationRequiredHandler(UnbindHandler))
	m.Del("/services/c/instances/:name", AuthorizationRequiredHandler(RemoveServiceInstanceHandler))
	m.Get("/services/instances/:instance/status", AuthorizationRequiredHandler(ServiceInstanceStatusHandler))

	m.Get("/services", AuthorizationRequiredHandler(ServicesHandler))
	m.Post("/services", AuthorizationRequiredHandler(CreateHandler))
	m.Put("/services", AuthorizationRequiredHandler(UpdateHandler))
	m.Del("/services/:name", AuthorizationRequiredHandler(DeleteHandler))
	m.Get("/services/:name", AuthorizationRequiredHandler(ServiceInfoHandler))
	m.Get("/services/c/:name/doc", AuthorizationRequiredHandler(Doc))
	m.Get("/services/:name/doc", AuthorizationRequiredHandler(GetDocHandler))
	m.Put("/services/:name/doc", AuthorizationRequiredHandler(AddDocHandler))
	m.Put("/services/:service/:team", AuthorizationRequiredHandler(GrantServiceAccessToTeamHandler))
	m.Del("/services/:service/:team", AuthorizationRequiredHandler(RevokeServiceAccessFromTeamHandler))

	m.Del("/apps/:name", AuthorizationRequiredHandler(AppDelete))
	m.Get("/apps/:name/repository/clone", Handler(CloneRepositoryHandler))
	m.Get("/apps/:name/avaliable", Handler(AppIsAvailableHandler))
	m.Get("/apps/:name", AuthorizationRequiredHandler(AppInfo))
	m.Post("/apps/:name/run", AuthorizationRequiredHandler(RunCommand))
	m.Get("/apps/:name/restart", AuthorizationRequiredHandler(RestartHandler))
	m.Get("/apps/:name/env", AuthorizationRequiredHandler(GetEnv))
	m.Post("/apps/:name/env", AuthorizationRequiredHandler(SetEnv))
	m.Del("/apps/:name/env", AuthorizationRequiredHandler(UnsetEnv))
	m.Get("/apps", AuthorizationRequiredHandler(AppList))
	m.Post("/apps", AuthorizationRequiredHandler(CreateAppHandler))
	m.Put("/apps/:name/units", AuthorizationRequiredHandler(AddUnitsHandler))
	m.Del("/apps/:name/units", AuthorizationRequiredHandler(RemoveUnitsHandler))
	m.Put("/apps/:app/:team", AuthorizationRequiredHandler(GrantAccessToTeamHandler))
	m.Del("/apps/:app/:team", AuthorizationRequiredHandler(RevokeAccessFromTeamHandler))
	m.Get("/apps/:name/log", AuthorizationRequiredHandler(AppLog))
	m.Post("/apps/:name/log", Handler(AddLogHandler))

	m.Post("/users", Handler(CreateUser))
	m.Post("/users/:email/tokens", Handler(Login))
	m.Put("/users/password", AuthorizationRequiredHandler(ChangePassword))
	m.Del("/users", AuthorizationRequiredHandler(RemoveUser))
	m.Post("/users/keys", AuthorizationRequiredHandler(AddKeyToUser))
	m.Del("/users/keys", AuthorizationRequiredHandler(RemoveKeyFromUser))

	m.Get("/teams", AuthorizationRequiredHandler(ListTeams))
	m.Post("/teams", AuthorizationRequiredHandler(CreateTeam))
	m.Del("/teams/:name", AuthorizationRequiredHandler(RemoveTeam))
	m.Put("/teams/:team/:user", AuthorizationRequiredHandler(AddUserToTeam))
	m.Del("/teams/:team/:user", AuthorizationRequiredHandler(RemoveUserFromTeam))

	m.Get("/healers", Handler(healers))
	m.Get("/healers/:healer", Handler(healer))

	if !*dry {
		provisioner, err := config.GetString("provisioner")
		if err != nil {
			fmt.Printf("Warning: %q didn't declare a provisioner, using default provisioner.\n", configFile)
			provisioner = "juju"
		}
		app.Provisioner, err = provision.Get(provisioner)
		if err != nil {
			fatal(err)
		}
		fmt.Printf("Using %q provisioner.\n\n", provisioner)

		listen, err := config.GetString("listen")
		if err != nil {
			fatal(err)
		}
		tls, _ := config.GetBool("use-tls")
		if tls {
			certFile, err := config.GetString("tls-cert-file")
			if err != nil {
				fatal(err)
			}
			keyFile, err := config.GetString("tls-key-file")
			if err != nil {
				fatal(err)
			}
			fmt.Printf("tsuru HTTP/TLS server listening at %s...\n", listen)
			fatal(http.ListenAndServeTLS(listen, certFile, keyFile, m))
		} else {
			fmt.Printf("tsuru HTTP server listening at %s...\n", listen)
			fatal(http.ListenAndServe(listen, m))
		}
	}
}
<commit_msg>api: imported local provisioner.<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/app"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	_ "github.com/globocom/tsuru/provision/juju"
	_ "github.com/globocom/tsuru/provision/local"
	stdlog "log"
	"log/syslog"
	"net/http"
	"os"
)

func fatal(err error) {
	fmt.Fprintln(os.Stderr, err)
	log.Fatal(err)
}

func main() {
	logger, err := syslog.NewLogger(syslog.LOG_INFO, stdlog.LstdFlags)
	if err != nil {
		stdlog.Fatal(err)
	}
	log.SetLogger(logger)
	configFile := flag.String("config", "/etc/tsuru/tsuru.conf", "tsuru config file")
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	flag.Parse()
	err = config.ReadAndWatchConfigFile(*configFile)
	if err != nil {
		fatal(err)
	}
	connString, err := config.GetString("database:url")
	if err != nil {
		fatal(err)
	}
	dbName, err := config.GetString("database:name")
	if err != nil {
		fatal(err)
	}
	fmt.Printf("Using the database %q from the server %q.\n\n", dbName, connString)

	m := pat.New()

	m.Get("/services/instances", AuthorizationRequiredHandler(ServicesInstancesHandler))
	m.Post("/services/instances", AuthorizationRequiredHandler(CreateInstanceHandler))
	m.Put("/services/instances/:instance/:app", AuthorizationRequiredHandler(BindHandler))
	m.Del("/services/instances/:instance/:app", AuthorizationRequiredHandler(UnbindHandler))
	m.Del("/services/c/instances/:name", AuthorizationRequiredHandler(RemoveServiceInstanceHandler))
	m.Get("/services/instances/:instance/status", AuthorizationRequiredHandler(ServiceInstanceStatusHandler))

	m.Get("/services", AuthorizationRequiredHandler(ServicesHandler))
	m.Post("/services", AuthorizationRequiredHandler(CreateHandler))
	m.Put("/services", AuthorizationRequiredHandler(UpdateHandler))
	m.Del("/services/:name", AuthorizationRequiredHandler(DeleteHandler))
	m.Get("/services/:name", AuthorizationRequiredHandler(ServiceInfoHandler))
	m.Get("/services/c/:name/doc", AuthorizationRequiredHandler(Doc))
	m.Get("/services/:name/doc", AuthorizationRequiredHandler(GetDocHandler))
	m.Put("/services/:name/doc", AuthorizationRequiredHandler(AddDocHandler))
	m.Put("/services/:service/:team", AuthorizationRequiredHandler(GrantServiceAccessToTeamHandler))
	m.Del("/services/:service/:team", AuthorizationRequiredHandler(RevokeServiceAccessFromTeamHandler))

	m.Del("/apps/:name", AuthorizationRequiredHandler(AppDelete))
	m.Get("/apps/:name/repository/clone", Handler(CloneRepositoryHandler))
	m.Get("/apps/:name/avaliable", Handler(AppIsAvailableHandler))
	m.Get("/apps/:name", AuthorizationRequiredHandler(AppInfo))
	m.Post("/apps/:name/run", AuthorizationRequiredHandler(RunCommand))
	m.Get("/apps/:name/restart", AuthorizationRequiredHandler(RestartHandler))
	m.Get("/apps/:name/env", AuthorizationRequiredHandler(GetEnv))
	m.Post("/apps/:name/env", AuthorizationRequiredHandler(SetEnv))
	m.Del("/apps/:name/env", AuthorizationRequiredHandler(UnsetEnv))
	m.Get("/apps", AuthorizationRequiredHandler(AppList))
	m.Post("/apps", AuthorizationRequiredHandler(CreateAppHandler))
	m.Put("/apps/:name/units", AuthorizationRequiredHandler(AddUnitsHandler))
	m.Del("/apps/:name/units", AuthorizationRequiredHandler(RemoveUnitsHandler))
	m.Put("/apps/:app/:team", AuthorizationRequiredHandler(GrantAccessToTeamHandler))
	m.Del("/apps/:app/:team", AuthorizationRequiredHandler(RevokeAccessFromTeamHandler))
	m.Get("/apps/:name/log", AuthorizationRequiredHandler(AppLog))
	m.Post("/apps/:name/log", Handler(AddLogHandler))

	m.Post("/users", Handler(CreateUser))
	m.Post("/users/:email/tokens", Handler(Login))
	m.Put("/users/password", AuthorizationRequiredHandler(ChangePassword))
	m.Del("/users", AuthorizationRequiredHandler(RemoveUser))
	m.Post("/users/keys", AuthorizationRequiredHandler(AddKeyToUser))
	m.Del("/users/keys", AuthorizationRequiredHandler(RemoveKeyFromUser))

	m.Get("/teams", AuthorizationRequiredHandler(ListTeams))
	m.Post("/teams", AuthorizationRequiredHandler(CreateTeam))
	m.Del("/teams/:name", AuthorizationRequiredHandler(RemoveTeam))
	m.Put("/teams/:team/:user", AuthorizationRequiredHandler(AddUserToTeam))
	m.Del("/teams/:team/:user", AuthorizationRequiredHandler(RemoveUserFromTeam))

	m.Get("/healers", Handler(healers))
	m.Get("/healers/:healer", Handler(healer))

	if !*dry {
		provisioner, err := config.GetString("provisioner")
		if err != nil {
			fmt.Printf("Warning: %q didn't declare a provisioner, using default provisioner.\n", configFile)
			provisioner = "juju"
		}
		app.Provisioner, err = provision.Get(provisioner)
		if err != nil {
			fatal(err)
		}
		fmt.Printf("Using %q provisioner.\n\n", provisioner)

		listen, err := config.GetString("listen")
		if err != nil {
			fatal(err)
		}
		tls, _ := config.GetBool("use-tls")
		if tls {
			certFile, err := config.GetString("tls-cert-file")
			if err != nil {
				fatal(err)
			}
			keyFile, err := config.GetString("tls-key-file")
			if err != nil {
				fatal(err)
			}
			fmt.Printf("tsuru HTTP/TLS server listening at %s...\n", listen)
			fatal(http.ListenAndServeTLS(listen, certFile, keyFile, m))
		} else {
			fmt.Printf("tsuru HTTP server listening at %s...\n", listen)
			fatal(http.ListenAndServe(listen, m))
		}
	}
}
<|endoftext|>


<commit_before>/*
Package base provides some base functions such as random code and valid phoneNumber check
*/
package base

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"github.com/evolsnow/samaritan/common/log"
	"io/ioutil"
	"math/rand"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

const (
	avatarPath = "static/avatar/"
)

//generate avatar url
func getGravatarUrl(phone string) string {
	email := fmt.Sprintf("%s@samaritan.tech", phone)
	h := md5.New()
	h.Write([]byte(email))
	hashed := hex.EncodeToString(h.Sum(nil))
	return fmt.Sprintf("https://cn.gravatar.com/avatar/%s.jpg?d=retro&s=40", hashed)
}

// GenerateAvatar saves avatar file and return relative path
func GenerateAvatar(phone string) (string, error) {
	resp, err := http.Get(getGravatarUrl(phone))
	if err == nil {
		defer resp.Body.Close()
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			//use default avatar
			return "", err
		}
		path := fmt.Sprintf("%s%s.jpg", avatarPath, phone)
		err = ioutil.WriteFile(path, data, 0644)
		if err != nil {
			return "", err
		}
		return path, nil
	}
	return "", err
}

// RandomCodeSix generates random 6-length code
func RandomCodeSix() string {
	rand.Seed(time.Now().UTC().UnixNano())
	code := 100000 + rand.Intn(900000)
	return strconv.Itoa(code)
}

// ValidPhone checks if the phone number is legal
func ValidPhone(phone string) bool {
	pattern := regexp.MustCompile("(13[0-9]|15[01235678]|17[0-9]|18[0-9]|14[57])[0-9]{8}")
	return pattern.MatchString(phone)
}

// ValidMail checks if the mail format is legal
func ValidMail(mail string) bool {
	pattern := regexp.MustCompile("[_a-z0-9-]+(\\.[_a-z0-9-]+)*@[a-z0-9-]+(\\.[a-z0-9-]+)*(\\.[a-z]{2,4})")
	return pattern.MatchString(mail)
}

// ValidSamId checks if the sam id is legal
func ValidSamId(samId string) bool {
	pattern := regexp.MustCompile("^(\\w)+$")
	return pattern.MatchString(samId)
}

// BadReqErr means 400 bad request error
func BadReqErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusBadRequest)
}

// ForbidErr means 403 forbidden error
func ForbidErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusForbidden)
}

// MethodNAErr means 405 method not allowed error
func MethodNAErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusMethodNotAllowed)
}

// UnAuthErr means 401 unauthorized error
func UnAuthErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusUnauthorized)
}

// NotFoundErr means 404 not found error
func NotFoundErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusNotFound)
}

// InternalErr means 500 internal error
func InternalErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusInternalServerError)
}

//set http status and reply error
func setError(w http.ResponseWriter, desc string, status int) {
	e := map[string]interface{}{"code": status, "msg": desc}
	msg, _ := json.Marshal(e)
	log.DebugJson(e)
	w.WriteHeader(status)
	w.Write(msg)
}

// InIntSlice checks if an int is in the slice []int
func InIntSlice(a int, list []int) bool {
	for _, v := range list {
		if a == v {
			return true
		}
	}
	return false
}
<commit_msg>Update utils.go<commit_after>/*
Package base provides some base functions such as random code and valid phoneNumber check
*/
package base

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"github.com/evolsnow/samaritan/common/log"
	"io/ioutil"
	"math/rand"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

const (
	avatarPath = "avatar/"
)

//generate avatar url
func getGravatarUrl(phone string) string {
	email := fmt.Sprintf("%s@samaritan.tech", phone)
	h := md5.New()
	h.Write([]byte(email))
	hashed := hex.EncodeToString(h.Sum(nil))
	return fmt.Sprintf("https://cn.gravatar.com/avatar/%s.jpg?d=retro&s=40", hashed)
}

// GenerateAvatar saves avatar file and return relative path
func GenerateAvatar(phone string) (string, error) {
	resp, err := http.Get(getGravatarUrl(phone))
	if err == nil {
		defer resp.Body.Close()
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			//use default avatar
			return "", err
		}
		path := fmt.Sprintf("%s%s.jpg", avatarPath, phone)
		err = ioutil.WriteFile(path, data, 0644)
		if err != nil {
			return "", err
		}
		return path, nil
	}
	return "", err
}

// RandomCodeSix generates random 6-length code
func RandomCodeSix() string {
	rand.Seed(time.Now().UTC().UnixNano())
	code := 100000 + rand.Intn(900000)
	return strconv.Itoa(code)
}

// ValidPhone checks if the phone number is legal
func ValidPhone(phone string) bool {
	pattern := regexp.MustCompile("(13[0-9]|15[01235678]|17[0-9]|18[0-9]|14[57])[0-9]{8}")
	return pattern.MatchString(phone)
}

// ValidMail checks if the mail format is legal
func ValidMail(mail string) bool {
	pattern := regexp.MustCompile("[_a-z0-9-]+(\\.[_a-z0-9-]+)*@[a-z0-9-]+(\\.[a-z0-9-]+)*(\\.[a-z]{2,4})")
	return pattern.MatchString(mail)
}

// ValidSamId checks if the sam id is legal
func ValidSamId(samId string) bool {
	pattern := regexp.MustCompile("^(\\w)+$")
	return pattern.MatchString(samId)
}

// BadReqErr means 400 bad request error
func BadReqErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusBadRequest)
}

// ForbidErr means 403 forbidden error
func ForbidErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusForbidden)
}

// MethodNAErr means 405 method not allowed error
func MethodNAErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusMethodNotAllowed)
}

// UnAuthErr means 401 unauthorized error
func UnAuthErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusUnauthorized)
}

// NotFoundErr means 404 not found error
func NotFoundErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusNotFound)
}

// InternalErr means 500 internal error
func InternalErr(w http.ResponseWriter, desc string) {
	setError(w, desc, http.StatusInternalServerError)
}

//set http status and reply error
func setError(w http.ResponseWriter, desc string, status int) {
	e := map[string]interface{}{"code": status, "msg": desc}
	msg, _ := json.Marshal(e)
	log.DebugJson(e)
	w.WriteHeader(status)
	w.Write(msg)
}

// InIntSlice checks if an int is in the slice []int
func InIntSlice(a int, list []int) bool {
	for _, v := range list {
		if a == v {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package api

import (
	"github.com/kennygrant/sanitize"
	"github.com/nathan-osman/go-cannon/email"
	"github.com/nathan-osman/go-cannon/queue"
	"github.com/zenazn/goji/web"

	"encoding/json"
	"html"
	"net/http"
)

// Parameters for the /send method.
type sendParams struct {
	From    string   `json:"from"`
	To      []string `json:"to"`
	Cc      []string `json:"cc"`
	Bcc     []string `json:"bcc"`
	Subject string   `json:"subject"`
	Text    string   `json:"text"`
	Html    string   `json:"html"`
}

// Send an email with the specified parameters.
func Send(c web.C, w http.ResponseWriter, r *http.Request) {
	var p sendParams
	if err := json.NewDecoder(r.Body).Decode(&p); err != nil {
		respondWithError(w, "malformed JSON")
	} else {

		// Ensure that if either 'text' or 'html' was not provided, its value
		// is populated by the other field
		if p.Html == "" {
			p.Html = html.EscapeString(p.Text)
		} else if p.Text == "" {
			p.Text = sanitize.HTML(p.Html)
		}

		// Create the individual emails to send and put them into the queue
		if emails, err := email.NewEmails(p.From, p.To, p.Cc, p.Bcc, p.Subject, p.Text, p.Html); err != nil {
			respondWithError(w, err.Error())
		} else {
			for _, e := range emails {
				c.Env["queue"].(*queue.Queue).Deliver(e)
			}
			respondWithJSON(w, struct{}{})
		}
	}
}
<commit_msg>Updated /send method to use Email and Attachment types.<commit_after>package api

import (
	"github.com/kennygrant/sanitize"
	"github.com/nathan-osman/go-cannon/email"
	"github.com/nathan-osman/go-cannon/queue"
	"github.com/zenazn/goji/web"

	"encoding/json"
	"html"
	"net/http"
)

// Send an email with the specified parameters.
func Send(c web.C, w http.ResponseWriter, r *http.Request) {

	// Attempt to decode the parameters as an email.Email instance
	var e email.Email
	if err := json.NewDecoder(r.Body).Decode(&e); err != nil {
		respondWithError(w, err.Error())
	} else {

		// Ensure that if either 'text' or 'html' was not provided, its value
		// is populated by the other field
		if e.Html == "" {
			e.Html = html.EscapeString(e.Text)
		} else if e.Text == "" {
			e.Text = sanitize.HTML(e.Html)
		}

		// Convert the email into an array of messages
		if messages, err := e.Messages(); err == nil {

			// Deliver each of the messages to the queue
			for _, m := range messages {
				c.Env["queue"].(*queue.Queue).Deliver(m)
			}

			// Respond with an empty object
			respondWithJSON(w, struct{}{})

		} else {
			respondWithError(w, err.Error())
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// syz-check does best-effort static correctness checking of the syscall descriptions in sys/os/*.txt.
// Use:
//	$ go install ./tools/syz-check
//	$ syz-check -obj /linux/vmlinux
// Currently it works only for linux and only for one arch at a time.
// The vmlinux files should include debug info and enable all relevant configs (since we parse dwarf).
// The results are produced in sys/os/*.warn files.
// On implementation level syz-check parses vmlinux dwarf, extracts struct descriptions
// and compares them with what we have (size, fields, alignment, etc).
package main

import (
	"bytes"
	"debug/dwarf"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"sort"

	"github.com/google/syzkaller/pkg/ast"
	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

func main() {
	var (
		flagOS           = flag.String("os", runtime.GOOS, "OS")
		flagArch         = flag.String("arch", runtime.GOARCH, "arch")
		flagKernelObject = flag.String("obj", "", "kernel object file")
		flagCPUProfile   = flag.String("cpuprofile", "", "write CPU profile to this file")
		flagMEMProfile   = flag.String("memprofile", "", "write memory profile to this file")
	)
	flag.Parse()
	if *flagCPUProfile != "" {
		f, err := os.Create(*flagCPUProfile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to create cpuprofile file: %v\n", err)
			os.Exit(1)
		}
		defer f.Close()
		if err := pprof.StartCPUProfile(f); err != nil {
			fmt.Fprintf(os.Stderr, "failed to start cpu profile: %v\n", err)
			os.Exit(1)
		}
		defer pprof.StopCPUProfile()
	}
	if *flagMEMProfile != "" {
		defer func() {
			f, err := os.Create(*flagMEMProfile)
			if err != nil {
				fmt.Fprintf(os.Stderr, "failed to create memprofile file: %v\n", err)
				os.Exit(1)
			}
			defer f.Close()
			runtime.GC()
			if err := pprof.WriteHeapProfile(f); err != nil {
				fmt.Fprintf(os.Stderr, "failed to write mem profile: %v\n", err)
				os.Exit(1)
			}
		}()
	}
	if err := check(*flagOS, *flagArch, *flagKernelObject); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func check(OS, arch, obj string) error {
	structs, err := parseKernelObject(obj)
	if err != nil {
		return err
	}
	structDescs, locs, err := parseDescriptions(OS, arch)
	if err != nil {
		return err
	}
	warnings, err := checkImpl(structs, structDescs, locs)
	if err != nil {
		return err
	}
	return writeWarnings(OS, arch, warnings)
}

type Warn struct {
	pos ast.Pos
	msg string
}

func writeWarnings(OS, arch string, warnings []Warn) error {
	allFiles, err := filepath.Glob(filepath.Join("sys", OS, "*.warn"))
	if err != nil {
		return err
	}
	toRemove := make(map[string]bool)
	for _, file := range allFiles {
		toRemove[file] = true
	}
	byFile := make(map[string][]Warn)
	for _, warn := range warnings {
		byFile[warn.pos.File] = append(byFile[warn.pos.File], warn)
	}
	for file, warns := range byFile {
		sort.Slice(warns, func(i, j int) bool {
			w1, w2 := warns[i], warns[j]
			if w1.pos.Line != w2.pos.Line {
				return w1.pos.Line < w2.pos.Line
			}
			return w1.msg < w2.msg
		})
		buf := new(bytes.Buffer)
		for _, warn := range warns {
			fmt.Fprintf(buf, "%v\n", warn.msg)
		}
		warnFile := filepath.Join("sys", OS, file+".warn")
		if err := osutil.WriteFile(warnFile, buf.Bytes()); err != nil {
			return err
		}
		delete(toRemove, warnFile)
	}
	for file := range toRemove {
		os.Remove(file)
	}
	return nil
}

func checkImpl(structs map[string]*dwarf.StructType, structDescs []*prog.KeyedStruct,
	locs map[string]*ast.Struct) ([]Warn, error) {
	var warnings []Warn
	checked := make(map[string]bool)
	for _, str := range structDescs {
		typ := str.Desc
		if typ.Varlen() {
			continue
		}
		astStruct := locs[typ.Name()]
		if astStruct == nil {
			// TODO: that's a template. Handle templates.
			continue
		}
		if checked[typ.Name()] {
			continue
		}
		checked[typ.Name()] = true
		warns, err := checkStruct(typ, astStruct, structs[typ.Name()])
		if err != nil {
			return nil, err
		}
		warnings = append(warnings, warns...)
	}
	return warnings, nil
}

func checkStruct(typ *prog.StructDesc, astStruct *ast.Struct, str *dwarf.StructType) ([]Warn, error) {
	var warnings []Warn
	warn := func(pos ast.Pos, msg string, args ...interface{}) {
		warnings = append(warnings, Warn{pos, fmt.Sprintf(msg, args...)})
	}
	if str == nil {
		warn(astStruct.Pos, "struct %v: no corresponding struct in kernel", typ.Name())
		return warnings, nil
	}
	if typ.Size() != uint64(str.ByteSize) {
		warn(astStruct.Pos, "struct %v: bad size: syz=%v kernel=%v", typ.Name(), typ.Size(), str.ByteSize)
	}
	// TODO: handle unions, currently we should report some false errors.
	// TODO: we could also check enums (elements match corresponding flags in syzkaller).
	// TODO: we could also check values of literal constants (dwarf should have that, right?).
	ai := 0
	offset := uint64(0)
	for _, field := range typ.Fields {
		if prog.IsPad(field) {
			offset += field.Size()
			continue
		}
		if ai < len(str.Field) {
			fld := str.Field[ai]
			if field.Size() != uint64(fld.Type.Size()) {
				warn(astStruct.Fields[ai].Pos, "field %v.%v/%v: bad size: syz=%v kernel=%v",
					typ.Name(), field.FieldName(), fld.Name, field.Size(), fld.Type.Size())
			}
			if offset != uint64(fld.ByteOffset) {
				warn(astStruct.Fields[ai].Pos, "field %v.%v/%v: bad offset: syz=%v kernel=%v",
					typ.Name(), field.FieldName(), fld.Name, offset, fld.ByteOffset)
			}
			// How would you define bitfield offset?
			// Offset of the beginning of the field from the beginning of the memory location, right?
			// No, DWARF defines it as offset of the end of the field from the end of the memory location.
			offset := fld.Type.Size()*8 - fld.BitOffset - fld.BitSize
			if fld.BitSize == 0 {
				// And to make things even more interesting this calculation
				// does not work for normal variables.
				offset = 0
			}
			if field.BitfieldLength() != uint64(fld.BitSize) ||
				field.BitfieldOffset() != uint64(offset) {
				warn(astStruct.Fields[ai].Pos, "field %v.%v/%v: bad bit size/offset: syz=%v/%v kernel=%v/%v",
					typ.Name(), field.FieldName(), fld.Name,
					field.BitfieldLength(), field.BitfieldOffset(),
					fld.BitSize, offset)
			}
		}
		ai++
		if !field.BitfieldMiddle() {
			offset += field.Size()
		}
	}
	if ai != len(str.Field) {
		warn(astStruct.Pos, "struct %v: bad number of fields: syz=%v kernel=%v", typ.Name(), ai, len(str.Field))
	}
	return warnings, nil
}

func parseDescriptions(OS, arch string) ([]*prog.KeyedStruct, map[string]*ast.Struct, error) {
	eh := func(pos ast.Pos, msg string) {}
	top := ast.ParseGlob(filepath.Join("sys", OS, "*.txt"), eh)
	if top == nil {
		return nil, nil, fmt.Errorf("failed to parse txt files")
	}
	consts := compiler.DeserializeConstsGlob(filepath.Join("sys", OS, "*_"+arch+".const"), eh)
	if consts == nil {
		return nil, nil, fmt.Errorf("failed to parse const files")
	}
	prg := compiler.Compile(top, consts, targets.Get(OS, arch), eh)
	if prg == nil {
		return nil, nil, fmt.Errorf("failed to compile descriptions")
	}
	prog.RestoreLinks(prg.Syscalls, prg.Resources, prg.StructDescs)
	locs := make(map[string]*ast.Struct)
	for _, decl := range top.Nodes {
		switch n := decl.(type) {
		case *ast.Struct:
			locs[n.Name.Name] = n
		}
	}
	return prg.StructDescs, locs, nil
}
<commit_msg>tools/syz-check: print descriptions compilation errors<commit_after>// Copyright 2019 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// syz-check does best-effort static correctness checking of the syscall descriptions in sys/os/*.txt.
// Use:
//	$ go install ./tools/syz-check
//	$ syz-check -obj /linux/vmlinux
// Currently it works only for linux and only for one arch at a time.
// The vmlinux files should include debug info and enable all relevant configs (since we parse dwarf).
// The results are produced in sys/os/*.warn files.
// On implementation level syz-check parses vmlinux dwarf, extracts struct descriptions
// and compares them with what we have (size, fields, alignment, etc).
package main

import (
	"bytes"
	"debug/dwarf"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"sort"

	"github.com/google/syzkaller/pkg/ast"
	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

func main() {
	var (
		flagOS           = flag.String("os", runtime.GOOS, "OS")
		flagArch         = flag.String("arch", runtime.GOARCH, "arch")
		flagKernelObject = flag.String("obj", "", "kernel object file")
		flagCPUProfile   = flag.String("cpuprofile", "", "write CPU profile to this file")
		flagMEMProfile   = flag.String("memprofile", "", "write memory profile to this file")
	)
	flag.Parse()
	if *flagCPUProfile != "" {
		f, err := os.Create(*flagCPUProfile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to create cpuprofile file: %v\n", err)
			os.Exit(1)
		}
		defer f.Close()
		if err := pprof.StartCPUProfile(f); err != nil {
			fmt.Fprintf(os.Stderr, "failed to start cpu profile: %v\n", err)
			os.Exit(1)
		}
		defer pprof.StopCPUProfile()
	}
	if *flagMEMProfile != "" {
		defer func() {
			f, err := os.Create(*flagMEMProfile)
			if err != nil {
				fmt.Fprintf(os.Stderr, "failed to create memprofile file: %v\n", err)
				os.Exit(1)
			}
			defer f.Close()
			runtime.GC()
			if err := pprof.WriteHeapProfile(f); err != nil {
				fmt.Fprintf(os.Stderr, "failed to write mem profile: %v\n", err)
				os.Exit(1)
			}
		}()
	}
	if err := check(*flagOS, *flagArch, *flagKernelObject); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func check(OS, arch, obj string) error {
	structDescs, locs, err := parseDescriptions(OS, arch)
	if err != nil {
		return err
	}
	structs, err := parseKernelObject(obj)
	if err != nil {
		return err
	}
	warnings, err := checkImpl(structs, structDescs, locs)
	if err != nil {
		return err
	}
	return writeWarnings(OS, arch, warnings)
}

type Warn struct {
	pos ast.Pos
	msg string
}

func writeWarnings(OS, arch string, warnings []Warn) error {
	allFiles, err := filepath.Glob(filepath.Join("sys", OS, "*.warn"))
	if err != nil {
		return err
	}
	toRemove := make(map[string]bool)
	for _, file := range allFiles {
		toRemove[file] = true
	}
	byFile := make(map[string][]Warn)
	for _, warn := range warnings {
		byFile[warn.pos.File] = append(byFile[warn.pos.File], warn)
	}
	for file, warns := range byFile {
		sort.Slice(warns, func(i, j int) bool {
			w1, w2 := warns[i], warns[j]
			if w1.pos.Line != w2.pos.Line {
				return w1.pos.Line < w2.pos.Line
			}
			return w1.msg < w2.msg
		})
		buf := new(bytes.Buffer)
		for _, warn := range warns {
			fmt.Fprintf(buf, "%v\n", warn.msg)
		}
		warnFile := filepath.Join("sys", OS, file+".warn")
		if err := osutil.WriteFile(warnFile, buf.Bytes()); err != nil {
			return err
		}
		delete(toRemove, warnFile)
	}
	for file := range toRemove {
		os.Remove(file)
	}
	return nil
}

func checkImpl(structs map[string]*dwarf.StructType, structDescs []*prog.KeyedStruct,
	locs map[string]*ast.Struct) ([]Warn, error) {
	var warnings []Warn
	checked := make(map[string]bool)
	for _, str := range structDescs {
		typ := str.Desc
		if typ.Varlen() {
			continue
		}
		astStruct := locs[typ.Name()]
		if astStruct == nil {
			// TODO: that's a template. Handle templates.
			continue
		}
		if checked[typ.Name()] {
			continue
		}
		checked[typ.Name()] = true
		warns, err := checkStruct(typ, astStruct, structs[typ.Name()])
		if err != nil {
			return nil, err
		}
		warnings = append(warnings, warns...)
	}
	return warnings, nil
}

func checkStruct(typ *prog.StructDesc, astStruct *ast.Struct, str *dwarf.StructType) ([]Warn, error) {
	var warnings []Warn
	warn := func(pos ast.Pos, msg string, args ...interface{}) {
		warnings = append(warnings, Warn{pos, fmt.Sprintf(msg, args...)})
	}
	if str == nil {
		warn(astStruct.Pos, "struct %v: no corresponding struct in kernel", typ.Name())
		return warnings, nil
	}
	if typ.Size() != uint64(str.ByteSize) {
		warn(astStruct.Pos, "struct %v: bad size: syz=%v kernel=%v", typ.Name(), typ.Size(), str.ByteSize)
	}
	// TODO: handle unions, currently we should report some false errors.
	// TODO: we could also check enums (elements match corresponding flags in syzkaller).
	// TODO: we could also check values of literal constants (dwarf should have that, right?).
	ai := 0
	offset := uint64(0)
	for _, field := range typ.Fields {
		if prog.IsPad(field) {
			offset += field.Size()
			continue
		}
		if ai < len(str.Field) {
			fld := str.Field[ai]
			if field.Size() != uint64(fld.Type.Size()) {
				warn(astStruct.Fields[ai].Pos, "field %v.%v/%v: bad size: syz=%v kernel=%v",
					typ.Name(), field.FieldName(), fld.Name, field.Size(), fld.Type.Size())
			}
			if offset != uint64(fld.ByteOffset) {
				warn(astStruct.Fields[ai].Pos, "field %v.%v/%v: bad offset: syz=%v kernel=%v",
					typ.Name(), field.FieldName(), fld.Name, offset, fld.ByteOffset)
			}
			// How would you define bitfield offset?
			// Offset of the beginning of the field from the beginning of the memory location, right?
			// No, DWARF defines it as offset of the end of the field from the end of the memory location.
			offset := fld.Type.Size()*8 - fld.BitOffset - fld.BitSize
			if fld.BitSize == 0 {
				// And to make things even more interesting this calculation
				// does not work for normal variables.
				offset = 0
			}
			if field.BitfieldLength() != uint64(fld.BitSize) ||
				field.BitfieldOffset() != uint64(offset) {
				warn(astStruct.Fields[ai].Pos, "field %v.%v/%v: bad bit size/offset: syz=%v/%v kernel=%v/%v",
					typ.Name(), field.FieldName(), fld.Name,
					field.BitfieldLength(), field.BitfieldOffset(),
					fld.BitSize, offset)
			}
		}
		ai++
		if !field.BitfieldMiddle() {
			offset += field.Size()
		}
	}
	if ai != len(str.Field) {
		warn(astStruct.Pos, "struct %v: bad number of fields: syz=%v kernel=%v", typ.Name(), ai, len(str.Field))
	}
	return warnings, nil
}

func parseDescriptions(OS, arch string) ([]*prog.KeyedStruct, map[string]*ast.Struct, error) {
	errorBuf := new(bytes.Buffer)
	eh := func(pos ast.Pos, msg string) {
		fmt.Fprintf(errorBuf, "%v: %v\n", pos, msg)
	}
	top := ast.ParseGlob(filepath.Join("sys", OS, "*.txt"), eh)
	if top == nil {
		return nil, nil, fmt.Errorf("failed to parse txt files:\n%s", errorBuf.Bytes())
	}
	consts := compiler.DeserializeConstsGlob(filepath.Join("sys", OS, "*_"+arch+".const"), eh)
	if consts == nil {
		return nil, nil, fmt.Errorf("failed to parse const files:\n%s", errorBuf.Bytes())
	}
	prg := compiler.Compile(top, consts, targets.Get(OS, arch), eh)
	if prg == nil {
		return nil, nil, fmt.Errorf("failed to compile descriptions:\n%s", errorBuf.Bytes())
	}
	prog.RestoreLinks(prg.Syscalls, prg.Resources, prg.StructDescs)
	locs := make(map[string]*ast.Struct)
	for _, decl := range top.Nodes {
		switch n := decl.(type) {
		case *ast.Struct:
			locs[n.Name.Name] = n
		}
	}
	return prg.StructDescs, locs, nil
}
<|endoftext|>