<commit_before>// Copyright 2014 The oauth2 Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package internal contains support packages for oauth2 package.
package internal

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"
)

// Token represents the crendentials used to authorize
// the requests to access protected resources on the OAuth 2.0
// provider's backend.
//
// This type is a mirror of oauth2.Token and exists to break
// an otherwise-circular dependency. Other internal packages
// should convert this Token into an oauth2.Token before use.
type Token struct {
	// AccessToken is the token that authorizes and authenticates
	// the requests.
	AccessToken string

	// TokenType is the type of token.
	// The Type method returns either this or "Bearer", the default.
	TokenType string

	// RefreshToken is a token that's used by the application
	// (as opposed to the user) to refresh the access token
	// if it expires.
	RefreshToken string

	// Expiry is the optional expiration time of the access token.
	//
	// If zero, TokenSource implementations will reuse the same
	// token forever and RefreshToken or equivalent
	// mechanisms for that TokenSource will not be used.
	Expiry time.Time

	// Raw optionally contains extra metadata from the server
	// when updating a token.
	Raw interface{}
}

// tokenJSON is the struct representing the HTTP response from OAuth2
// providers returning a token in JSON form.
type tokenJSON struct {
	AccessToken  string         `json:"access_token"`
	TokenType    string         `json:"token_type"`
	RefreshToken string         `json:"refresh_token"`
	ExpiresIn    expirationTime `json:"expires_in"` // at least PayPal returns string, while most return number
	Expires      expirationTime `json:"expires"`    // broken Facebook spelling of expires_in
}

func (e *tokenJSON) expiry() (t time.Time) {
	if v := e.ExpiresIn; v != 0 {
		return time.Now().Add(time.Duration(v) * time.Second)
	}
	if v := e.Expires; v != 0 {
		return time.Now().Add(time.Duration(v) * time.Second)
	}
	return
}

type expirationTime int32

func (e *expirationTime) UnmarshalJSON(b []byte) error {
	var n json.Number
	err := json.Unmarshal(b, &n)
	if err != nil {
		return err
	}
	i, err := n.Int64()
	if err != nil {
		return err
	}
	*e = expirationTime(i)
	return nil
}

var brokenAuthHeaderProviders = []string{
	"https://accounts.google.com/",
	"https://www.googleapis.com/",
	"https://github.com/",
	"https://api.instagram.com/",
	"https://www.douban.com/",
	"https://api.dropbox.com/",
	"https://api.soundcloud.com/",
	"https://www.linkedin.com/",
	"https://api.twitch.tv/",
	"https://oauth.vk.com/",
	"https://api.odnoklassniki.ru/",
	"https://connect.stripe.com/",
	"https://api.pushbullet.com/",
	"https://oauth.sandbox.trainingpeaks.com/",
	"https://oauth.trainingpeaks.com/",
	"https://www.strava.com/oauth/",
	"https://app.box.com/",
	"https://test-sandbox.auth.corp.google.com",
}

// providerAuthHeaderWorks reports whether the OAuth2 server identified by the tokenURL
// implements the OAuth2 spec correctly
// See https://code.google.com/p/goauth2/issues/detail?id=31 for background.
// In summary:
// - Reddit only accepts client secret in the Authorization header
// - Dropbox accepts either it in URL param or Auth header, but not both.
// - Google only accepts URL param (not spec compliant?), not Auth header
// - Stripe only accepts client secret in Auth header with Bearer method, not Basic
func providerAuthHeaderWorks(tokenURL string) bool {
	for _, s := range brokenAuthHeaderProviders {
		if strings.HasPrefix(tokenURL, s) {
			// Some sites fail to implement the OAuth2 spec fully.
			return false
		}
	}

	// Assume the provider implements the spec properly
	// otherwise. We can add more exceptions as they're
	// discovered. We will _not_ be adding configurable hooks
	// to this package to let users select server bugs.
	return true
}

func RetrieveToken(ctx context.Context, ClientID, ClientSecret, TokenURL string, v url.Values) (*Token, error) {
	hc, err := ContextClient(ctx)
	if err != nil {
		return nil, err
	}
	v.Set("client_id", ClientID)
	bustedAuth := !providerAuthHeaderWorks(TokenURL)
	if bustedAuth && ClientSecret != "" {
		v.Set("client_secret", ClientSecret)
	}
	req, err := http.NewRequest("POST", TokenURL, strings.NewReader(v.Encode()))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	if !bustedAuth {
		req.SetBasicAuth(ClientID, ClientSecret)
	}
	r, err := hc.Do(req)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1<<20))
	if err != nil {
		return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err)
	}
	if code := r.StatusCode; code < 200 || code > 299 {
		return nil, fmt.Errorf("oauth2: cannot fetch token: %v\nResponse: %s", r.Status, body)
	}

	var token *Token
	content, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
	switch content {
	case "application/x-www-form-urlencoded", "text/plain":
		vals, err := url.ParseQuery(string(body))
		if err != nil {
			return nil, err
		}
		token = &Token{
			AccessToken:  vals.Get("access_token"),
			TokenType:    vals.Get("token_type"),
			RefreshToken: vals.Get("refresh_token"),
			Raw:          vals,
		}
		e := vals.Get("expires_in")
		if e == "" {
			// TODO(jbd): Facebook's OAuth2 implementation is broken and
			// returns expires_in field in expires. Remove the fallback to expires,
			// when Facebook fixes their implementation.
			e = vals.Get("expires")
		}
		expires, _ := strconv.Atoi(e)
		if expires != 0 {
			token.Expiry = time.Now().Add(time.Duration(expires) * time.Second)
		}
	default:
		var tj tokenJSON
		if err = json.Unmarshal(body, &tj); err != nil {
			return nil, err
		}
		token = &Token{
			AccessToken:  tj.AccessToken,
			TokenType:    tj.TokenType,
			RefreshToken: tj.RefreshToken,
			Expiry:       tj.expiry(),
			Raw:          make(map[string]interface{}),
		}
		json.Unmarshal(body, &token.Raw) // no error checks for optional fields
	}
	// Don't overwrite `RefreshToken` with an empty value
	// if this was a token refreshing request.
	if token.RefreshToken == "" {
		token.RefreshToken = v.Get("refresh_token")
	}
	return token, nil
}
<commit_msg>oauth2: add user.gini.net to the broken providers<commit_after>// Copyright 2014 The oauth2 Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package internal contains support packages for oauth2 package.
package internal

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"
)

// Token represents the crendentials used to authorize
// the requests to access protected resources on the OAuth 2.0
// provider's backend.
//
// This type is a mirror of oauth2.Token and exists to break
// an otherwise-circular dependency. Other internal packages
// should convert this Token into an oauth2.Token before use.
type Token struct {
	// AccessToken is the token that authorizes and authenticates
	// the requests.
	AccessToken string

	// TokenType is the type of token.
	// The Type method returns either this or "Bearer", the default.
	TokenType string

	// RefreshToken is a token that's used by the application
	// (as opposed to the user) to refresh the access token
	// if it expires.
	RefreshToken string

	// Expiry is the optional expiration time of the access token.
	//
	// If zero, TokenSource implementations will reuse the same
	// token forever and RefreshToken or equivalent
	// mechanisms for that TokenSource will not be used.
	Expiry time.Time

	// Raw optionally contains extra metadata from the server
	// when updating a token.
	Raw interface{}
}

// tokenJSON is the struct representing the HTTP response from OAuth2
// providers returning a token in JSON form.
type tokenJSON struct {
	AccessToken  string         `json:"access_token"`
	TokenType    string         `json:"token_type"`
	RefreshToken string         `json:"refresh_token"`
	ExpiresIn    expirationTime `json:"expires_in"` // at least PayPal returns string, while most return number
	Expires      expirationTime `json:"expires"`    // broken Facebook spelling of expires_in
}

func (e *tokenJSON) expiry() (t time.Time) {
	if v := e.ExpiresIn; v != 0 {
		return time.Now().Add(time.Duration(v) * time.Second)
	}
	if v := e.Expires; v != 0 {
		return time.Now().Add(time.Duration(v) * time.Second)
	}
	return
}

type expirationTime int32

func (e *expirationTime) UnmarshalJSON(b []byte) error {
	var n json.Number
	err := json.Unmarshal(b, &n)
	if err != nil {
		return err
	}
	i, err := n.Int64()
	if err != nil {
		return err
	}
	*e = expirationTime(i)
	return nil
}

var brokenAuthHeaderProviders = []string{
	"https://accounts.google.com/",
	"https://www.googleapis.com/",
	"https://github.com/",
	"https://api.instagram.com/",
	"https://www.douban.com/",
	"https://api.dropbox.com/",
	"https://api.soundcloud.com/",
	"https://www.linkedin.com/",
	"https://api.twitch.tv/",
	"https://oauth.vk.com/",
	"https://api.odnoklassniki.ru/",
	"https://connect.stripe.com/",
	"https://api.pushbullet.com/",
	"https://oauth.sandbox.trainingpeaks.com/",
	"https://oauth.trainingpeaks.com/",
	"https://www.strava.com/oauth/",
	"https://app.box.com/",
	"https://test-sandbox.auth.corp.google.com",
	"https://user.gini.net/",
}

// providerAuthHeaderWorks reports whether the OAuth2 server identified by the tokenURL
// implements the OAuth2 spec correctly
// See https://code.google.com/p/goauth2/issues/detail?id=31 for background.
// In summary:
// - Reddit only accepts client secret in the Authorization header
// - Dropbox accepts either it in URL param or Auth header, but not both.
// - Google only accepts URL param (not spec compliant?), not Auth header
// - Stripe only accepts client secret in Auth header with Bearer method, not Basic
func providerAuthHeaderWorks(tokenURL string) bool {
	for _, s := range brokenAuthHeaderProviders {
		if strings.HasPrefix(tokenURL, s) {
			// Some sites fail to implement the OAuth2 spec fully.
			return false
		}
	}

	// Assume the provider implements the spec properly
	// otherwise. We can add more exceptions as they're
	// discovered. We will _not_ be adding configurable hooks
	// to this package to let users select server bugs.
	return true
}

func RetrieveToken(ctx context.Context, ClientID, ClientSecret, TokenURL string, v url.Values) (*Token, error) {
	hc, err := ContextClient(ctx)
	if err != nil {
		return nil, err
	}
	v.Set("client_id", ClientID)
	bustedAuth := !providerAuthHeaderWorks(TokenURL)
	if bustedAuth && ClientSecret != "" {
		v.Set("client_secret", ClientSecret)
	}
	req, err := http.NewRequest("POST", TokenURL, strings.NewReader(v.Encode()))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	if !bustedAuth {
		req.SetBasicAuth(ClientID, ClientSecret)
	}
	r, err := hc.Do(req)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1<<20))
	if err != nil {
		return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err)
	}
	if code := r.StatusCode; code < 200 || code > 299 {
		return nil, fmt.Errorf("oauth2: cannot fetch token: %v\nResponse: %s", r.Status, body)
	}

	var token *Token
	content, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
	switch content {
	case "application/x-www-form-urlencoded", "text/plain":
		vals, err := url.ParseQuery(string(body))
		if err != nil {
			return nil, err
		}
		token = &Token{
			AccessToken:  vals.Get("access_token"),
			TokenType:    vals.Get("token_type"),
			RefreshToken: vals.Get("refresh_token"),
			Raw:          vals,
		}
		e := vals.Get("expires_in")
		if e == "" {
			// TODO(jbd): Facebook's OAuth2 implementation is broken and
			// returns expires_in field in expires. Remove the fallback to expires,
			// when Facebook fixes their implementation.
			e = vals.Get("expires")
		}
		expires, _ := strconv.Atoi(e)
		if expires != 0 {
			token.Expiry = time.Now().Add(time.Duration(expires) * time.Second)
		}
	default:
		var tj tokenJSON
		if err = json.Unmarshal(body, &tj); err != nil {
			return nil, err
		}
		token = &Token{
			AccessToken:  tj.AccessToken,
			TokenType:    tj.TokenType,
			RefreshToken: tj.RefreshToken,
			Expiry:       tj.expiry(),
			Raw:          make(map[string]interface{}),
		}
		json.Unmarshal(body, &token.Raw) // no error checks for optional fields
	}
	// Don't overwrite `RefreshToken` with an empty value
	// if this was a token refreshing request.
	if token.RefreshToken == "" {
		token.RefreshToken = v.Get("refresh_token")
	}
	return token, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"runtime"
	"strings"
	"syscall"

	"github.com/datawire/teleproxy/pkg/supervisor"
	"github.com/kballard/go-shellquote"
	"github.com/pkg/errors"
)

// RunAsInfo contains the information required to launch a subprocess as the
// user such that it is likely to function as if the user launched it
// themselves.
type RunAsInfo struct {
	Name string
	Cwd  string
	Env  []string
}

// GetRunAsInfo returns an RAI for the current user context
func GetRunAsInfo() (*RunAsInfo, error) {
	user, err := user.Current()
	if err != nil {
		return nil, errors.Wrap(err, "user.Current()")
	}
	cwd, err := os.Getwd()
	if err != nil {
		return nil, errors.Wrap(err, "os.Getwd()")
	}
	rai := &RunAsInfo{
		Name: user.Username,
		Cwd:  cwd,
		Env:  os.Environ(),
	}
	return rai, nil
}

// GuessRunAsInfo attempts to construct a RunAsInfo for the user logged in at
// the primary display
func GuessRunAsInfo(p *supervisor.Process) (*RunAsInfo, error) {
	res := RunAsInfo{}
	if runtime.GOOS != "linux" {
		return &res, nil
	}
	pidDirs, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, errors.Wrap(err, "read /proc")
	}
	for _, fi := range pidDirs {
		if !fi.IsDir() { // Skip /proc files
			continue
		}
		if fi.Sys().(*syscall.Stat_t).Uid == 0 { // Skip root processes
			continue
		}
		// Read the command line for this proc
		cmdline, err := ioutil.ReadFile("/proc/" + fi.Name() + "/cmdline")
		if err != nil {
			p.Logf("Guess/cmdline: Skipping %q: %v", fi.Name(), err)
			continue
		}
		// Skip programs that are not X
		args := bytes.FieldsFunc(cmdline, func(r rune) bool { return r == 0 || r == 32 })
		if len(args) == 0 || !bytes.ContainsRune(args[0], 'X') {
			continue
		}
		p.Logf("Guess: Trying env info from: %q", args[0])
		// Capture the environment for this proc
		environBlob, err := ioutil.ReadFile("/proc/" + fi.Name() + "/environ")
		if err != nil {
			p.Logf("Guess/environ: Skipping %q: %v", fi.Name(), err)
			continue
		}
		environBytes := bytes.Split(environBlob, []byte{0})
		environ := make([]string, len(environBytes))
		display := ""
		for idx := 0; idx < len(environBytes); idx++ {
			entry := string(environBytes[idx])
			environ[idx] = entry
			switch {
			case strings.HasPrefix(entry, "USER="):
				res.Name = entry[5:]
			case strings.HasPrefix(entry, "HOME="):
				res.Cwd = entry[5:]
			case strings.HasPrefix(entry, "DISPLAY="):
				display = entry[8:]
			}
		}
		if len(display) == 0 {
			display = os.Getenv("DISPLAY")
			if len(display) > 0 {
				environ = append(environ, fmt.Sprintf("DISPLAY=%s", display))
			}
		}
		res.Env = environ
		break
	}
	if len(res.Env) == 0 {
		return nil, errors.New("Guess: X server process not found")
	}
	if len(res.Cwd) == 0 || len(res.Name) == 0 {
		return nil, errors.New("Guess: Valid USER/HOME not found")
	}
	return &res, nil
}

// Command returns a supervisor.Cmd that is configured to run a subprocess as
// the user in this context.
func (rai *RunAsInfo) Command(p *supervisor.Process, args ...string) *supervisor.Cmd {
	if rai == nil {
		rai = &RunAsInfo{}
	}
	var cmd *supervisor.Cmd
	if rai.Name == "root" || len(rai.Name) == 0 {
		cmd = p.Command(args[0], args[1:]...)
	} else {
		cmd = p.Command("su", "-m", rai.Name, "-c", shellquote.Join(args...))
	}
	cmd.Env = rai.Env
	cmd.Dir = rai.Cwd
	return cmd
}
<commit_msg>Hack in a special case for MacOS. FIXME: should not be special<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"runtime"
	"strings"
	"syscall"

	"github.com/datawire/teleproxy/pkg/supervisor"
	"github.com/kballard/go-shellquote"
	"github.com/pkg/errors"
)

// RunAsInfo contains the information required to launch a subprocess as the
// user such that it is likely to function as if the user launched it
// themselves.
type RunAsInfo struct {
	Name string
	Cwd  string
	Env  []string
}

// GetRunAsInfo returns an RAI for the current user context
func GetRunAsInfo() (*RunAsInfo, error) {
	user, err := user.Current()
	if err != nil {
		return nil, errors.Wrap(err, "user.Current()")
	}
	cwd, err := os.Getwd()
	if err != nil {
		return nil, errors.Wrap(err, "os.Getwd()")
	}
	rai := &RunAsInfo{
		Name: user.Username,
		Cwd:  cwd,
		Env:  os.Environ(),
	}
	return rai, nil
}

// GuessRunAsInfo attempts to construct a RunAsInfo for the user logged in at
// the primary display
func GuessRunAsInfo(p *supervisor.Process) (*RunAsInfo, error) {
	res := RunAsInfo{}
	if runtime.GOOS != "linux" {
		return &res, nil
	}
	pidDirs, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, errors.Wrap(err, "read /proc")
	}
	for _, fi := range pidDirs {
		if !fi.IsDir() { // Skip /proc files
			continue
		}
		if fi.Sys().(*syscall.Stat_t).Uid == 0 { // Skip root processes
			continue
		}
		// Read the command line for this proc
		cmdline, err := ioutil.ReadFile("/proc/" + fi.Name() + "/cmdline")
		if err != nil {
			p.Logf("Guess/cmdline: Skipping %q: %v", fi.Name(), err)
			continue
		}
		// Skip programs that are not X
		args := bytes.FieldsFunc(cmdline, func(r rune) bool { return r == 0 || r == 32 })
		if len(args) == 0 || !bytes.ContainsRune(args[0], 'X') {
			continue
		}
		p.Logf("Guess: Trying env info from: %q", args[0])
		// Capture the environment for this proc
		environBlob, err := ioutil.ReadFile("/proc/" + fi.Name() + "/environ")
		if err != nil {
			p.Logf("Guess/environ: Skipping %q: %v", fi.Name(), err)
			continue
		}
		environBytes := bytes.Split(environBlob, []byte{0})
		environ := make([]string, len(environBytes))
		display := ""
		for idx := 0; idx < len(environBytes); idx++ {
			entry := string(environBytes[idx])
			environ[idx] = entry
			switch {
			case strings.HasPrefix(entry, "USER="):
				res.Name = entry[5:]
			case strings.HasPrefix(entry, "HOME="):
				res.Cwd = entry[5:]
			case strings.HasPrefix(entry, "DISPLAY="):
				display = entry[8:]
			}
		}
		if len(display) == 0 {
			display = os.Getenv("DISPLAY")
			if len(display) > 0 {
				environ = append(environ, fmt.Sprintf("DISPLAY=%s", display))
			}
		}
		res.Env = environ
		break
	}
	if len(res.Env) == 0 {
		return nil, errors.New("Guess: X server process not found")
	}
	if len(res.Cwd) == 0 || len(res.Name) == 0 {
		return nil, errors.New("Guess: Valid USER/HOME not found")
	}
	return &res, nil
}

// Command returns a supervisor.Cmd that is configured to run a subprocess as
// the user in this context.
func (rai *RunAsInfo) Command(p *supervisor.Process, args ...string) *supervisor.Cmd {
	if rai == nil {
		rai = &RunAsInfo{}
	}
	var cmd *supervisor.Cmd
	if rai.Name == "root" || len(rai.Name) == 0 {
		cmd = p.Command(args[0], args[1:]...)
	} else {
		if runtime.GOOS == "darwin" {
			// MacOS `su` doesn't appear to propagate signals and
			// `sudo` is always (?) available.
			sudoOpts := []string{"--user", rai.Name, "--set-home", "--preserve-env", "--"}
			cmd = p.Command("sudo", append(sudoOpts, args...)...)
		} else {
			// FIXME(ark3): The above _should_ work on Linux, but
			// doesn't work on my machine. I don't know why (yet).
			cmd = p.Command("su", "-m", rai.Name, "-c", shellquote.Join(args...))
		}
	}
	cmd.Env = rai.Env
	cmd.Dir = rai.Cwd
	return cmd
}
<|endoftext|>


<commit_before>package entrypoint

import (
	"context"
	"reflect"
	"sync"

	consulapi "github.com/hashicorp/consul/api"

	amb "github.com/datawire/ambassador/pkg/api/getambassador.io/v2"
	"github.com/datawire/ambassador/pkg/consulwatch"
	snapshotTypes "github.com/datawire/ambassador/pkg/snapshot/v1"
	"github.com/datawire/ambassador/pkg/watt"
)

func ReconcileConsul(ctx context.Context, consul *consul, s *snapshotTypes.KubernetesSnapshot) {
	var mappings []*amb.Mapping
	for _, a := range s.Annotations {
		m, ok := a.(*amb.Mapping)
		if ok && include(m.Spec.AmbassadorID) {
			mappings = append(mappings, m)
		}
	}

	var resolvers []*amb.ConsulResolver
	for _, cr := range s.ConsulResolvers {
		if include(cr.Spec.AmbassadorID) {
			resolvers = append(resolvers, cr)
		}
	}

	for _, m := range s.Mappings {
		if include(m.Spec.AmbassadorID) {
			mappings = append(mappings, m)
		}
	}

	consul.reconcile(s.ConsulResolvers, mappings)
}

type consul struct {
	watcher                   Watcher
	resolvers                 map[string]*resolver
	firstReconcileHasHappened bool

	// The changed method returns this channel. We write down this channel to signal that a new
	// snapshot is available since the last time the update method was invoke.
	coalescedDirty chan struct{}
	// Individual watches write to this when new endpoint data is available. It is always being read
	// by the implementation, so writing will never block.
	endpointsCh chan consulwatch.Endpoints

	// The mutex protects access to endpoints, keysForBootstrap, and bootstrapped.
	mutex            sync.Mutex
	endpoints        map[string]consulwatch.Endpoints
	keysForBootstrap []string
	bootstrapped     bool
}

func newConsul(ctx context.Context, watcher Watcher) *consul {
	result := &consul{
		watcher:        watcher,
		resolvers:      make(map[string]*resolver),
		coalescedDirty: make(chan struct{}),
		endpointsCh:    make(chan consulwatch.Endpoints),
		endpoints:      make(map[string]consulwatch.Endpoints),
	}
	go result.run(ctx)
	return result
}

func (c *consul) run(ctx context.Context) {
	dirty := false
	for {
		if dirty {
			select {
			case c.coalescedDirty <- struct{}{}:
				dirty = false
			case ep := <-c.endpointsCh:
				c.updateEndpoints(ep)
				dirty = true
			case <-ctx.Done():
				c.cleanup()
				return
			}
		} else {
			select {
			case ep := <-c.endpointsCh:
				c.updateEndpoints(ep)
				dirty = true
			case <-ctx.Done():
				c.cleanup()
				return
			}
		}
	}
}

func (c *consul) updateEndpoints(endpoints consulwatch.Endpoints) {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	c.endpoints[endpoints.Service] = endpoints
}

func (c *consul) changed() chan struct{} {
	return c.coalescedDirty
}

func (c *consul) update(snap *watt.ConsulSnapshot) {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	snap.Endpoints = make(map[string]consulwatch.Endpoints, len(c.endpoints))
	for k, v := range c.endpoints {
		snap.Endpoints[k] = v
	}
}

func (c *consul) isBootstrapped() bool {
	if !c.firstReconcileHasHappened {
		return false
	}
	c.mutex.Lock()
	defer c.mutex.Unlock()

	// we want bootstrappedness to be idempotent
	if c.bootstrapped {
		return true
	}

	for _, key := range c.keysForBootstrap {
		if _, ok := c.endpoints[key]; !ok {
			return false
		}
	}

	c.bootstrapped = true

	return true
}

// Stop all service watches.
func (c *consul) cleanup() {
	// XXX: do we care about a clean shutdown
	/*go func() {
		<-ctx.Done()
		w.Stop()
	}()*/

	c.reconcile(nil, nil)
}

// Start and stop consul service watches as needed in order to match the supplied set of resolvers
// and mappings.
func (c *consul) reconcile(resolvers []*amb.ConsulResolver, mappings []*amb.Mapping) {
	// ==First we compute resolvers and their related mappings without actualy changing anything.==
	resolversByName := make(map[string]*amb.ConsulResolver)
	for _, cr := range resolvers {
		// Ambassador can find resolvers in any namespace, but they're not partitioned
		// by namespace once located, so just save using the name.
		resolversByName[cr.GetName()] = cr
	}

	mappingsByResolver := make(map[string][]*amb.Mapping)
	for _, m := range mappings {
		// Everything here is keyed off m.Spec.Resolver -- again, it's fine to use a resolver
		// from any namespace, as long as it was loaded.
		//
		// (This implies that if you typo a resolver name, things won't work.)

		rname := m.Spec.Resolver

		if rname == "" {
			continue
		}

		_, ok := resolversByName[rname]
		if !ok {
			continue
		}
		mappingsByResolver[rname] = append(mappingsByResolver[rname], m)
	}

	// Prune any resolvers that don't actually have mappings
	for name := range resolversByName {
		_, ok := mappingsByResolver[name]
		if !ok {
			delete(resolversByName, name)
		}
	}

	// ==Now we implement the changes implied by resolversByName and mappingsByResolver.==

	// First we (re)create any new or modified resolvers.
	for name, cr := range resolversByName {
		oldr, ok := c.resolvers[name]
		// The resolver hasn't change so continue. Make sure we only compare the spec, since we
		// don't want to delete/recreate resolvers on things like label changes.
		if ok && reflect.DeepEqual(oldr.resolver.Spec, cr.Spec) {
			continue
		}
		// It exists, but is different, so we delete/recreate i.
		if ok {
			oldr.deleted()
		}
		c.resolvers[name] = newResolver(cr)
	}

	// Now we delete unneeded resolvers.
	for name, resolver := range c.resolvers {
		_, ok := resolversByName[name]
		if !ok {
			resolver.deleted()
			delete(c.resolvers, name)
		}
	}

	// Finally we reconcile each mapping.
	for rname, mappings := range mappingsByResolver {
		res := c.resolvers[rname]
		res.reconcile(c.watcher, mappings, c.endpointsCh)
	}

	// If this is the first time we are reconciling, we need to compute conditions for being
	// bootstrapped.
	if !c.firstReconcileHasHappened {
		c.firstReconcileHasHappened = true
		var keysForBootstrap []string
		for _, mappings := range mappingsByResolver {
			for _, m := range mappings {
				keysForBootstrap = append(keysForBootstrap, m.Spec.Service)
			}
		}
		c.mutex.Lock()
		defer c.mutex.Unlock()
		c.keysForBootstrap = keysForBootstrap
	}
}

type resolver struct {
	resolver *amb.ConsulResolver
	watches  map[string]Stopper
}

func newResolver(spec *amb.ConsulResolver) *resolver {
	return &resolver{resolver: spec, watches: make(map[string]Stopper)}
}

func (r *resolver) deleted() {
	for _, w := range r.watches {
		w.Stop()
	}
}

func (r *resolver) reconcile(watcher Watcher, mappings []*amb.Mapping, endpoints chan consulwatch.Endpoints) {
	servicesByName := make(map[string]bool)
	for _, m := range mappings {
		// XXX: how to parse this?
		svc := m.Spec.Service
		servicesByName[svc] = true
		w, ok := r.watches[svc]
		if !ok {
			w = watcher.Watch(r.resolver, m, endpoints)
			r.watches[svc] = w
		}
	}

	for name, w := range r.watches {
		_, ok := servicesByName[name]
		if !ok {
			w.Stop()
			delete(r.watches, name)
		}
	}
}

type Watcher interface {
	Watch(resolver *amb.ConsulResolver, mapping *amb.Mapping, endpoints chan consulwatch.Endpoints) Stopper
}

type Stopper interface {
	Stop()
}

type consulWatcher struct{}

func (cw *consulWatcher) Watch(resolver *amb.ConsulResolver, mapping *amb.Mapping,
	endpointsCh chan consulwatch.Endpoints) Stopper {
	// XXX: should this part be shared?
	consulConfig := consulapi.DefaultConfig()
	consulConfig.Address = resolver.Spec.Address
	consul, err := consulapi.NewClient(consulConfig)
	if err != nil {
		panic(err)
	}

	// this part is per service
	svc := mapping.Spec.Service
	w, err := consulwatch.New(consul, resolver.Spec.Datacenter, svc, true)
	if err != nil {
		panic(err)
	}
	w.Watch(func(endpoints consulwatch.Endpoints, e error) {
		endpointsCh <- endpoints
	})

	go func() {
		err = w.Start(context.TODO())
		if err != nil {
			panic(err)
		}
	}()

	return w
}
<commit_msg>(from AES) Pass the Consul datacenter back as the ID, like we always have.<commit_after>package entrypoint

import (
	"context"
	"reflect"
	"sync"

	consulapi "github.com/hashicorp/consul/api"

	amb "github.com/datawire/ambassador/pkg/api/getambassador.io/v2"
	"github.com/datawire/ambassador/pkg/consulwatch"
	snapshotTypes "github.com/datawire/ambassador/pkg/snapshot/v1"
	"github.com/datawire/ambassador/pkg/watt"
)

func ReconcileConsul(ctx context.Context, consul *consul, s *snapshotTypes.KubernetesSnapshot) {
	var mappings []*amb.Mapping
	for _, a := range s.Annotations {
		m, ok := a.(*amb.Mapping)
		if ok && include(m.Spec.AmbassadorID) {
			mappings = append(mappings, m)
		}
	}

	var resolvers []*amb.ConsulResolver
	for _, cr := range s.ConsulResolvers {
		if include(cr.Spec.AmbassadorID) {
			resolvers = append(resolvers, cr)
		}
	}

	for _, m := range s.Mappings {
		if include(m.Spec.AmbassadorID) {
			mappings = append(mappings, m)
		}
	}

	consul.reconcile(s.ConsulResolvers, mappings)
}

type consul struct {
	watcher                   Watcher
	resolvers                 map[string]*resolver
	firstReconcileHasHappened bool

	// The changed method returns this channel. We write down this channel to signal that a new
	// snapshot is available since the last time the update method was invoke.
	coalescedDirty chan struct{}
	// Individual watches write to this when new endpoint data is available. It is always being read
	// by the implementation, so writing will never block.
	endpointsCh chan consulwatch.Endpoints

	// The mutex protects access to endpoints, keysForBootstrap, and bootstrapped.
	mutex            sync.Mutex
	endpoints        map[string]consulwatch.Endpoints
	keysForBootstrap []string
	bootstrapped     bool
}

func newConsul(ctx context.Context, watcher Watcher) *consul {
	result := &consul{
		watcher:        watcher,
		resolvers:      make(map[string]*resolver),
		coalescedDirty: make(chan struct{}),
		endpointsCh:    make(chan consulwatch.Endpoints),
		endpoints:      make(map[string]consulwatch.Endpoints),
	}
	go result.run(ctx)
	return result
}

func (c *consul) run(ctx context.Context) {
	dirty := false
	for {
		if dirty {
			select {
			case c.coalescedDirty <- struct{}{}:
				dirty = false
			case ep := <-c.endpointsCh:
				c.updateEndpoints(ep)
				dirty = true
			case <-ctx.Done():
				c.cleanup()
				return
			}
		} else {
			select {
			case ep := <-c.endpointsCh:
				c.updateEndpoints(ep)
				dirty = true
			case <-ctx.Done():
				c.cleanup()
				return
			}
		}
	}
}

func (c *consul) updateEndpoints(endpoints consulwatch.Endpoints) {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	c.endpoints[endpoints.Service] = endpoints
}

func (c *consul) changed() chan struct{} {
	return c.coalescedDirty
}

func (c *consul) update(snap *watt.ConsulSnapshot) {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	snap.Endpoints = make(map[string]consulwatch.Endpoints, len(c.endpoints))
	for k, v := range c.endpoints {
		snap.Endpoints[k] = v
	}
}

func (c *consul) isBootstrapped() bool {
	if !c.firstReconcileHasHappened {
		return false
	}
	c.mutex.Lock()
	defer c.mutex.Unlock()

	// we want bootstrappedness to be idempotent
	if c.bootstrapped {
		return true
	}

	for _, key := range c.keysForBootstrap {
		if _, ok := c.endpoints[key]; !ok {
			return false
		}
	}

	c.bootstrapped = true

	return true
}

// Stop all service watches.
func (c *consul) cleanup() {
	// XXX: do we care about a clean shutdown
	/*go func() {
		<-ctx.Done()
		w.Stop()
	}()*/

	c.reconcile(nil, nil)
}

// Start and stop consul service watches as needed in order to match the supplied set of resolvers
// and mappings.
func (c *consul) reconcile(resolvers []*amb.ConsulResolver, mappings []*amb.Mapping) {
	// ==First we compute resolvers and their related mappings without actualy changing anything.==
	resolversByName := make(map[string]*amb.ConsulResolver)
	for _, cr := range resolvers {
		// Ambassador can find resolvers in any namespace, but they're not partitioned
		// by namespace once located, so just save using the name.
		resolversByName[cr.GetName()] = cr
	}

	mappingsByResolver := make(map[string][]*amb.Mapping)
	for _, m := range mappings {
		// Everything here is keyed off m.Spec.Resolver -- again, it's fine to use a resolver
		// from any namespace, as long as it was loaded.
		//
		// (This implies that if you typo a resolver name, things won't work.)

		rname := m.Spec.Resolver

		if rname == "" {
			continue
		}

		_, ok := resolversByName[rname]
		if !ok {
			continue
		}
		mappingsByResolver[rname] = append(mappingsByResolver[rname], m)
	}

	// Prune any resolvers that don't actually have mappings
	for name := range resolversByName {
		_, ok := mappingsByResolver[name]
		if !ok {
			delete(resolversByName, name)
		}
	}

	// ==Now we implement the changes implied by resolversByName and mappingsByResolver.==

	// First we (re)create any new or modified resolvers.
	for name, cr := range resolversByName {
		oldr, ok := c.resolvers[name]
		// The resolver hasn't change so continue. Make sure we only compare the spec, since we
		// don't want to delete/recreate resolvers on things like label changes.
		if ok && reflect.DeepEqual(oldr.resolver.Spec, cr.Spec) {
			continue
		}
		// It exists, but is different, so we delete/recreate i.
		if ok {
			oldr.deleted()
		}
		c.resolvers[name] = newResolver(cr)
	}

	// Now we delete unneeded resolvers.
	for name, resolver := range c.resolvers {
		_, ok := resolversByName[name]
		if !ok {
			resolver.deleted()
			delete(c.resolvers, name)
		}
	}

	// Finally we reconcile each mapping.
	for rname, mappings := range mappingsByResolver {
		res := c.resolvers[rname]
		res.reconcile(c.watcher, mappings, c.endpointsCh)
	}

	// If this is the first time we are reconciling, we need to compute conditions for being
	// bootstrapped.
	if !c.firstReconcileHasHappened {
		c.firstReconcileHasHappened = true
		var keysForBootstrap []string
		for _, mappings := range mappingsByResolver {
			for _, m := range mappings {
				keysForBootstrap = append(keysForBootstrap, m.Spec.Service)
			}
		}
		c.mutex.Lock()
		defer c.mutex.Unlock()
		c.keysForBootstrap = keysForBootstrap
	}
}

type resolver struct {
	resolver *amb.ConsulResolver
	watches  map[string]Stopper
}

func newResolver(spec *amb.ConsulResolver) *resolver {
	return &resolver{resolver: spec, watches: make(map[string]Stopper)}
}

func (r *resolver) deleted() {
	for _, w := range r.watches {
		w.Stop()
	}
}

func (r *resolver) reconcile(watcher Watcher, mappings []*amb.Mapping, endpoints chan consulwatch.Endpoints) {
	servicesByName := make(map[string]bool)
	for _, m := range mappings {
		// XXX: how to parse this?
		svc := m.Spec.Service
		servicesByName[svc] = true
		w, ok := r.watches[svc]
		if !ok {
			w = watcher.Watch(r.resolver, m, endpoints)
			r.watches[svc] = w
		}
	}

	for name, w := range r.watches {
		_, ok := servicesByName[name]
		if !ok {
			w.Stop()
			delete(r.watches, name)
		}
	}
}

type Watcher interface {
	Watch(resolver *amb.ConsulResolver, mapping *amb.Mapping, endpoints chan consulwatch.Endpoints) Stopper
}

type Stopper interface {
	Stop()
}

type consulWatcher struct{}

func (cw *consulWatcher) Watch(resolver *amb.ConsulResolver, mapping *amb.Mapping,
	endpointsCh chan consulwatch.Endpoints) Stopper {
	// XXX: should this part be shared?
	consulConfig := consulapi.DefaultConfig()
	consulConfig.Address = resolver.Spec.Address
	consul, err := consulapi.NewClient(consulConfig)
	if err != nil {
		panic(err)
	}

	// this part is per service
	svc := mapping.Spec.Service
	w, err := consulwatch.New(consul, resolver.Spec.Datacenter, svc, true)
	if err != nil {
		panic(err)
	}

	w.Watch(func(endpoints consulwatch.Endpoints, e error) {
		if endpoints.Id == "" {
			// For Ambassador, overwrite the Id with the resolver's datacenter -- the
			// Consul watcher doesn't actually hand back the DC, and we need it.
			endpoints.Id = resolver.Spec.Datacenter
		}

		endpointsCh <- endpoints
	})

	go func() {
		err = w.Start(context.TODO())
		if err != nil {
			panic(err)
		}
	}()

	return w
}
<|endoftext|>


<commit_before>package jwtauthd

import (
    "github.com/spf13/cobra"
    "github.com/spf13/pflag"
    "github.com/spf13/viper"
)

type Args struct {
    Port int
    Conf string
}

type JwtAuthCommand struct {
    Args  Args
    Cmd   *cobra.Command
    Viper *viper.Viper
}

var JwtAuth *JwtAuthCommand = &JwtAuthCommand{}

func UsageTemplate() string {
    return `Usage:{{if .Runnable}}
  {{if .HasAvailableFlags}}{{appendIfNotPresent .UseLine "[flags]"}}{{else}}{{.UseLine}}{{end}}{{end}}{{if .HasAvailableSubCommands}}
  {{ .CommandPath}} [command]{{end}}{{if gt .Aliases 0}}
Aliases:
  {{.NameAndAliases}}
{{end}}{{if .HasExample}}
Examples:
{{ .Example }}{{end}}{{ if .HasAvailableSubCommands}}
Commands:{{range .Commands}}{{if .IsAvailableCommand}}
  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableLocalFlags}}
Flags:
{{.LocalFlags.FlagUsages | trimRightSpace}}{{end}}{{ if .HasAvailableInheritedFlags}}
Global Flags:
{{.InheritedFlags.FlagUsages | trimRightSpace}}{{end}}{{if .HasHelpSubCommands}}
Additional help topics:{{range .Commands}}{{if .IsHelpCommand}}
  {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableSubCommands }}
Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
`
}

func init() {
    JwtAuth.Viper = viper.GetViper()

    JwtAuth.Cmd = &cobra.Command{
        Use: "jwt-authd",
        Short: "jwt auth server",
        Long: "a simple jwt auth server",
        SilenceErrors: true,
        RunE: func(cmd *cobra.Command, args []string) error {
            // todo
            return nil
        },
        PersistentPreRun: func(cmd *cobra.Command, args []string) {
            // todo
        },
    }

    var PFlags *pflag.FlagSet = JwtAuth.Cmd.Flags()

    PFlags.IntVarP(&JwtAuth.Args.Port, "port", "p", 6010, "Set the server listening port")
    PFlags.StringVarP(&JwtAuth.Args.Conf, "config", "c", "/etc/jwt_authd.json", "Set the config file path")

    // todo

    JwtAuth.Cmd.SetUsageTemplate(UsageTemplate())
}<commit_msg>useage<commit_after>package jwtauthd

import (
    "github.com/spf13/cobra"
    "github.com/spf13/pflag"
    "github.com/spf13/viper"
)

type Args struct {
    Port int
    Conf string
}

type JwtAuthCommand struct {
    Args  Args
    Cmd   *cobra.Command
    Viper *viper.Viper
}

var JwtAuth *JwtAuthCommand = &JwtAuthCommand{}

func UsageTemplate() string {
    return `Usage:{{if .Runnable}}
  {{if .HasAvailableFlags}}{{appendIfNotPresent .UseLine "[OPTIONS] COMMAND [arg...]"}}{{else}}{{.UseLine}}{{end}}{{end}}{{if .HasAvailableSubCommands}}
  {{ .CommandPath}} [command]{{end}}{{if gt .Aliases 0}}
Aliases:
  {{.NameAndAliases}}
{{end}}{{if .HasExample}}
Examples:
{{ .Example }}{{end}}{{ if .HasAvailableSubCommands}}
Commands:{{range .Commands}}{{if .IsAvailableCommand}}
  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableLocalFlags}}
Flags:
{{.LocalFlags.FlagUsages | trimRightSpace}}{{end}}{{ if .HasAvailableInheritedFlags}}
Global Flags:
{{.InheritedFlags.FlagUsages | trimRightSpace}}{{end}}{{if .HasHelpSubCommands}}
Additional help topics:{{range .Commands}}{{if .IsHelpCommand}}
  {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableSubCommands }}
Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
`
}

func init() {
    JwtAuth.Viper = viper.GetViper()
    JwtAuth.Cmd = &cobra.Command{
        Use: "jwt-authd",
        Short: "jwt auth server",
        Long: "a simple jwt auth server",
        SilenceErrors: true,
        RunE: func(cmd *cobra.Command, args []string) error {
            // todo
            return nil
        },
        PersistentPreRun: func(cmd *cobra.Command, args []string) {
            // todo
        },
    }

    var PFlags *pflag.FlagSet = JwtAuth.Cmd.Flags()

    PFlags.IntVarP(&JwtAuth.Args.Port, "port", "p", 6010, "Set the server listening port")
    PFlags.StringVarP(&JwtAuth.Args.Conf, "config", "c", "/etc/jwt_authd.json", "Set the config file path")

    // todo

    JwtAuth.Cmd.SetUsageTemplate(UsageTemplate())
}<|endoftext|>


<commit_before>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/ikawaha/kagome/tokenizer"
)

// subcommand property
var (
	CommandName  = "server"
	Description  = `run tokenize server`
	usageMessage = "%s [-http=:6060] [-udic userdic_file] [-mode (normal|search|extended)]\n"
	ErrorWriter  = os.Stderr
)

// options
type option struct {
	http    string
	udic    string
	mode    string
	flagSet *flag.FlagSet
}

// ContinueOnError ErrorHandling // Return a descriptive error.
// ExitOnError                   // Call os.Exit(2).
// PanicOnError                  // Call panic with a descriptive error.flag.ContinueOnError
func newOption(w io.Writer, eh flag.ErrorHandling) (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, eh),
	}
	// option settings
	o.flagSet.StringVar(&o.http, "http", ":6060", "HTTP service address (e.g., ':6060')")
	o.flagSet.StringVar(&o.udic, "udic", "", "user dictionary")
	o.flagSet.StringVar(&o.mode, "mode", "normal", "tokenize mode (normal|search|extended)")

	return
}

func (o *option) parse(args []string) (err error) {
	if err = o.flagSet.Parse(args); err != nil {
		return
	}
	// validations
	if nonFlag := o.flagSet.Args(); len(nonFlag) != 0 {
		return fmt.Errorf("invalid argument: %v", nonFlag)
	}
	if o.mode != "normal" && o.mode != "search" && o.mode != "extended" {
		return fmt.Errorf("unknown mode: %v", o.mode)
	}
	return
}

func OptionCheck(args []string) (err error) {
	opt := newOption(ioutil.Discard, flag.ContinueOnError)
	if e := opt.parse(args); e != nil {
		return fmt.Errorf("%v, %v", CommandName, e)
	}
	return nil
}

// command main
func command(opt *option) error {
	var udic tokenizer.UserDic
	if opt.udic != "" {
		var err error
		if udic, err = tokenizer.NewUserDic(opt.udic); err != nil {
			return err
		}
	}
	t := tokenizer.New()
	t.SetUserDic(udic)

	mux := http.NewServeMux()
	mux.Handle("/", &TokenizeDemoHandler{tokenizer: t})
	mux.Handle("/a", &TokenizeHandler{tokenizer: t})
	log.Fatal(http.ListenAndServe(opt.http, mux))

	return nil
}

type TokenizeHandler struct {
	tokenizer tokenizer.Tokenizer
}

func (h *TokenizeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	type record struct {
		ID       int      `json:"id"`
		Start    int      `json:"start"`
		End      int      `json:"end"`
		Surface  string   `json:"surface"`
		Class    string   `json:"class"`
		Features []string `json:"features"`
	}

	var body struct {
		Input string `json:"sentence"`
	}
	e := json.NewDecoder(r.Body).Decode(&body)
	if e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
	if body.Input == "" {
		fmt.Fprint(w, "{\"status\":true,\"tokens\":[]}")
		return
	}
	tokens := h.tokenizer.Analyze(body.Input, tokenizer.Normal)
	var rsp []record
	for _, tok := range tokens {
		if tok.ID == tokenizer.BosEosID {
			continue
		}
		fs := tok.Features()
		m := record{
			ID:       tok.ID,
			Class:    fmt.Sprintf("%v", tok.Class),
			Start:    tok.Start,
			End:      tok.End,
			Surface:  tok.Surface,
			Features: fs,
		}
		rsp = append(rsp, m)
	}
	j, e := json.Marshal(struct {
		Status bool     `json:"status"`
		Tokens []record `json:"tokens"`
	}{Status: true, Tokens: rsp})
	if e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
	if _, e := w.Write(j); e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
}

type TokenizeDemoHandler struct {
	tokenizer tokenizer.Tokenizer
}

func (h *TokenizeDemoHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	type record struct {
		Surface        string
		Pos            string
		Baseform       string
		Reading        string
		Pronounciation string
	}
	sen := r.FormValue("s")
	opt := r.FormValue("r")

	const cmdTimeout = 15 * time.Second
	var (
		records []record
		tokens  []tokenizer.Token
		svg     string
		cmdErr  string
	)

	switch opt {
	case "normal":
		tokens = h.tokenizer.Analyze(sen, tokenizer.Normal)
	case "search":
		tokens = h.tokenizer.Analyze(sen, tokenizer.Search)
	case "extended":
		tokens = h.tokenizer.Analyze(sen, tokenizer.Extended)
	case "lattice":
		if _, e := exec.LookPath("dot"); e != nil {
			log.Print("graphviz is not in your future\n")
			break
		}
		var buf bytes.Buffer
		cmd := exec.Command("dot", "-Tsvg")
		r, w := io.Pipe()
		cmd.Stdin = r
		cmd.Stdout = &buf
		cmd.Stderr = ErrorWriter
		if err := cmd.Start(); err != nil {
			cmdErr = "Error"
			log.Printf("process done with error = %v", err)
		}
		h.tokenizer.Dot(sen, w)
		w.Close()

		done := make(chan error, 1)
		go func() {
			done <- cmd.Wait()
		}()
		select {
		case <-time.After(cmdTimeout):
			if err := cmd.Process.Kill(); err != nil {
				log.Fatal("failed to kill: ", err)
			}
			cmdErr = "Time out"
			<-done
		case err := <-done:
			if err != nil {
				cmdErr = "Error"
				log.Printf("process done with error = %v", err)
			}
		}
		svg = buf.String()
		if pos := strings.Index(svg, "<svg"); pos > 0 {
			svg = svg[pos:]
		}
	}
	for _, tok := range tokens {
		if tok.ID == tokenizer.BosEosID {
			continue
		}
		m := record{Surface: tok.Surface}
		fs := tok.Features()
		switch len(fs) {
		case 9:
			m.Pos = strings.Join(fs[0:5], ",")
			m.Baseform = fs[6]
			m.Reading = fs[7]
			m.Pronounciation = fs[8]
		case 7:
			m.Pos = strings.Join(fs[0:5], ",")
			m.Baseform = fs[6]
			m.Reading = "*"
			m.Pronounciation = "*"
		case 3:
			m.Pos = fs[0]
			m.Baseform = fs[1]
			m.Reading = fs[2]
			m.Pronounciation = "*"
		}
		records = append(records, m)
	}
	d := struct {
		Sentence string
		Tokens   []record
		CmdErr   string
		GraphSvg template.HTML
		RadioOpt string
	}{Sentence: sen, Tokens: records, CmdErr: cmdErr, GraphSvg: template.HTML(svg), RadioOpt: opt}
	t := template.Must(template.New("top").Parse(demoHTML))
	if e := t.Execute(w, d); e != nil {
		http.Error(w, e.Error(), http.StatusInternalServerError)
	}
}

func Run(args []string) error {
	opt := newOption(ErrorWriter, flag.ExitOnError)
	if e := opt.parse(args); e != nil {
		Usage()
		PrintDefaults(flag.ExitOnError)
		return fmt.Errorf("%v, %v", CommandName, e)
	}
	return command(opt)
}

func Usage() {
	fmt.Fprintf(os.Stderr, usageMessage, CommandName)
}

func PrintDefaults(eh flag.ErrorHandling) {
	o := newOption(ErrorWriter, eh)
	o.flagSet.PrintDefaults()
}

var demoHTML = `
<!DOCTYPE html>

  <meta charset="UTF-8">
  <title>Kagome demo - japanese morphological analyzer</title>

Kagome

    Kagome is an open source Japanese morphological analyzer written in Golang

Feature summary

Word segmentation. Segmenting text into words (or morphemes)

Part-of-speech tagging. Assign word-categories (nouns, verbs, particles, adjectives, etc.)

Lemmatization. Get dictionary forms for inflected verbs and adjectives

Readings. Extract readings for kanji.

  <form class="frm" action="/_demo" method="POST">

    <textarea class="txar" rows="3" name="s" placeholder="Enter Japanese text below in UTF-8 and click tokenize.">{{.Sentence}}</textarea>

<input type="radio" name="r" value="normal" checked>Normal

<input type="radio" name="r" value="search" {{if eq .RadioOpt "search"}}checked{{end}}>Search

<input type="radio" name="r" value="extended" {{if eq .RadioOpt "extended"}}checked{{end}}>Extended

<input type="radio" name="r" value="lattice" {{if eq .RadioOpt "lattice"}}checked{{end}}>Lattice

<input class="btn" type="submit" value="Tokenize"/>

  </form>
  {{if .CmdErr}}
    {{.CmdErr}}
  {{end}}
  {{if .GraphSvg}}
    {{.GraphSvg}}
  {{end}}
  {{if .Tokens}}

    <thread>

      Surface
      Part-of-Speech
      Base Form
      Reading
      Pronounciation

</thread>

    {{range .Tokens}}

      {{.Surface}}
      {{.Pos}}
      {{.Baseform}}
      {{.Reading}}
      {{.Pronounciation}}

    {{end}}

  {{end}}

`
<commit_msg>Fix to show error message when graphviz is not installed<commit_after>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/ikawaha/kagome/tokenizer"
)

// subcommand property
var (
	CommandName  = "server"
	Description  = `run tokenize server`
	usageMessage = "%s [-http=:6060] [-udic userdic_file] [-mode (normal|search|extended)]\n"
	ErrorWriter  = os.Stderr
)

// options
type option struct {
	http    string
	udic    string
	mode    string
	flagSet *flag.FlagSet
}

// ContinueOnError ErrorHandling // Return a descriptive error.
// ExitOnError                   // Call os.Exit(2).
// PanicOnError                  // Call panic with a descriptive error.flag.ContinueOnError
func newOption(w io.Writer, eh flag.ErrorHandling) (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, eh),
	}
	// option settings
	o.flagSet.StringVar(&o.http, "http", ":6060", "HTTP service address (e.g., ':6060')")
	o.flagSet.StringVar(&o.udic, "udic", "", "user dictionary")
	o.flagSet.StringVar(&o.mode, "mode", "normal", "tokenize mode (normal|search|extended)")

	return
}

func (o *option) parse(args []string) (err error) {
	if err = o.flagSet.Parse(args); err != nil {
		return
	}
	// validations
	if nonFlag := o.flagSet.Args(); len(nonFlag) != 0 {
		return fmt.Errorf("invalid argument: %v", nonFlag)
	}
	if o.mode != "normal" && o.mode != "search" && o.mode != "extended" {
		return fmt.Errorf("unknown mode: %v", o.mode)
	}
	return
}

func OptionCheck(args []string) (err error) {
	opt := newOption(ioutil.Discard, flag.ContinueOnError)
	if e := opt.parse(args); e != nil {
		return fmt.Errorf("%v, %v", CommandName, e)
	}
	return nil
}

// command main
func command(opt *option) error {
	var udic tokenizer.UserDic
	if opt.udic != "" {
		var err error
		if udic, err = tokenizer.NewUserDic(opt.udic); err != nil {
			return err
		}
	}
	t := tokenizer.New()
	t.SetUserDic(udic)

	mux := http.NewServeMux()
	mux.Handle("/", &TokenizeDemoHandler{tokenizer: t})
	mux.Handle("/a", &TokenizeHandler{tokenizer: t})
	log.Fatal(http.ListenAndServe(opt.http, mux))

	return nil
}

type TokenizeHandler struct {
	tokenizer tokenizer.Tokenizer
}

func (h *TokenizeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	type record struct {
		ID       int      `json:"id"`
		Start    int      `json:"start"`
		End      int      `json:"end"`
		Surface  string   `json:"surface"`
		Class    string   `json:"class"`
		Features []string `json:"features"`
	}

	var body struct {
		Input string `json:"sentence"`
	}
	e := json.NewDecoder(r.Body).Decode(&body)
	if e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
	if body.Input == "" {
		fmt.Fprint(w, "{\"status\":true,\"tokens\":[]}")
		return
	}
	tokens := h.tokenizer.Analyze(body.Input, tokenizer.Normal)
	var rsp []record
	for _, tok := range tokens {
		if tok.ID == tokenizer.BosEosID {
			continue
		}
		fs := tok.Features()
		m := record{
			ID:       tok.ID,
			Class:    fmt.Sprintf("%v", tok.Class),
			Start:    tok.Start,
			End:      tok.End,
			Surface:  tok.Surface,
			Features: fs,
		}
		rsp = append(rsp, m)
	}
	j, e := json.Marshal(struct {
		Status bool     `json:"status"`
		Tokens []record `json:"tokens"`
	}{Status: true, Tokens: rsp})
	if e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
	if _, e := w.Write(j); e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
}

type TokenizeDemoHandler struct {
	tokenizer tokenizer.Tokenizer
}

func (h *TokenizeDemoHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	type record struct {
		Surface        string
		Pos            string
		Baseform       string
		Reading        string
		Pronounciation string
	}
	sen := r.FormValue("s")
	opt := r.FormValue("r")

	const cmdTimeout = 15 * time.Second
	var (
		records []record
		tokens  []tokenizer.Token
		svg     string
		cmdErr  string
	)

	switch opt {
	case "normal":
		tokens = h.tokenizer.Analyze(sen, tokenizer.Normal)
	case "search":
		tokens = h.tokenizer.Analyze(sen, tokenizer.Search)
	case "extended":
		tokens = h.tokenizer.Analyze(sen, tokenizer.Extended)
	case "lattice":
		if _, e := exec.LookPath("dot"); e != nil {
			cmdErr = "Error: graphviz is not in your furure"
			log.Print("graphviz is not in your future\n")
			break
		}
		var buf bytes.Buffer
		cmd := exec.Command("dot", "-Tsvg")
		r, w := io.Pipe()
		cmd.Stdin = r
		cmd.Stdout = &buf
		cmd.Stderr = ErrorWriter
		if err := cmd.Start(); err != nil {
			cmdErr = "Error"
			log.Printf("process done with error = %v", err)
		}
		h.tokenizer.Dot(sen, w)
		w.Close()

		done := make(chan error, 1)
		go func() {
			done <- cmd.Wait()
		}()
		select {
		case <-time.After(cmdTimeout):
			if err := cmd.Process.Kill(); err != nil {
				log.Fatal("failed to kill: ", err)
			}
			cmdErr = "Time out"
			<-done
		case err := <-done:
			if err != nil {
				cmdErr = "Error"
				log.Printf("process done with error = %v", err)
			}
		}
		svg = buf.String()
		if pos := strings.Index(svg, "<svg"); pos > 0 {
			svg = svg[pos:]
		}
	}
	for _, tok := range tokens {
		if tok.ID == tokenizer.BosEosID {
			continue
		}
		m := record{Surface: tok.Surface}
		fs := tok.Features()
		switch len(fs) {
		case 9:
			m.Pos = strings.Join(fs[0:5], ",")
			m.Baseform = fs[6]
			m.Reading = fs[7]
			m.Pronounciation = fs[8]
		case 7:
			m.Pos = strings.Join(fs[0:5], ",")
			m.Baseform = fs[6]
			m.Reading = "*"
			m.Pronounciation = "*"
		case 3:
			m.Pos = fs[0]
			m.Baseform = fs[1]
			m.Reading = fs[2]
			m.Pronounciation = "*"
		}
		records = append(records, m)
	}
	d := struct {
		Sentence string
		Tokens   []record
		CmdErr   string
		GraphSvg template.HTML
		RadioOpt string
	}{Sentence: sen, Tokens: records, CmdErr: cmdErr, GraphSvg: template.HTML(svg), RadioOpt: opt}
	t := template.Must(template.New("top").Parse(demoHTML))
	if e := t.Execute(w, d); e != nil {
		http.Error(w, e.Error(), http.StatusInternalServerError)
	}
}

func Run(args []string) error {
	opt := newOption(ErrorWriter, flag.ExitOnError)
	if e := opt.parse(args); e != nil {
		Usage()
		PrintDefaults(flag.ExitOnError)
		return fmt.Errorf("%v, %v", CommandName, e)
	}
	return command(opt)
}

func Usage() {
	fmt.Fprintf(os.Stderr, usageMessage, CommandName)
}

func PrintDefaults(eh flag.ErrorHandling) {
	o := newOption(ErrorWriter, eh)
	o.flagSet.PrintDefaults()
}

var demoHTML = `
<!DOCTYPE html>

  <meta charset="UTF-8">
  <title>Kagome demo - japanese morphological analyzer</title>

Kagome

    Kagome is an open source Japanese morphological analyzer written in Golang

Feature summary

Word segmentation. Segmenting text into words (or morphemes)

Part-of-speech tagging. Assign word-categories (nouns, verbs, particles, adjectives, etc.)

Lemmatization. Get dictionary forms for inflected verbs and adjectives

Readings. Extract readings for kanji.

  <form class="frm" action="/_demo" method="POST">

    <textarea class="txar" rows="3" name="s" placeholder="Enter Japanese text below in UTF-8 and click tokenize.">{{.Sentence}}</textarea>

<input type="radio" name="r" value="normal" checked>Normal

<input type="radio" name="r" value="search" {{if eq .RadioOpt "search"}}checked{{end}}>Search

<input type="radio" name="r" value="extended" {{if eq .RadioOpt "extended"}}checked{{end}}>Extended

<input type="radio" name="r" value="lattice" {{if eq .RadioOpt "lattice"}}checked{{end}}>Lattice

<input class="btn" type="submit" value="Tokenize"/>

  </form>
  {{if .CmdErr}}
    {{.CmdErr}}
  {{end}}
  {{if .GraphSvg}}
    {{.GraphSvg}}
  {{end}}
  {{if .Tokens}}

    <thread>

      Surface
      Part-of-Speech
      Base Form
      Reading
      Pronounciation

</thread>

    {{range .Tokens}}

      {{.Surface}}
      {{.Pos}}
      {{.Baseform}}
      {{.Reading}}
      {{.Pronounciation}}

    {{end}}

  {{end}}

`
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"

	"github.com/mattn/go-mastodon"
	"github.com/urfave/cli"
)

func cmdAccount(c *cli.Context) error {
	client := c.App.Metadata["client"].(*mastodon.Client)
	account, err := client.GetAccountCurrentUser(context.Background())
	if err != nil {
		return err
	}
	fmt.Fprintf(c.App.Writer, "URI           : %v\n", account.Acct)
	fmt.Fprintf(c.App.Writer, "ID            : %v\n", account.ID)
	fmt.Fprintf(c.App.Writer, "Username      : %v\n", account.Username)
	fmt.Fprintf(c.App.Writer, "Acct          : %v\n", account.Acct)
	fmt.Fprintf(c.App.Writer, "DisplayName   : %v\n", account.DisplayName)
	fmt.Fprintf(c.App.Writer, "Locked        : %v\n", account.Locked)
	fmt.Fprintf(c.App.Writer, "CreatedAt     : %v\n", account.CreatedAt.Local())
	fmt.Fprintf(c.App.Writer, "FollowersCount: %v\n", account.FollowersCount)
	fmt.Fprintf(c.App.Writer, "FollowingCount: %v\n", account.FollowingCount)
	fmt.Fprintf(c.App.Writer, "StatusesCount : %v\n", account.StatusesCount)
	fmt.Fprintf(c.App.Writer, "Note          : %v\n", account.Note)
	fmt.Fprintf(c.App.Writer, "URL           : %v\n", account.URL)
	return nil
}
<commit_msg>extract text in account<commit_after>package main

import (
	"context"
	"fmt"

	"github.com/mattn/go-mastodon"
	"github.com/urfave/cli"
)

func cmdAccount(c *cli.Context) error {
	client := c.App.Metadata["client"].(*mastodon.Client)
	account, err := client.GetAccountCurrentUser(context.Background())
	if err != nil {
		return err
	}
	fmt.Fprintf(c.App.Writer, "URI           : %v\n", account.Acct)
	fmt.Fprintf(c.App.Writer, "ID            : %v\n", account.ID)
	fmt.Fprintf(c.App.Writer, "Username      : %v\n", account.Username)
	fmt.Fprintf(c.App.Writer, "Acct          : %v\n", account.Acct)
	fmt.Fprintf(c.App.Writer, "DisplayName   : %v\n", account.DisplayName)
	fmt.Fprintf(c.App.Writer, "Locked        : %v\n", account.Locked)
	fmt.Fprintf(c.App.Writer, "CreatedAt     : %v\n", account.CreatedAt.Local())
	fmt.Fprintf(c.App.Writer, "FollowersCount: %v\n", account.FollowersCount)
	fmt.Fprintf(c.App.Writer, "FollowingCount: %v\n", account.FollowingCount)
	fmt.Fprintf(c.App.Writer, "StatusesCount : %v\n", account.StatusesCount)
	fmt.Fprintf(c.App.Writer, "Note          : %v\n", textContent(account.Note))
	fmt.Fprintf(c.App.Writer, "URL           : %v\n", account.URL)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
	"net/http"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/grafana/metrictank/clock"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	"github.com/grafana/metrictank/util/align"
	log "github.com/sirupsen/logrus"
)

var (
	httpError    = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=http")    // could not execute http request
	decodeError  = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=decode")  // could not decode http response
	invalidError = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=invalid") // any other problem with the response itself
)

var metricsBySeries []partitionMetrics

func initMetricsBySeries() {
	for p := 0; p < int(partitionCount); p++ {
		metricsBySeries = append(metricsBySeries, NewPartitionMetrics(p))
	}
}

type partitionMetrics struct {
	lag              *stats.Gauge32 // time since the last value was recorded
	deltaSum         *stats.Gauge32 // total amount of drift between expected value and actual values
	numNans          *stats.Gauge32 // number of missing values for each series
	numPostNans      *stats.Gauge32 // number of non-null values for each series that come after a nan value
	numNonMatching   *stats.Gauge32 // number of points where value != ts
	correctNumPoints *stats.Bool    // whether the expected number of points were received
	correctAlignment *stats.Bool    // whether the last ts matches `now`
	correctSpacing   *stats.Bool    // whether all points are sorted and 1 period apart
}

func NewPartitionMetrics(p int) partitionMetrics {
	return partitionMetrics{
		//TODO enable metrics2docs by adding 'metric' prefix to each metric
		// parrot.monitoring.lag is the time since the last value was recorded
		lag: stats.NewGauge32WithTags("parrot.monitoring.lag", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.deltaSum is the total amount of drift between expected value and actual values
		deltaSum: stats.NewGauge32WithTags("parrot.monitoring.deltaSum", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.nans is the number of missing values for each series
		numNans: stats.NewGauge32WithTags("parrot.monitoring.nans", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.postnans is the number of non-null values for each series that come after a nan value
		numPostNans: stats.NewGauge32WithTags("parrot.monitoring.postnans", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.nonmatching is the total number of entries where drift occurred
		numNonMatching: stats.NewGauge32WithTags("parrot.monitoring.nonmatching", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.correctNumPoints is whether the expected number of points were received
		correctNumPoints: stats.NewBoolWithTags("parrot.monitoring.correctNumPoints", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.correctAlignment is whether the last ts matches `now`
		correctAlignment: stats.NewBoolWithTags("parrot.monitoring.correctAlignment", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.correctSpacing is whether all points are sorted and 1 period apart
		correctSpacing: stats.NewBoolWithTags("parrot.monitoring.correctSpacing", fmt.Sprintf(";partition=%d", p)),
	}
}

func monitor() {
	initMetricsBySeries()
	for tick := range clock.AlignedTickLossy(queryInterval) {

		resp := graphite.ExecuteRenderQuery(buildRequest(tick))
		if resp.HTTPErr != nil {
			httpError.Inc()
			continue
		}
		if resp.DecodeErr != nil {
			decodeError.Inc()
			continue
		}

		seenPartitions := make(map[int]struct{})

		// the response should contain partitionCount series entries, each of which named by a number,
		// covering each partition exactly once. and each partition series should be correct. otherwise is invalid.
		invalid := len(resp.Decoded) == int(partitionCount)

		for _, s := range resp.Decoded {
			partition, err := strconv.Atoi(s.Target)
			if err != nil {
				log.Debug("unable to parse partition", err)
				invalid = true
			} else {
				_, ok := seenPartitions[partition]
				if ok {
					// should not see same partition twice!
					invalid = true
				} else {
					ok := processPartitionSeries(s.Datapoints, partition, tick)
					if !ok {
						invalid = true
					}
				}
			}
		}

		// check whether we encountered all partitions we expected
		for p := 0; p < int(partitionCount); p++ {
			_, ok := seenPartitions[p]
			if !ok {
				invalid = true
			}
		}

		if invalid {
			invalidError.Inc()
		}
		statsGraphite.Report(tick)
	}
}

// a,b,c,null,null <-- valid response. just a bit laggy
// a,null,c,d,null <- invalid response.

func processPartitionSeries(points []graphite.Point, partition int, now time.Time) bool {
	lastTs := align.Forward(uint32(now.Unix()), uint32(testMetricsInterval.Seconds()))

	var nans, nonMatching, lastSeen uint32
	var deltaSum float64
	var postNanPoints uint32

	correctSpacing := true

	for i, dp := range points {
		if i > 0 {
			prev := points[i-1]
			if dp.Ts-prev.Ts != uint32(testMetricsInterval.Seconds()) {
				correctSpacing = false
			}
		}

		if math.IsNaN(dp.Val) {
			nans++
			continue
		}

		if nans > 0 {
			postNanPoints++
		}

		lastSeen = dp.Ts
		if diff := dp.Val - float64(dp.Ts); diff != 0 {
			log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
			deltaSum += diff
			nonMatching++
		}
	}

	metrics := metricsBySeries[partition]
	lag := uint32(atomic.LoadInt64(&lastPublish)) - lastSeen

	metrics.lag.SetUint32(lag)
	metrics.deltaSum.Set(int(math.Ceil(deltaSum)))
	metrics.numNans.SetUint32(nans)
	metrics.numPostNans.SetUint32(postNanPoints)
	metrics.numNonMatching.SetUint32(nonMatching)

	correctNumPoints := len(points) == int(lookbackPeriod/testMetricsInterval)
	correctAlignment := points[len(points)-1].Ts == lastTs

	metrics.correctNumPoints.Set(correctNumPoints)
	metrics.correctAlignment.Set(correctAlignment)
	metrics.correctSpacing.Set(correctSpacing)

	// we allow for lag and nans, that's subjective and should be monitored separately.
	// but we will say it's invalid if any of the clear-cut signals say so.
	return postNanPoints == 0 && nonMatching == 0 && correctNumPoints && correctAlignment && correctSpacing

}

func buildRequest(now time.Time) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "aliasByNode(parrot.testdata.*.identity.*, 2)")
	q.Set("from", strconv.Itoa(int(now.Add(-1*lookbackPeriod).Unix())))
	q.Set("until", strconv.Itoa(int(now.Unix())))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<commit_msg>fix lastPoint and num points check<commit_after>package main

import (
	"fmt"
	"math"
	"net/http"
	"strconv"
	"sync/atomic"

	"github.com/grafana/metrictank/clock"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	"github.com/grafana/metrictank/util/align"
	log "github.com/sirupsen/logrus"
)

var (
	httpError    = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=http")    // could not execute http request
	decodeError  = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=decode")  // could not decode http response
	invalidError = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=invalid") // any other problem with the response itself
)

var metricsBySeries []partitionMetrics

func initMetricsBySeries() {
	for p := 0; p < int(partitionCount); p++ {
		metricsBySeries = append(metricsBySeries, NewPartitionMetrics(p))
	}
}

type partitionMetrics struct {
	lag              *stats.Gauge32 // time since the last value was recorded
	deltaSum         *stats.Gauge32 // total amount of drift between expected value and actual values
	numNans          *stats.Gauge32 // number of missing values for each series
	numPostNans      *stats.Gauge32 // number of non-null values for each series that come after a nan value
	numNonMatching   *stats.Gauge32 // number of points where value != ts
	correctNumPoints *stats.Bool    // whether the expected number of points were received
	correctAlignment *stats.Bool    // whether the last ts matches `now`
	correctSpacing   *stats.Bool    // whether all points are sorted and 1 period apart
}

func NewPartitionMetrics(p int) partitionMetrics {
	return partitionMetrics{
		//TODO enable metrics2docs by adding 'metric' prefix to each metric
		// parrot.monitoring.lag is the time since the last value was recorded
		lag: stats.NewGauge32WithTags("parrot.monitoring.lag", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.deltaSum is the total amount of drift between expected value and actual values
		deltaSum: stats.NewGauge32WithTags("parrot.monitoring.deltaSum", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.nans is the number of missing values for each series
		numNans: stats.NewGauge32WithTags("parrot.monitoring.nans", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.postnans is the number of non-null values for each series that come after a nan value
		numPostNans: stats.NewGauge32WithTags("parrot.monitoring.postnans", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.nonmatching is the total number of entries where drift occurred
		numNonMatching: stats.NewGauge32WithTags("parrot.monitoring.nonmatching", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.correctNumPoints is whether the expected number of points were received
		correctNumPoints: stats.NewBoolWithTags("parrot.monitoring.correctNumPoints", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.correctAlignment is whether the last ts matches `now`
		correctAlignment: stats.NewBoolWithTags("parrot.monitoring.correctAlignment", fmt.Sprintf(";partition=%d", p)),
		// parrot.monitoring.correctSpacing is whether all points are sorted and 1 period apart
		correctSpacing: stats.NewBoolWithTags("parrot.monitoring.correctSpacing", fmt.Sprintf(";partition=%d", p)),
	}
}

func monitor() {
	initMetricsBySeries()
	for tick := range clock.AlignedTickLossy(queryInterval) {

		from := uint32(tick.Add(-1 * lookbackPeriod).Unix())
		to := uint32(tick.Unix())
		resp := graphite.ExecuteRenderQuery(buildRequest(from, to))
		if resp.HTTPErr != nil {
			httpError.Inc()
			continue
		}
		if resp.DecodeErr != nil {
			decodeError.Inc()
			continue
		}

		seenPartitions := make(map[int]struct{})

		// the response should contain partitionCount series entries, each of which named by a number,
		// covering each partition exactly once. and each partition series should be correct. otherwise is invalid.
		invalid := len(resp.Decoded) == int(partitionCount)

		for _, s := range resp.Decoded {
			partition, err := strconv.Atoi(s.Target)
			if err != nil {
				log.Debug("unable to parse partition", err)
				invalid = true
			} else {
				_, ok := seenPartitions[partition]
				if ok {
					// should not see same partition twice!
					invalid = true
				} else {
					ok := processPartitionSeries(s.Datapoints, partition, from, to)
					if !ok {
						invalid = true
					}
				}
			}
		}

		// check whether we encountered all partitions we expected
		for p := 0; p < int(partitionCount); p++ {
			_, ok := seenPartitions[p]
			if !ok {
				invalid = true
			}
		}

		if invalid {
			invalidError.Inc()
		}
		statsGraphite.Report(tick)
	}
}

// a,b,c,null,null <-- valid response. just a bit laggy
// a,null,c,d,null <- invalid response.

func processPartitionSeries(points []graphite.Point, partition int, from, to uint32) bool {

	var nans, nonMatching, lastSeen uint32
	var deltaSum float64
	var postNanPoints uint32

	correctSpacing := true

	for i, dp := range points {
		if i > 0 {
			prev := points[i-1]
			if dp.Ts-prev.Ts != uint32(testMetricsInterval.Seconds()) {
				correctSpacing = false
			}
		}

		if math.IsNaN(dp.Val) {
			nans++
			continue
		}

		if nans > 0 {
			postNanPoints++
		}

		lastSeen = dp.Ts
		if diff := dp.Val - float64(dp.Ts); diff != 0 {
			log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
			deltaSum += diff
			nonMatching++
		}
	}

	metrics := metricsBySeries[partition]
	lag := uint32(atomic.LoadInt64(&lastPublish)) - lastSeen

	metrics.lag.SetUint32(lag)
	metrics.deltaSum.Set(int(math.Ceil(deltaSum)))
	metrics.numNans.SetUint32(nans)
	metrics.numPostNans.SetUint32(postNanPoints)
	metrics.numNonMatching.SetUint32(nonMatching)

	// render api has from exclusive, to inclusive.
	expFirstTs := align.ForwardIfNotAligned(from+1, uint32(testMetricsInterval.Seconds()))
	expLastTs := align.BackwardIfNotAligned(to, uint32(testMetricsInterval.Seconds()))
	expNumPoints := int((expLastTs-expFirstTs)/uint32(testMetricsInterval.Seconds()) + 1)

	correctNumPoints := len(points) == expNumPoints
	correctAlignment := points[len(points)-1].Ts == expLastTs

	metrics.correctNumPoints.Set(correctNumPoints)
	metrics.correctAlignment.Set(correctAlignment)
	metrics.correctSpacing.Set(correctSpacing)

	// we allow for lag and nans, that's subjective and should be monitored separately.
	// but we will say it's invalid if any of the clear-cut signals say so.
	return postNanPoints == 0 && nonMatching == 0 && correctNumPoints && correctAlignment && correctSpacing

}

func buildRequest(from, to uint32) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "aliasByNode(parrot.testdata.*.identity.*, 2)")
	q.Set("from", strconv.Itoa(int(from)))
	q.Set("until", strconv.Itoa(int(to)))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<|endoftext|>


<commit_before>package cmd_test

import (
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/testing"
)

func initDefenestrate(args []string) (*cmd.SuperCommand, *TestCommand, error) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest"})
	tc := &TestCommand{Name: "defenestrate"}
	jc.Register(tc)
	return jc, tc, testing.InitCommand(jc, args)
}

type SuperCommandSuite struct{}

var _ = Suite(&SuperCommandSuite{})

const helpText = "\n    help\\s+- show help on a command or other topic"
const helpCommandsText = "commands:" + helpText

func (s *SuperCommandSuite) TestDispatch(c *C) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest"})
	info := jc.Info()
	c.Assert(info.Name, Equals, "jujutest")
	c.Assert(info.Args, Equals, "<command> ...")
	c.Assert(info.Doc, Matches, helpCommandsText)

	jc, _, err := initDefenestrate([]string{"discombobulate"})
	c.Assert(err, ErrorMatches, "unrecognized command: jujutest discombobulate")
	info = jc.Info()
	c.Assert(info.Name, Equals, "jujutest")
	c.Assert(info.Args, Equals, "<command> ...")
	c.Assert(info.Doc, Matches, "commands:\n    defenestrate - defenestrate the juju"+helpText)

	jc, tc, err := initDefenestrate([]string{"defenestrate"})
	c.Assert(err, IsNil)
	c.Assert(tc.Option, Equals, "")
	info = jc.Info()
	c.Assert(info.Name, Equals, "jujutest defenestrate")
	c.Assert(info.Args, Equals, "<something>")
	c.Assert(info.Doc, Equals, "defenestrate-doc")

	_, tc, err = initDefenestrate([]string{"defenestrate", "--option", "firmly"})
	c.Assert(err, IsNil)
	c.Assert(tc.Option, Equals, "firmly")

	_, tc, err = initDefenestrate([]string{"defenestrate", "gibberish"})
	c.Assert(err, ErrorMatches, `unrecognized args: \["gibberish"\]`)
}

func (s *SuperCommandSuite) TestRegister(c *C) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest"})
	jc.Register(&TestCommand{Name: "flip"})
	jc.Register(&TestCommand{Name: "flap"})
	badCall := func() { jc.Register(&TestCommand{Name: "flap"}) }
	c.Assert(badCall, PanicMatches, "command already registered: flap")
}

func (s *SuperCommandSuite) TestRegisterAlias(c *C) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest"})
	jc.Register(&TestCommand{Name: "flip", Aliases: []string{"flap", "flop"}})

	info := jc.Info()
	c.Assert(info.Doc, Equals, `commands:
    flap - alias for flip
    flip - flip the juju
    flop - alias for flip
    help - show help on a command or other topic`)
}

var commandsDoc = `commands:
    flapbabble - flapbabble the juju
    flip       - flip the juju`

func (s *SuperCommandSuite) TestInfo(c *C) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{
		Name:    "jujutest",
		Purpose: "to be purposeful",
		Doc:     "doc\nblah\ndoc",
	})
	info := jc.Info()
	c.Assert(info.Name, Equals, "jujutest")
	c.Assert(info.Purpose, Equals, "to be purposeful")
	// info doc starts with the jc.Doc and ends with the help command
	c.Assert(info.Doc, Matches, jc.Doc+"(.|\n)*")
	c.Assert(info.Doc, Matches, "(.|\n)*"+helpCommandsText)

	jc.Register(&TestCommand{Name: "flip"})
	jc.Register(&TestCommand{Name: "flapbabble"})
	info = jc.Info()
	c.Assert(info.Doc, Matches, jc.Doc+"\n\n"+commandsDoc+helpText)

	jc.Doc = ""
	info = jc.Info()
	c.Assert(info.Doc, Matches, commandsDoc+helpText)
}

func (s *SuperCommandSuite) TestLogging(c *C) {
	target, debug := log.Target(), log.Debug
	defer func() {
		log.SetTarget(target)
		log.Debug = debug
	}()
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest", Log: &cmd.Log{}})
	jc.Register(&TestCommand{Name: "blah"})
	ctx := testing.Context(c)
	code := cmd.Main(jc, ctx, []string{"blah", "--option", "error", "--debug"})
	c.Assert(code, Equals, 1)
	c.Assert(bufferString(ctx.Stderr), Matches, `^.* ERROR command failed: BAM!
error: BAM!
`)
}
<commit_msg>Test that the callback is called, errors are propogated, and the context is wired in.<commit_after>package cmd_test

import (
	"fmt"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/testing"
)

func initDefenestrate(args []string) (*cmd.SuperCommand, *TestCommand, error) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest"})
	tc := &TestCommand{Name: "defenestrate"}
	jc.Register(tc)
	return jc, tc, testing.InitCommand(jc, args)
}

type SuperCommandSuite struct{}

var _ = Suite(&SuperCommandSuite{})

const helpText = "\n    help\\s+- show help on a command or other topic"
const helpCommandsText = "commands:" + helpText

func (s *SuperCommandSuite) TestDispatch(c *C) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest"})
	info := jc.Info()
	c.Assert(info.Name, Equals, "jujutest")
	c.Assert(info.Args, Equals, "<command> ...")
	c.Assert(info.Doc, Matches, helpCommandsText)

	jc, _, err := initDefenestrate([]string{"discombobulate"})
	c.Assert(err, ErrorMatches, "unrecognized command: jujutest discombobulate")
	info = jc.Info()
	c.Assert(info.Name, Equals, "jujutest")
	c.Assert(info.Args, Equals, "<command> ...")
	c.Assert(info.Doc, Matches, "commands:\n    defenestrate - defenestrate the juju"+helpText)

	jc, tc, err := initDefenestrate([]string{"defenestrate"})
	c.Assert(err, IsNil)
	c.Assert(tc.Option, Equals, "")
	info = jc.Info()
	c.Assert(info.Name, Equals, "jujutest defenestrate")
	c.Assert(info.Args, Equals, "<something>")
	c.Assert(info.Doc, Equals, "defenestrate-doc")

	_, tc, err = initDefenestrate([]string{"defenestrate", "--option", "firmly"})
	c.Assert(err, IsNil)
	c.Assert(tc.Option, Equals, "firmly")

	_, tc, err = initDefenestrate([]string{"defenestrate", "gibberish"})
	c.Assert(err, ErrorMatches, `unrecognized args: \["gibberish"\]`)
}

func (s *SuperCommandSuite) TestRegister(c *C) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest"})
	jc.Register(&TestCommand{Name: "flip"})
	jc.Register(&TestCommand{Name: "flap"})
	badCall := func() { jc.Register(&TestCommand{Name: "flap"}) }
	c.Assert(badCall, PanicMatches, "command already registered: flap")
}

func (s *SuperCommandSuite) TestRegisterAlias(c *C) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest"})
	jc.Register(&TestCommand{Name: "flip", Aliases: []string{"flap", "flop"}})

	info := jc.Info()
	c.Assert(info.Doc, Equals, `commands:
    flap - alias for flip
    flip - flip the juju
    flop - alias for flip
    help - show help on a command or other topic`)
}

var commandsDoc = `commands:
    flapbabble - flapbabble the juju
    flip       - flip the juju`

func (s *SuperCommandSuite) TestInfo(c *C) {
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{
		Name:    "jujutest",
		Purpose: "to be purposeful",
		Doc:     "doc\nblah\ndoc",
	})
	info := jc.Info()
	c.Assert(info.Name, Equals, "jujutest")
	c.Assert(info.Purpose, Equals, "to be purposeful")
	// info doc starts with the jc.Doc and ends with the help command
	c.Assert(info.Doc, Matches, jc.Doc+"(.|\n)*")
	c.Assert(info.Doc, Matches, "(.|\n)*"+helpCommandsText)

	jc.Register(&TestCommand{Name: "flip"})
	jc.Register(&TestCommand{Name: "flapbabble"})
	info = jc.Info()
	c.Assert(info.Doc, Matches, jc.Doc+"\n\n"+commandsDoc+helpText)

	jc.Doc = ""
	info = jc.Info()
	c.Assert(info.Doc, Matches, commandsDoc+helpText)
}

func (s *SuperCommandSuite) TestLogging(c *C) {
	target, debug := log.Target(), log.Debug
	defer func() {
		log.SetTarget(target)
		log.Debug = debug
	}()
	jc := cmd.NewSuperCommand(cmd.SuperCommandParams{Name: "jujutest", Log: &cmd.Log{}})
	jc.Register(&TestCommand{Name: "blah"})
	ctx := testing.Context(c)
	code := cmd.Main(jc, ctx, []string{"blah", "--option", "error", "--debug"})
	c.Assert(code, Equals, 1)
	c.Assert(bufferString(ctx.Stderr), Matches, `^.* ERROR command failed: BAM!
error: BAM!
`)
}

func (s *SuperCommandSuite) TestMissingCallback(c *C) {
	var (
		calledName string
		calledArgs []string
	)

	callback := func(ctx *cmd.Context, subcommand string, args []string) error {
		calledName = subcommand
		calledArgs = args
		return nil
	}

	super := cmd.NewSuperCommand(cmd.SuperCommandParams{
		Name:            "jujutest",
		Log:             &cmd.Log{},
		MissingCallback: callback,
	})

	ctx := testing.Context(c)
	code := cmd.Main(super, ctx, []string{"foo", "bar", "baz", "--debug"})
	c.Assert(code, Equals, 0)
	c.Assert(testing.Stdout(ctx), Equals, "")
	c.Assert(testing.Stderr(ctx), Equals, "")
	c.Assert(calledName, Equals, "foo")
	c.Assert(calledArgs, DeepEquals, []string{"bar", "baz", "--debug"})
}

func (s *SuperCommandSuite) TestMissingCallbackErrors(c *C) {
	var (
		calledName string
		calledArgs []string
	)

	callback := func(ctx *cmd.Context, subcommand string, args []string) error {
		calledName = subcommand
		calledArgs = args
		return fmt.Errorf("command not found %q", subcommand)
	}

	super := cmd.NewSuperCommand(cmd.SuperCommandParams{
		Name:            "jujutest",
		Log:             &cmd.Log{},
		MissingCallback: callback,
	})

	ctx := testing.Context(c)
	code := cmd.Main(super, ctx, []string{"foo", "bar", "baz", "--debug"})
	c.Assert(code, Equals, 1)
	c.Assert(testing.Stdout(ctx), Equals, "")
	c.Assert(testing.Stderr(ctx), Equals, "error: command not found \"foo\"\n")
	c.Assert(calledName, Equals, "foo")
	c.Assert(calledArgs, DeepEquals, []string{"bar", "baz", "--debug"})
}

func (s *SuperCommandSuite) TestMissingCallbackContextWiredIn(c *C) {
	callback := func(ctx *cmd.Context, subcommand string, args []string) error {
		fmt.Fprintf(ctx.Stdout, "this is std out")
		fmt.Fprintf(ctx.Stderr, "this is std err")
		return nil
	}

	super := cmd.NewSuperCommand(cmd.SuperCommandParams{
		Name:            "jujutest",
		Log:             &cmd.Log{},
		MissingCallback: callback,
	})

	ctx := testing.Context(c)
	code := cmd.Main(super, ctx, []string{"foo", "bar", "baz", "--debug"})
	c.Assert(code, Equals, 0)
	c.Assert(testing.Stdout(ctx), Equals, "this is std out")
	c.Assert(testing.Stderr(ctx), Equals, "this is std err")
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"
)

var (
	stdoutFirstLines []string // The first 10 lines of stdout
	stdoutLastLines  []string // The last 50 lines of stdout
	stdoutMut        sync.Mutex
)

const (
	countRestarts = 5
	loopThreshold = 15 * time.Second
)

func monitorMain() {
	os.Setenv("STNORESTART", "yes")
	os.Setenv("STMONITORED", "yes")
	l.SetPrefix("[monitor] ")

	args := os.Args
	var restarts [countRestarts]time.Time

	sign := make(chan os.Signal, 1)
	sigTerm := syscall.Signal(0xf)
	signal.Notify(sign, os.Interrupt, sigTerm, os.Kill)

	for {
		if t := time.Since(restarts[0]); t < loopThreshold {
			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
			os.Exit(exitError)
		}

		copy(restarts[0:], restarts[1:])
		restarts[len(restarts)-1] = time.Now()

		cmd := exec.Command(args[0], args[1:]...)

		stderr, err := cmd.StderrPipe()
		if err != nil {
			l.Fatalln(err)
		}

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			l.Fatalln(err)
		}

		l.Infoln("Starting syncthing")
		err = cmd.Start()
		if err != nil {
			l.Fatalln(err)
		}

		// Let the next child process know that this is not the first time
		// it's starting up.
		os.Setenv("STRESTART", "yes")

		stdoutMut.Lock()
		stdoutFirstLines = make([]string, 0, 10)
		stdoutLastLines = make([]string, 0, 50)
		stdoutMut.Unlock()

		go copyStderr(stderr)
		go copyStdout(stdout)

		exit := make(chan error)

		go func() {
			exit <- cmd.Wait()
		}()

		select {
		case s := <-sign:
			l.Infof("Signal %d received; exiting", s)
			cmd.Process.Kill()
			<-exit
			return

		case err = <-exit:
			if err == nil {
				// Successfull exit indicates an intentional shutdown
				return
			} else if exiterr, ok := err.(*exec.ExitError); ok {
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					switch status.ExitStatus() {
					case exitUpgrading:
						// Restart the monitor process to release the .old
						// binary as part of the upgrade process.
						l.Infoln("Restarting monitor...")
						os.Setenv("STNORESTART", "")
						err := exec.Command(args[0], args[1:]...).Start()
						if err != nil {
							l.Warnln("restart:", err)
						}
						return
					}
				}
			}
		}

		l.Infoln("Syncthing exited:", err)
		time.Sleep(1 * time.Second)
	}
}

func copyStderr(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)

	var panicFd *os.File
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		if panicFd == nil {
			os.Stderr.WriteString(line)

			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
				panicFd, err = os.Create(filepath.Join(confDir, time.Now().Format("panic-20060102-150405.log")))
				if err != nil {
					l.Warnln("Create panic log:", err)
					continue
				}

				l.Warnf("Panic detected, writing to \"%s\"", panicFd.Name())
				l.Warnln("Please create an issue at https://github.com/syncthing/syncthing/issues/ with the panic log attached")

				panicFd.WriteString("Panic at " + time.Now().Format(time.RFC1123) + "\n")
				stdoutMut.Lock()
				for _, line := range stdoutFirstLines {
					panicFd.WriteString(line)
				}
				panicFd.WriteString("...\n")
				for _, line := range stdoutLastLines {
					panicFd.WriteString(line)
				}
			}
		}

		if panicFd != nil {
			panicFd.WriteString(line)
		}
	}
}

func copyStdout(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		stdoutMut.Lock()
		if len(stdoutFirstLines) < cap(stdoutFirstLines) {
			stdoutFirstLines = append(stdoutFirstLines, line)
		}
		if l := len(stdoutLastLines); l == cap(stdoutLastLines) {
			stdoutLastLines = stdoutLastLines[:l-1]
		}
		stdoutLastLines = append(stdoutLastLines, line)
		stdoutMut.Unlock()

		os.Stdout.WriteString(line)
	}
}
<commit_msg>Slightly improve panic log format (...)<commit_after>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"
)

var (
	stdoutFirstLines []string // The first 10 lines of stdout
	stdoutLastLines  []string // The last 50 lines of stdout
	stdoutMut        sync.Mutex
)

const (
	countRestarts = 5
	loopThreshold = 15 * time.Second
)

func monitorMain() {
	os.Setenv("STNORESTART", "yes")
	os.Setenv("STMONITORED", "yes")
	l.SetPrefix("[monitor] ")

	args := os.Args
	var restarts [countRestarts]time.Time

	sign := make(chan os.Signal, 1)
	sigTerm := syscall.Signal(0xf)
	signal.Notify(sign, os.Interrupt, sigTerm, os.Kill)

	for {
		if t := time.Since(restarts[0]); t < loopThreshold {
			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
			os.Exit(exitError)
		}

		copy(restarts[0:], restarts[1:])
		restarts[len(restarts)-1] = time.Now()

		cmd := exec.Command(args[0], args[1:]...)

		stderr, err := cmd.StderrPipe()
		if err != nil {
			l.Fatalln(err)
		}

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			l.Fatalln(err)
		}

		l.Infoln("Starting syncthing")
		err = cmd.Start()
		if err != nil {
			l.Fatalln(err)
		}

		// Let the next child process know that this is not the first time
		// it's starting up.
		os.Setenv("STRESTART", "yes")

		stdoutMut.Lock()
		stdoutFirstLines = make([]string, 0, 10)
		stdoutLastLines = make([]string, 0, 50)
		stdoutMut.Unlock()

		go copyStderr(stderr)
		go copyStdout(stdout)

		exit := make(chan error)

		go func() {
			exit <- cmd.Wait()
		}()

		select {
		case s := <-sign:
			l.Infof("Signal %d received; exiting", s)
			cmd.Process.Kill()
			<-exit
			return

		case err = <-exit:
			if err == nil {
				// Successfull exit indicates an intentional shutdown
				return
			} else if exiterr, ok := err.(*exec.ExitError); ok {
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					switch status.ExitStatus() {
					case exitUpgrading:
						// Restart the monitor process to release the .old
						// binary as part of the upgrade process.
						l.Infoln("Restarting monitor...")
						os.Setenv("STNORESTART", "")
						err := exec.Command(args[0], args[1:]...).Start()
						if err != nil {
							l.Warnln("restart:", err)
						}
						return
					}
				}
			}
		}

		l.Infoln("Syncthing exited:", err)
		time.Sleep(1 * time.Second)
	}
}

func copyStderr(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)

	var panicFd *os.File
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		if panicFd == nil {
			os.Stderr.WriteString(line)

			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
				panicFd, err = os.Create(filepath.Join(confDir, time.Now().Format("panic-20060102-150405.log")))
				if err != nil {
					l.Warnln("Create panic log:", err)
					continue
				}

				l.Warnf("Panic detected, writing to \"%s\"", panicFd.Name())
				l.Warnln("Please create an issue at https://github.com/syncthing/syncthing/issues/ with the panic log attached")

				stdoutMut.Lock()
				for _, line := range stdoutFirstLines {
					panicFd.WriteString(line)
				}
				panicFd.WriteString("...\n")
				for _, line := range stdoutLastLines {
					panicFd.WriteString(line)
				}
			}

			panicFd.WriteString("Panic at " + time.Now().Format(time.RFC3339) + "\n")
		}

		if panicFd != nil {
			panicFd.WriteString(line)
		}
	}
}

func copyStdout(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		stdoutMut.Lock()
		if len(stdoutFirstLines) < cap(stdoutFirstLines) {
			stdoutFirstLines = append(stdoutFirstLines, line)
		} else {
			if l := len(stdoutLastLines); l == cap(stdoutLastLines) {
				stdoutLastLines = stdoutLastLines[:l-1]
			}
			stdoutLastLines = append(stdoutLastLines, line)
		}
		stdoutMut.Unlock()

		os.Stdout.WriteString(line)
	}
}
<|endoftext|>