<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"encoding/json"
	"fmt"
	"path/filepath"

	"os"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/install"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	"golang.org/x/net/context"
)

const (
	defaultBytes = 1024 * 1024 * 16
	maxBytes     = 1024 * 1024 * 128
)

func NewCmdLogSend(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "send",
		Usage: "Send recent debug logs to keybase",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdLogSend{Contextified: libkb.NewContextified(g)}, "send", c)
			cl.SetForkCmd(libcmdline.NoFork)
			cl.SetLogForward(libcmdline.LogForwardNone)
		},
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "n",
				Usage: "Number of bytes in each log file to read",
			},
			cli.BoolFlag{
				Name:  "no-confirm",
				Usage: "Send logs without confirming",
			},
		},
	}
}

type CmdLogSend struct {
	libkb.Contextified
	numBytes  int
	noConfirm bool
	feedback  string
}

func (c *CmdLogSend) Run() error {

	if !c.noConfirm {
		if err := c.confirm(); err != nil {
			return err
		}
		if err := c.getFeedback(); err != nil {
			return err
		}
	}

	// if this fails for any reason, it is not a fatal error.
	// highly possible that the service isn't running and thus
	// can't get status.
	c.G().Log.Debug("attempting retrieval of keybase service status")
	var statusJSON string
	statusCmd := &CmdStatus{Contextified: libkb.NewContextified(c.G())}
	status, err := statusCmd.load()
	if err != nil {
		c.G().Log.Info("ignoring error getting keybase status: %s", err)
		statusJSON = c.errJSON(err)
	} else {
		json, err := json.Marshal(status)
		if err != nil {
			c.G().Log.Info("ignoring status json marshal error: %s", err)
			statusJSON = c.errJSON(err)
		} else {
			statusJSON = string(json)
		}
	}

	err = c.pokeUI()
	if err != nil {
		c.G().Log.Info("ignoring UI logs: %s", err)
	}

	logs := c.logFiles(status)
	// So far, install logs are Windows only
	if logs.Install != "" {
		defer os.Remove(logs.Install)
	}

	logSendContext := libkb.LogSendContext{
		Contextified: libkb.NewContextified(c.G()),
		Logs:         logs,
	}

	id, err := logSendContext.LogSend(statusJSON, c.feedback, true, c.numBytes)
	if err != nil {
		return err
	}

	c.outputInstructions(id)
	return nil
}

func (c *CmdLogSend) pokeUI() error {
	cli, err := GetLogsendClient(c.G())
	if err != nil {
		return err
	}
	return cli.PrepareLogsend(context.Background())
}

func (c *CmdLogSend) confirm() error {
	ui := c.G().UI.GetTerminalUI()
	ui.Printf("This command will send recent keybase log entries to keybase.io\n")
	ui.Printf("for debugging purposes only.\n\n")
	ui.Printf("These logs don’t include your private keys or encrypted data,\n")
	ui.Printf("but they will include filenames and other metadata keybase normally\n")
	ui.Printf("can’t read, for debugging purposes.\n\n")
	return ui.PromptForConfirmation("Continue sending logs to keybase.io?")
}

func (c *CmdLogSend) getFeedback() error {
	ui := c.G().UI.GetTerminalUI()
	var err error
	for err == nil {
		in, err := ui.Prompt(0, "Enter feedback (or ENTER to send): ")
		if err != nil {
			return err
		}
		if in != "" {
			c.feedback = c.feedback + in + "\n"
		} else {
			break
		}
	}
	return nil
}

func (c *CmdLogSend) outputInstructions(id string) {
	ui := c.G().UI.GetTerminalUI()

	ui.Printf("\n\n------------\n")
	ui.Printf("Success! Your log ID is:\n\n")
	ui.Printf("  %s\n\n", id)
	ui.Printf("Here's a URL to submit new bug reports containing this ID:\n\n")
	ui.Output("  https://github.com/keybase/client/issues/new?body=[write%20something%20useful%20and%20descriptive%20here]%0A%0Amy%20log%20id:%20" + id)
	ui.Printf("\n\nThanks!\n")
	ui.Printf("------------\n\n")
}

func (c *CmdLogSend) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) > 0 {
		return UnexpectedArgsError("log send")
	}
	c.noConfirm = ctx.Bool("no-confirm")
	c.numBytes = ctx.Int("n")
	if c.numBytes < 1 {
		c.numBytes = defaultBytes
	} else if c.numBytes > maxBytes {
		c.numBytes = maxBytes
	}
	return nil
}

func (c *CmdLogSend) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
		API:    true,
	}
}

func (c *CmdLogSend) logFiles(status *fstatus) libkb.Logs {
	logDir := c.G().Env.GetLogDir()
	installLogPath, err := install.InstallLogPath()
	if err != nil {
		c.G().Log.Errorf("Error (InstallLogPath): %s", err)
	}
	if status != nil {
		return libkb.Logs{
			Desktop: status.Desktop.Log,
			Kbfs:    status.KBFS.Log,
			Service: status.Service.Log,
			Updater: status.Updater.Log,
			Start:   status.Start.Log,
			Install: installLogPath,
			System:  install.SystemLogPath(),
		}
	}

	return libkb.Logs{
		Desktop: filepath.Join(logDir, libkb.DesktopLogFileName),
		Kbfs:    filepath.Join(logDir, libkb.KBFSLogFileName),
		Service: filepath.Join(logDir, libkb.ServiceLogFileName),
		Updater: filepath.Join(logDir, libkb.UpdaterLogFileName),
		Start:   filepath.Join(logDir, libkb.StartLogFileName),
		Install: installLogPath,
	}
}

func (c *CmdLogSend) errJSON(err error) string {
	return fmt.Sprintf("{\"Error\":%q}", err)
}
<commit_msg>Look for service PID when status fails during log send (#7290)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"encoding/json"
	"fmt"
	"path/filepath"

	"os"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/install"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	"golang.org/x/net/context"
)

const (
	defaultBytes = 1024 * 1024 * 16
	maxBytes     = 1024 * 1024 * 128
)

func NewCmdLogSend(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "send",
		Usage: "Send recent debug logs to keybase",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdLogSend{Contextified: libkb.NewContextified(g)}, "send", c)
			cl.SetForkCmd(libcmdline.NoFork)
			cl.SetLogForward(libcmdline.LogForwardNone)
		},
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "n",
				Usage: "Number of bytes in each log file to read",
			},
			cli.BoolFlag{
				Name:  "no-confirm",
				Usage: "Send logs without confirming",
			},
		},
	}
}

type CmdLogSend struct {
	libkb.Contextified
	numBytes  int
	noConfirm bool
	feedback  string
}

func (c *CmdLogSend) Run() error {

	if !c.noConfirm {
		if err := c.confirm(); err != nil {
			return err
		}
		if err := c.getFeedback(); err != nil {
			return err
		}
	}

	// if this fails for any reason, it is not a fatal error.
	// highly possible that the service isn't running and thus
	// can't get status.
	c.G().Log.Debug("attempting retrieval of keybase service status")
	var statusJSON string
	statusCmd := &CmdStatus{Contextified: libkb.NewContextified(c.G())}
	status, err := statusCmd.load()
	if err != nil {
		c.G().Log.Info("ignoring error getting keybase status: %s", err)
		// pid will be -1 if not found here
		pid, err2 := getPid(c.G())
		if err2 == nil {
			// Look for the process. os.FindProcess()
			// only fails if on Windows and no process is found.
			_, err2 := os.FindProcess(pid)
			if err2 != nil {
				pid = 0
			}
		}
		statusJSON = fmt.Sprintf("{\"pid\":%d, \"Error\":%q}", pid, err)
	} else {
		json, err := json.Marshal(status)
		if err != nil {
			c.G().Log.Info("ignoring status json marshal error: %s", err)
			statusJSON = c.errJSON(err)
		} else {
			statusJSON = string(json)
		}
	}

	err = c.pokeUI()
	if err != nil {
		c.G().Log.Info("ignoring UI logs: %s", err)
	}

	logs := c.logFiles(status)
	// So far, install logs are Windows only
	if logs.Install != "" {
		defer os.Remove(logs.Install)
	}

	logSendContext := libkb.LogSendContext{
		Contextified: libkb.NewContextified(c.G()),
		Logs:         logs,
	}

	id, err := logSendContext.LogSend(statusJSON, c.feedback, true, c.numBytes)
	if err != nil {
		return err
	}

	c.outputInstructions(id)
	return nil
}

func (c *CmdLogSend) pokeUI() error {
	cli, err := GetLogsendClient(c.G())
	if err != nil {
		return err
	}
	return cli.PrepareLogsend(context.Background())
}

func (c *CmdLogSend) confirm() error {
	ui := c.G().UI.GetTerminalUI()
	ui.Printf("This command will send recent keybase log entries to keybase.io\n")
	ui.Printf("for debugging purposes only.\n\n")
	ui.Printf("These logs don’t include your private keys or encrypted data,\n")
	ui.Printf("but they will include filenames and other metadata keybase normally\n")
	ui.Printf("can’t read, for debugging purposes.\n\n")
	return ui.PromptForConfirmation("Continue sending logs to keybase.io?")
}

func (c *CmdLogSend) getFeedback() error {
	ui := c.G().UI.GetTerminalUI()
	var err error
	for err == nil {
		in, err := ui.Prompt(0, "Enter feedback (or ENTER to send): ")
		if err != nil {
			return err
		}
		if in != "" {
			c.feedback = c.feedback + in + "\n"
		} else {
			break
		}
	}
	return nil
}

func (c *CmdLogSend) outputInstructions(id string) {
	ui := c.G().UI.GetTerminalUI()

	ui.Printf("\n\n------------\n")
	ui.Printf("Success! Your log ID is:\n\n")
	ui.Printf("  %s\n\n", id)
	ui.Printf("Here's a URL to submit new bug reports containing this ID:\n\n")
	ui.Output("  https://github.com/keybase/client/issues/new?body=[write%20something%20useful%20and%20descriptive%20here]%0A%0Amy%20log%20id:%20" + id)
	ui.Printf("\n\nThanks!\n")
	ui.Printf("------------\n\n")
}

func (c *CmdLogSend) ParseArgv(ctx *cli.Context) error {
	if len(ctx.Args()) > 0 {
		return UnexpectedArgsError("log send")
	}
	c.noConfirm = ctx.Bool("no-confirm")
	c.numBytes = ctx.Int("n")
	if c.numBytes < 1 {
		c.numBytes = defaultBytes
	} else if c.numBytes > maxBytes {
		c.numBytes = maxBytes
	}
	return nil
}

func (c *CmdLogSend) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config: true,
		API:    true,
	}
}

func (c *CmdLogSend) logFiles(status *fstatus) libkb.Logs {
	logDir := c.G().Env.GetLogDir()
	installLogPath, err := install.InstallLogPath()
	if err != nil {
		c.G().Log.Errorf("Error (InstallLogPath): %s", err)
	}
	if status != nil {
		return libkb.Logs{
			Desktop: status.Desktop.Log,
			Kbfs:    status.KBFS.Log,
			Service: status.Service.Log,
			Updater: status.Updater.Log,
			Start:   status.Start.Log,
			Install: installLogPath,
			System:  install.SystemLogPath(),
		}
	}

	return libkb.Logs{
		Desktop: filepath.Join(logDir, libkb.DesktopLogFileName),
		Kbfs:    filepath.Join(logDir, libkb.KBFSLogFileName),
		Service: filepath.Join(logDir, libkb.ServiceLogFileName),
		Updater: filepath.Join(logDir, libkb.UpdaterLogFileName),
		Start:   filepath.Join(logDir, libkb.StartLogFileName),
		Install: installLogPath,
	}
}

func (c *CmdLogSend) errJSON(err error) string {
	return fmt.Sprintf("{\"Error\":%q}", err)
}
<|endoftext|>


<commit_before>package ephemeral

import (
	"encoding/json"
	"fmt"

	"github.com/keybase/client/go/kbcrypto"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

type DeviceEKSeed keybase1.Bytes32

func newDeviceEphemeralSeed() (seed DeviceEKSeed, err error) {
	randomSeed, err := makeNewRandomSeed()
	if err != nil {
		return seed, err
	}
	return DeviceEKSeed(randomSeed), nil
}

func (s *DeviceEKSeed) DeriveDHKey() *libkb.NaclDHKeyPair {
	return deriveDHKey(keybase1.Bytes32(*s), libkb.DeriveReasonDeviceEKEncryption)
}

func postNewDeviceEK(mctx libkb.MetaContext, sig string) (err error) {
	defer mctx.TraceTimed("postNewDeviceEK", func() error { return err })()

	apiArg := libkb.APIArg{
		Endpoint:    "user/device_ek",
		SessionType: libkb.APISessionTypeREQUIRED,
		Args: libkb.HTTPArgs{
			"sig":       libkb.S{Val: sig},
			"device_id": libkb.S{Val: string(mctx.ActiveDevice().DeviceID())},
		},
	}
	_, err = mctx.G().GetAPI().Post(mctx, apiArg)
	return err
}

func serverMaxDeviceEK(mctx libkb.MetaContext, merkleRoot libkb.MerkleRoot) (maxGeneration keybase1.EkGeneration, err error) {
	defer mctx.TraceTimed("serverMaxDeviceEK", func() error { return err })()

	deviceEKs, err := allDeviceEKMetadataMaybeStale(mctx, merkleRoot)
	if err != nil {
		return maxGeneration, err
	}
	deviceID := mctx.ActiveDevice().DeviceID()
	metadata, ok := deviceEKs[deviceID]
	if ok {
		return metadata.Generation, nil
	}
	// We may not have an EK yet, let's try with this and fail if the server
	// rejects.
	mctx.Debug("No deviceEK found on the server")
	return 0, nil
}

func publishNewDeviceEK(mctx libkb.MetaContext, merkleRoot libkb.MerkleRoot) (metadata keybase1.DeviceEkMetadata, err error) {
	defer mctx.TraceTimed("publishNewDeviceEK", func() error { return err })()

	seed, err := newDeviceEphemeralSeed()
	if err != nil {
		return metadata, err
	}

	storage := mctx.G().GetDeviceEKStorage()
	generation, err := storage.MaxGeneration(mctx, true)
	if err != nil || generation < 0 {
		// Let's try to get the max from the server
		generation, err = serverMaxDeviceEK(mctx, merkleRoot)
		if err != nil {
			return metadata, err
		}
	}
	// This is our first generation
	if generation < 0 {
		generation = 0
	}
	generation++

	metadata, err = signAndPostDeviceEK(mctx, generation, seed, merkleRoot)
	if err != nil {
		mctx.Debug("Error posting deviceEK, retrying with server maxGeneration")
		// Let's retry posting with the server given max
		generation, err = serverMaxDeviceEK(mctx, merkleRoot)
		if err != nil {
			return metadata, err
		}
		generation++
		metadata, err = signAndPostDeviceEK(mctx, generation, seed, merkleRoot)
		if err != nil {
			return metadata, err
		}
	}

	return metadata, err
}

func signAndPostDeviceEK(mctx libkb.MetaContext, generation keybase1.EkGeneration,
	seed DeviceEKSeed, merkleRoot libkb.MerkleRoot) (metadata keybase1.DeviceEkMetadata, err error) {
	defer mctx.TraceTimed("signAndPostDeviceEK", func() error { return err })()

	storage := mctx.G().GetDeviceEKStorage()

	// Sign the statement blob with the device's long term signing key.
	signingKey, err := mctx.ActiveDevice().SigningKey()
	if err != nil {
		return metadata, err
	}

	dhKeypair := seed.DeriveDHKey()
	statement, signedStatement, err := signDeviceEKStatement(generation, dhKeypair, signingKey, merkleRoot)

	metadata = statement.CurrentDeviceEkMetadata
	// Ensure we successfully write the secret to disk before posting to the
	// server since the secret never leaves the device.
	if err = storage.Put(mctx, generation, keybase1.DeviceEk{
		Seed:     keybase1.Bytes32(seed),
		Metadata: metadata,
	}); err != nil {
		return metadata, err
	}

	err = postNewDeviceEK(mctx, signedStatement)
	if err != nil {
		storage.ClearCache()
		serr := NewDeviceEKStorage(mctx).Delete(mctx, generation)
		if serr != nil {
			mctx.Debug("DeviceEK deletion failed %v", err)
		}
	}

	return metadata, err
}

func signDeviceEKStatement(generation keybase1.EkGeneration, dhKeypair *libkb.NaclDHKeyPair, signingKey libkb.GenericKey, merkleRoot libkb.MerkleRoot) (statement keybase1.DeviceEkStatement, signedStatement string, err error) {
	metadata := keybase1.DeviceEkMetadata{
		Kid:        dhKeypair.GetKID(),
		Generation: generation,
		HashMeta:   merkleRoot.HashMeta(),
		// The ctime is derivable from the hash meta, by fetching the hashed
		// root from the server, but including it saves readers a potential
		// extra round trip.
		Ctime: keybase1.TimeFromSeconds(merkleRoot.Ctime()),
	}
	statement = keybase1.DeviceEkStatement{
		CurrentDeviceEkMetadata: metadata,
	}

	statementJSON, err := json.Marshal(statement)
	if err != nil {
		return statement, signedStatement, err
	}

	signedStatement, _, err = signingKey.SignToString(statementJSON)
	return statement, signedStatement, err
}

type deviceEKStatementResponse struct {
	Sigs []string `json:"sigs"`
}

func allDeviceEKMetadataMaybeStale(mctx libkb.MetaContext, merkleRoot libkb.MerkleRoot) (metadata map[keybase1.DeviceID]keybase1.DeviceEkMetadata, err error) {
	defer mctx.TraceTimed("allDeviceEKMetadataMaybeStale", func() error { return err })()

	apiArg := libkb.APIArg{
		Endpoint:    "user/device_eks",
		SessionType: libkb.APISessionTypeREQUIRED,
		Args:        libkb.HTTPArgs{},
	}
	res, err := mctx.G().GetAPI().Get(mctx, apiArg)
	if err != nil {
		return nil, err
	}

	parsedResponse := deviceEKStatementResponse{}
	err = res.Body.UnmarshalAgain(&parsedResponse)
	if err != nil {
		return nil, err
	}

	// Make a map of the user's own active devices, by KID. We'll use this to
	// match deviceEK sigs with the device that issued them below. (Checking
	// the signing key is intentionally the only way to do this, so that we're
	// forced to check authenticity.)
	kidToDevice := map[keybase1.KID]keybase1.PublicKey{}
	self, _, err := mctx.G().GetUPAKLoader().Load(libkb.NewLoadUserArgWithMetaContext(
		mctx).WithUID(mctx.ActiveDevice().UID()))
	if err != nil {
		return nil, err
	}
	for _, device := range self.Base.DeviceKeys {
		if device.IsRevoked {
			continue
		}
		kidToDevice[device.KID] = device
	}

	// The client now needs to verify two things about these blobs its
	// received: 1) Each is validly signed. 2) The signing key belongs to one
	// of the current user's devices.
	metadata = map[keybase1.DeviceID]keybase1.DeviceEkMetadata{}
	for _, sig := range parsedResponse.Sigs {
		// Verify the sig.
		signerKey, payload, _, err := kbcrypto.NaclVerifyAndExtract(sig)
		if err != nil {
			return nil, err
		}

		// Find the device that matches the signing key. This checks
		// authenticity.
		matchingDevice, ok := kidToDevice[signerKey.GetKID()]
		if !ok {
			return nil, fmt.Errorf("deviceEK returned for unknown device KID %s", signerKey.GetKID())
		}

		// Decode the signed JSON.
		var verifiedStatement keybase1.DeviceEkStatement
		err = json.Unmarshal(payload, &verifiedStatement)
		if err != nil {
			return nil, err
		}

		metadata[matchingDevice.DeviceID] = verifiedStatement.CurrentDeviceEkMetadata
	}

	return metadata, nil
}

// allActiveDeviceEKMetadata fetches the latest deviceEK for each of your
// devices, filtering out the ones that are stale.
func allActiveDeviceEKMetadata(mctx libkb.MetaContext, merkleRoot libkb.MerkleRoot) (metadata map[keybase1.DeviceID]keybase1.DeviceEkMetadata, err error) {
	defer mctx.TraceTimed("allActiveDeviceEKMetadata", func() error { return err })()

	maybeStale, err := allDeviceEKMetadataMaybeStale(mctx, merkleRoot)
	if err != nil {
		return nil, err
	}

	active := map[keybase1.DeviceID]keybase1.DeviceEkMetadata{}
	for deviceID, metadata := range maybeStale {
		// Check whether the key is stale. This isn't considered an error,
		// since the server doesn't do this check for us. We log these cases
		// and skip them.
		if ctimeIsStale(metadata.Ctime.Time(), merkleRoot) {
			mctx.Debug("skipping stale deviceEK %s for device KID %s", metadata.Kid, deviceID)
			continue
		}
		active[deviceID] = metadata
	}

	return active, nil
}
<commit_msg>force reload upak if we are missing an active device (#20166)<commit_after>package ephemeral

import (
	"encoding/json"
	"fmt"

	"github.com/keybase/client/go/kbcrypto"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

type DeviceEKSeed keybase1.Bytes32

func newDeviceEphemeralSeed() (seed DeviceEKSeed, err error) {
	randomSeed, err := makeNewRandomSeed()
	if err != nil {
		return seed, err
	}
	return DeviceEKSeed(randomSeed), nil
}

func (s *DeviceEKSeed) DeriveDHKey() *libkb.NaclDHKeyPair {
	return deriveDHKey(keybase1.Bytes32(*s), libkb.DeriveReasonDeviceEKEncryption)
}

func postNewDeviceEK(mctx libkb.MetaContext, sig string) (err error) {
	defer mctx.TraceTimed("postNewDeviceEK", func() error { return err })()

	apiArg := libkb.APIArg{
		Endpoint:    "user/device_ek",
		SessionType: libkb.APISessionTypeREQUIRED,
		Args: libkb.HTTPArgs{
			"sig":       libkb.S{Val: sig},
			"device_id": libkb.S{Val: string(mctx.ActiveDevice().DeviceID())},
		},
	}
	_, err = mctx.G().GetAPI().Post(mctx, apiArg)
	return err
}

func serverMaxDeviceEK(mctx libkb.MetaContext, merkleRoot libkb.MerkleRoot) (maxGeneration keybase1.EkGeneration, err error) {
	defer mctx.TraceTimed("serverMaxDeviceEK", func() error { return err })()

	deviceEKs, err := allDeviceEKMetadataMaybeStale(mctx, merkleRoot)
	if err != nil {
		return maxGeneration, err
	}
	deviceID := mctx.ActiveDevice().DeviceID()
	metadata, ok := deviceEKs[deviceID]
	if ok {
		return metadata.Generation, nil
	}
	// We may not have an EK yet, let's try with this and fail if the server
	// rejects.
	mctx.Debug("No deviceEK found on the server")
	return 0, nil
}

func publishNewDeviceEK(mctx libkb.MetaContext, merkleRoot libkb.MerkleRoot) (metadata keybase1.DeviceEkMetadata, err error) {
	defer mctx.TraceTimed("publishNewDeviceEK", func() error { return err })()

	seed, err := newDeviceEphemeralSeed()
	if err != nil {
		return metadata, err
	}

	storage := mctx.G().GetDeviceEKStorage()
	generation, err := storage.MaxGeneration(mctx, true)
	if err != nil || generation < 0 {
		// Let's try to get the max from the server
		generation, err = serverMaxDeviceEK(mctx, merkleRoot)
		if err != nil {
			return metadata, err
		}
	}
	// This is our first generation
	if generation < 0 {
		generation = 0
	}
	generation++

	metadata, err = signAndPostDeviceEK(mctx, generation, seed, merkleRoot)
	if err != nil {
		mctx.Debug("Error posting deviceEK, retrying with server maxGeneration")
		// Let's retry posting with the server given max
		generation, err = serverMaxDeviceEK(mctx, merkleRoot)
		if err != nil {
			return metadata, err
		}
		generation++
		metadata, err = signAndPostDeviceEK(mctx, generation, seed, merkleRoot)
		if err != nil {
			return metadata, err
		}
	}

	return metadata, err
}

func signAndPostDeviceEK(mctx libkb.MetaContext, generation keybase1.EkGeneration,
	seed DeviceEKSeed, merkleRoot libkb.MerkleRoot) (metadata keybase1.DeviceEkMetadata, err error) {
	defer mctx.TraceTimed("signAndPostDeviceEK", func() error { return err })()

	storage := mctx.G().GetDeviceEKStorage()

	// Sign the statement blob with the device's long term signing key.
	signingKey, err := mctx.ActiveDevice().SigningKey()
	if err != nil {
		return metadata, err
	}

	dhKeypair := seed.DeriveDHKey()
	statement, signedStatement, err := signDeviceEKStatement(generation, dhKeypair, signingKey, merkleRoot)

	metadata = statement.CurrentDeviceEkMetadata
	// Ensure we successfully write the secret to disk before posting to the
	// server since the secret never leaves the device.
	if err = storage.Put(mctx, generation, keybase1.DeviceEk{
		Seed:     keybase1.Bytes32(seed),
		Metadata: metadata,
	}); err != nil {
		return metadata, err
	}

	err = postNewDeviceEK(mctx, signedStatement)
	if err != nil {
		storage.ClearCache()
		serr := NewDeviceEKStorage(mctx).Delete(mctx, generation)
		if serr != nil {
			mctx.Debug("DeviceEK deletion failed %v", err)
		}
	}

	return metadata, err
}

func signDeviceEKStatement(generation keybase1.EkGeneration, dhKeypair *libkb.NaclDHKeyPair, signingKey libkb.GenericKey, merkleRoot libkb.MerkleRoot) (statement keybase1.DeviceEkStatement, signedStatement string, err error) {
	metadata := keybase1.DeviceEkMetadata{
		Kid:        dhKeypair.GetKID(),
		Generation: generation,
		HashMeta:   merkleRoot.HashMeta(),
		// The ctime is derivable from the hash meta, by fetching the hashed
		// root from the server, but including it saves readers a potential
		// extra round trip.
		Ctime: keybase1.TimeFromSeconds(merkleRoot.Ctime()),
	}
	statement = keybase1.DeviceEkStatement{
		CurrentDeviceEkMetadata: metadata,
	}

	statementJSON, err := json.Marshal(statement)
	if err != nil {
		return statement, signedStatement, err
	}

	signedStatement, _, err = signingKey.SignToString(statementJSON)
	return statement, signedStatement, err
}

type deviceEKStatementResponse struct {
	Sigs []string `json:"sigs"`
}

func allDeviceEKMetadataMaybeStale(mctx libkb.MetaContext, merkleRoot libkb.MerkleRoot) (metadata map[keybase1.DeviceID]keybase1.DeviceEkMetadata, err error) {
	defer mctx.TraceTimed("allDeviceEKMetadataMaybeStale", func() error { return err })()

	apiArg := libkb.APIArg{
		Endpoint:    "user/device_eks",
		SessionType: libkb.APISessionTypeREQUIRED,
		Args:        libkb.HTTPArgs{},
	}
	res, err := mctx.G().GetAPI().Get(mctx, apiArg)
	if err != nil {
		return nil, err
	}

	parsedResponse := deviceEKStatementResponse{}
	err = res.Body.UnmarshalAgain(&parsedResponse)
	if err != nil {
		return nil, err
	}

	// Make a map of the user's own active devices, by KID. We'll use this to
	// match deviceEK sigs with the device that issued them below. (Checking
	// the signing key is intentionally the only way to do this, so that we're
	// forced to check authenticity.)
	getDeviceKIDs := func(force bool) (map[keybase1.KID]keybase1.PublicKey, error) {
		arg := libkb.NewLoadUserArgWithMetaContext(
			mctx).WithUID(mctx.ActiveDevice().UID())
		if force {
			arg = arg.WithForceReload()
		}
		self, _, err := mctx.G().GetUPAKLoader().Load(arg)
		if err != nil {
			return nil, err
		}
		kidToDevice := map[keybase1.KID]keybase1.PublicKey{}
		for _, device := range self.Base.DeviceKeys {
			if device.IsRevoked {
				continue
			}
			kidToDevice[device.KID] = device
		}
		return kidToDevice, nil
	}

	kidToDevice, err := getDeviceKIDs(false)
	if err != nil {
		return nil, err
	}

	if len(kidToDevice) != len(parsedResponse.Sigs) {
		mctx.Debug("mismatch of active devices in UPAK to device EK sigs (%d (upak) != %d (ek sigs), attempting force reload.", len(kidToDevice), len(parsedResponse.Sigs))
		// force a reload in case we are missing a device
		kidToDevice, err = getDeviceKIDs(true)
		if err != nil {
			return nil, err
		}
		mctx.Debug("%d active devices found after force reload vs %d sigs.", len(kidToDevice), len(parsedResponse.Sigs))
	}

	// The client now needs to verify two things about these blobs its
	// received: 1) Each is validly signed. 2) The signing key belongs to one
	// of the current user's devices.
	metadata = map[keybase1.DeviceID]keybase1.DeviceEkMetadata{}
	for _, sig := range parsedResponse.Sigs {
		// Verify the sig.
		signerKey, payload, _, err := kbcrypto.NaclVerifyAndExtract(sig)
		if err != nil {
			return nil, err
		}

		// Find the device that matches the signing key. This checks
		// authenticity.
		matchingDevice, ok := kidToDevice[signerKey.GetKID()]
		if !ok {
			return nil, fmt.Errorf("deviceEK returned for unknown device KID %s", signerKey.GetKID())
		}

		// Decode the signed JSON.
		var verifiedStatement keybase1.DeviceEkStatement
		err = json.Unmarshal(payload, &verifiedStatement)
		if err != nil {
			return nil, err
		}

		metadata[matchingDevice.DeviceID] = verifiedStatement.CurrentDeviceEkMetadata
	}

	return metadata, nil
}

// allActiveDeviceEKMetadata fetches the latest deviceEK for each of your
// devices, filtering out the ones that are stale.
func allActiveDeviceEKMetadata(mctx libkb.MetaContext, merkleRoot libkb.MerkleRoot) (metadata map[keybase1.DeviceID]keybase1.DeviceEkMetadata, err error) {
	defer mctx.TraceTimed("allActiveDeviceEKMetadata", func() error { return err })()

	maybeStale, err := allDeviceEKMetadataMaybeStale(mctx, merkleRoot)
	if err != nil {
		return nil, err
	}

	active := map[keybase1.DeviceID]keybase1.DeviceEkMetadata{}
	for deviceID, metadata := range maybeStale {
		// Check whether the key is stale. This isn't considered an error,
		// since the server doesn't do this check for us. We log these cases
		// and skip them.
		if ctimeIsStale(metadata.Ctime.Time(), merkleRoot) {
			mctx.Debug("skipping stale deviceEK %s for device KID %s", metadata.Kid, deviceID)
			continue
		}
		active[deviceID] = metadata
	}

	return active, nil
}
<|endoftext|>


<commit_before>package piglatin

import (
	"fmt"
	"strings"
)

var consonants = []string{"p", "k"}

func Sentence(sentence string) string {
	if startsWithVowel(sentence) ||
		strings.HasPrefix(sentence, "xr") ||
		strings.HasPrefix(sentence, "yt") {
		return sentence + "ay"
	}
	if startsWithConsonant(sentence) {
		return handleConsonant(sentence)
	}

	return sentence
}

func startsWithVowel(sentence string) bool {
	return strings.HasPrefix(sentence, "a") ||
		strings.HasPrefix(sentence, "e") ||
		strings.HasPrefix(sentence, "i") ||
		strings.HasPrefix(sentence, "o") ||
		strings.HasPrefix(sentence, "u")
}

func startsWithConsonant(sentence string) bool {
	for _, consonant := range consonants {
		if strings.HasPrefix(sentence, consonant) {
			return true
		}
	}
	return false
}

func handleConsonant(sentence string) string {
	for _, consonant := range consonants {
		if strings.HasPrefix(sentence, consonant) {
			return strings.TrimPrefix(sentence, consonant) + consonant + "ay"
		}
	}
	panic(fmt.Sprintf("could not find consonant prefix for sentence %v", sentence))
}
<commit_msg>Handle all consonant sounds<commit_after>package piglatin

import (
	"fmt"
	"strings"
)

var vowels = []string{"a", "e", "i", "o", "u"}
var consonantSounds = []string{"squ", "thr", "th", "sch", "qu", "ch", "rh", "p", "k", "x", "q", "y", "m"}

func Sentence(sentence string) string {
	if startsWithVowel(sentence) ||
		strings.HasPrefix(sentence, "xr") ||
		strings.HasPrefix(sentence, "yt") {
		return handleVowel(sentence)
	}
	if startsWithConsonantSound(sentence) {
		return handleConsonantSound(sentence)
	}

	return sentence
}

func startsWithVowel(sentence string) bool {
	for _, vowel := range vowels {
		if strings.HasPrefix(sentence, vowel) {
			return true
		}
	}
	return false
}

func startsWithConsonantSound(sentence string) bool {
	for _, cs := range consonantSounds {
		if strings.HasPrefix(sentence, cs) {
			return true
		}
	}
	return false
}

func handleConsonantSound(sentence string) string {
	for _, cs := range consonantSounds {
		if strings.HasPrefix(sentence, cs) {
			return strings.TrimPrefix(sentence, cs) + cs + "ay"
		}
	}
	panic(fmt.Sprintf("could not find consonant sound prefix for sentence %v", sentence))
}

func handleVowel(sentence string) string {
	return sentence + "ay"
}
<|endoftext|>


<commit_before>package virt

import (
	"errors"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"labix.org/v2/mgo/bson"
)

func (vm *VM) Start() error {
	if out, err := exec.Command("/usr/bin/lxc-start", "--name", vm.String(), "--daemon").CombinedOutput(); err != nil {
		return commandError("lxc-start failed.", err, out)
	}
	return vm.WaitForState("RUNNING", time.Second)
}

func (vm *VM) Stop() error {
	if out, err := exec.Command("/usr/bin/lxc-stop", "--name", vm.String()).CombinedOutput(); err != nil {
		return commandError("lxc-stop failed.", err, out)
	}
	return vm.WaitForState("STOPPED", time.Second)
}

func (vm *VM) Shutdown() error {
	if out, err := exec.Command("/usr/bin/lxc-shutdown", "--name", vm.String()).CombinedOutput(); err != nil {
		if vm.GetState() != "STOPPED" {
			return commandError("lxc-shutdown failed.", err, out)
		}
	}
	vm.WaitForState("STOPPED", 5*time.Second) // may time out, then vm is force stopped
	return vm.Stop()
}

func (vm *VM) AttachCommand(uid int, tty string, command ...string) *exec.Cmd {
	args := []string{"--name", vm.String()}
	if tty != "" {
		args = append(args, "--tty", tty)
	}
	args = append(args, "--", "/usr/bin/sudo", "-i", "-u", "#"+strconv.Itoa(uid), "--")
	args = append(args, command...)
	cmd := exec.Command("/usr/bin/lxc-attach", args...)
	cmd.Env = []string{"TERM=xterm-256color"}
	return cmd
}

func GetVMState(vmId bson.ObjectId) string {
	out, err := exec.Command("/usr/bin/lxc-info", "--name", VMName(vmId), "--state").CombinedOutput()
	if err != nil {
		lastError = commandError("lxc-info failed ", err, out)
		return "UNKNOWN"
	}
	return strings.TrimSpace(string(out)[6:])
}

func (vm *VM) GetState() string {
	return GetVMState(vm.Id)
}

func (vm *VM) WaitForState(state string, timeout time.Duration) error {
	tryUntil := time.Now().Add(timeout)
	for vm.GetState() != state {
		if time.Now().After(tryUntil) {
			return errors.New("Timeout while waiting for VM state.")
		}
		time.Sleep(time.Millisecond * 500)
	}
	return nil
}

func (vm *VM) SendMessageToVMUsers(message string) error {
	cmd := exec.Command("/usr/bin/lxc-attach", "--name", vm.String(), "--", "/usr/bin/wall", "--nobanner")
	cmd.Stdin = strings.NewReader(message)
	if out, err := cmd.CombinedOutput(); err != nil {
		return commandError("wall failed.", err, out)
	}
	return nil
}

func (vm *VM) WaitForNetwork() error {
	timeout := time.Second * 5
	isNetworkUp := func() bool {
		// neglect error because it's not important and we also going to timeout
		out, _ := exec.Command("/usr/bin/lxc-attach", "--name", vm.String(),
			"--", "/bin/cat", "/sys/class/net/eth0/operstate").CombinedOutput()

		if strings.TrimSpace(string(out)) == "up" {
			if out, err := exec.Command("/usr/bin/lxc-attach", "--name", vm.String(),
				"--", "/usr/bin/stat", "/usr/bin/screen").CombinedOutput(); err != nil {
				return false
			} else {
				return true
			}
			return false
		}
	}
	tryUntil := time.Now().Add(timeout)
	for {
		if up := isNetworkUp(); up {
			return nil
		}

		if time.Now().After(tryUntil) {
			return errors.New("Timeout while waiting for VM Network state.")
		}

		time.Sleep(time.Millisecond * 100)
	}
}
<commit_msg>We only really need to check this aever 500ms too<commit_after>package virt

import (
	"errors"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"labix.org/v2/mgo/bson"
)

func (vm *VM) Start() error {
	if out, err := exec.Command("/usr/bin/lxc-start", "--name", vm.String(), "--daemon").CombinedOutput(); err != nil {
		return commandError("lxc-start failed.", err, out)
	}
	return vm.WaitForState("RUNNING", time.Second)
}

func (vm *VM) Stop() error {
	if out, err := exec.Command("/usr/bin/lxc-stop", "--name", vm.String()).CombinedOutput(); err != nil {
		return commandError("lxc-stop failed.", err, out)
	}
	return vm.WaitForState("STOPPED", time.Second)
}

func (vm *VM) Shutdown() error {
	if out, err := exec.Command("/usr/bin/lxc-shutdown", "--name", vm.String()).CombinedOutput(); err != nil {
		if vm.GetState() != "STOPPED" {
			return commandError("lxc-shutdown failed.", err, out)
		}
	}
	vm.WaitForState("STOPPED", 5*time.Second) // may time out, then vm is force stopped
	return vm.Stop()
}

func (vm *VM) AttachCommand(uid int, tty string, command ...string) *exec.Cmd {
	args := []string{"--name", vm.String()}
	if tty != "" {
		args = append(args, "--tty", tty)
	}
	args = append(args, "--", "/usr/bin/sudo", "-i", "-u", "#"+strconv.Itoa(uid), "--")
	args = append(args, command...)
	cmd := exec.Command("/usr/bin/lxc-attach", args...)
	cmd.Env = []string{"TERM=xterm-256color"}
	return cmd
}

func GetVMState(vmId bson.ObjectId) string {
	out, err := exec.Command("/usr/bin/lxc-info", "--name", VMName(vmId), "--state").CombinedOutput()
	if err != nil {
		lastError = commandError("lxc-info failed ", err, out)
		return "UNKNOWN"
	}
	return strings.TrimSpace(string(out)[6:])
}

func (vm *VM) GetState() string {
	return GetVMState(vm.Id)
}

func (vm *VM) WaitForState(state string, timeout time.Duration) error {
	tryUntil := time.Now().Add(timeout)
	for vm.GetState() != state {
		if time.Now().After(tryUntil) {
			return errors.New("Timeout while waiting for VM state.")
		}
		time.Sleep(time.Millisecond * 500)
	}
	return nil
}

func (vm *VM) SendMessageToVMUsers(message string) error {
	cmd := exec.Command("/usr/bin/lxc-attach", "--name", vm.String(), "--", "/usr/bin/wall", "--nobanner")
	cmd.Stdin = strings.NewReader(message)
	if out, err := cmd.CombinedOutput(); err != nil {
		return commandError("wall failed.", err, out)
	}
	return nil
}

func (vm *VM) WaitForNetwork() error {
	timeout := time.Second * 5
	isNetworkUp := func() bool {
		// neglect error because it's not important and we also going to timeout
		out, _ := exec.Command("/usr/bin/lxc-attach", "--name", vm.String(),
			"--", "/bin/cat", "/sys/class/net/eth0/operstate").CombinedOutput()

		if strings.TrimSpace(string(out)) == "up" {
			if out, err := exec.Command("/usr/bin/lxc-attach", "--name", vm.String(),
				"--", "/usr/bin/stat", "/usr/bin/screen").CombinedOutput(); err != nil {
				return false
			} else {
				return true
			}
			return false
		}
	}
	tryUntil := time.Now().Add(timeout)
	for {
		if up := isNetworkUp(); up {
			return nil
		}

		if time.Now().After(tryUntil) {
			return errors.New("Timeout while waiting for VM Network state.")
		}

		time.Sleep(time.Millisecond * 500)
	}
}
<|endoftext|>


<commit_before>package statsd

import (
	"encoding/json"
	"expvar"
	"flag"
	"fmt"
	"strings"

	"github.com/DataDog/datadog-go/statsd"
	"vitess.io/vitess/go/stats"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/servenv"
)

var (
	statsdAddress    = flag.String("statsd_address", "", "Address for statsd client")
	statsdSampleRate = flag.Float64("statsd_sample_rate", 1.0, "")
)

// StatsBackend implements PullBackend using statsd
type StatsBackend struct {
	namespace    string
	statsdClient *statsd.Client
	sampleRate   float64
}

var (
	sb StatsBackend
)

// makeLabel builds a tag list with a single label + value.
func makeLabel(labelName string, labelVal string) []string {
	return []string{fmt.Sprintf("%s:%s", labelName, labelVal)}
}

// makeLabels takes the vitess stat representation of label values ("."-separated list) and breaks it
// apart into a map of label name -> label value.
func makeLabels(labelNames []string, labelValsCombined string) []string {
	tags := make([]string, len(labelNames))
	labelVals := strings.Split(labelValsCombined, ".")
	for i, v := range labelVals {
		tags[i] = fmt.Sprintf("%s:%s", labelNames[i], v)
	}
	return tags
}

// Init initializes the statsd with the given namespace.
func Init(namespace string) {
	servenv.OnRun(func() {
		if *statsdAddress == "" {
			return
		}
		statsdC, err := statsd.NewBuffered(*statsdAddress, 100)
		if err != nil {
			log.Errorf("Failed to create statsd client %v", err)
			return
		}
		statsdC.Namespace = namespace + "."
		sb.namespace = namespace
		sb.statsdClient = statsdC
		sb.sampleRate = *statsdSampleRate
		stats.RegisterPushBackend("statsd", sb)
	})
}

func (sb StatsBackend) addExpVar(kv expvar.KeyValue) {
	k := kv.Key
	switch v := kv.Value.(type) {
	case *stats.String:
		if err := sb.statsdClient.Set(k, v.Get(), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add String %v for key %v", v, k)
		}
	case *stats.Counter:
		if err := sb.statsdClient.Count(k, v.Get(), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add Counter %v for key %v", v, k)
		}
	case *stats.Gauge:
		if err := sb.statsdClient.Gauge(k, float64(v.Get()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add Gauge %v for key %v", v, k)
		}
	case *stats.GaugeFunc:
		if err := sb.statsdClient.Gauge(k, float64(v.F()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add GaugeFunc %v for key %v", v, k)
		}
	case *stats.CounterFunc:
		if err := sb.statsdClient.Gauge(k, float64(v.F()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add CounterFunc %v for key %v", v, k)
		}
	case *stats.CounterDuration:
		if err := sb.statsdClient.TimeInMilliseconds(k, float64(v.Get().Milliseconds()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add CounterDuration %v for key %v", v, k)
		}
	case *stats.CounterDurationFunc:
		if err := sb.statsdClient.TimeInMilliseconds(k, float64(v.F().Milliseconds()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add CounterDuration %v for key %v", v, k)
		}
	case *stats.GaugeDuration:
		if err := sb.statsdClient.TimeInMilliseconds(k, float64(v.Get().Milliseconds()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add GaugeDuration %v for key %v", v, k)
		}
	case *stats.GaugeDurationFunc:
		if err := sb.statsdClient.TimeInMilliseconds(k, float64(v.F().Milliseconds()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add GaugeDuration %v for key %v", v, k)
		}
	case *stats.CountersWithSingleLabel:
		for labelVal, val := range v.Counts() {
			if err := sb.statsdClient.Count(k, val, makeLabel(v.Label(), labelVal), sb.sampleRate); err != nil {
				log.Errorf("Failed to add CountersWithSingleLabel %v for key %v", v, k)
			}
		}
	case *stats.CountersWithMultiLabels:
		for labelVals, val := range v.Counts() {
			if err := sb.statsdClient.Count(k, val, makeLabels(v.Labels(), labelVals), sb.sampleRate); err != nil {
				log.Errorf("Failed to add CountersFuncWithMultiLabels %v for key %v", v, k)
			}
		}
	case *stats.CountersFuncWithMultiLabels:
		for labelVals, val := range v.Counts() {
			if err := sb.statsdClient.Count(k, val, makeLabels(v.Labels(), labelVals), sb.sampleRate); err != nil {
				log.Errorf("Failed to add CountersFuncWithMultiLabels %v for key %v", v, k)
			}
		}
	case *stats.GaugesWithMultiLabels:
		for labelVals, val := range v.Counts() {
			if err := sb.statsdClient.Gauge(k, float64(val), makeLabels(v.Labels(), labelVals), sb.sampleRate); err != nil {
				log.Errorf("Failed to add GaugesWithMultiLabels %v for key %v", v, k)
			}
		}
	case *stats.GaugesFuncWithMultiLabels:
		for labelVals, val := range v.Counts() {
			if err := sb.statsdClient.Gauge(k, float64(val), makeLabels(v.Labels(), labelVals), sb.sampleRate); err != nil {
				log.Errorf("Failed to add GaugesFuncWithMultiLabels %v for key %v", v, k)
			}
		}
	case *stats.GaugesWithSingleLabel:
		for labelVal, val := range v.Counts() {
			if err := sb.statsdClient.Gauge(k, float64(val), makeLabel(v.Label(), labelVal), sb.sampleRate); err != nil {
				log.Errorf("Failed to add GaugesWithSingleLabel %v for key %v", v, k)
			}
		}
	case *stats.MultiTimings:
		labels := v.Labels()
		buffers := v.Buffers()
		for labelValsCombined, buffer := range buffers {
			tags := makeLabels(labels, labelValsCombined)
			for _, elapsedNs := range buffer.Values() {
				if err := sb.statsdClient.TimeInMilliseconds(k, float64(elapsedNs)/1000.0/1000.0, tags, sb.sampleRate); err != nil {
					log.Errorf("Failed to add TimeInMilliseconds %v for key %v", buffer.Values(), k)
				}
			}
		}
	case *stats.Timings:
		label := v.Label()
		buffers := v.Buffers()
		for labelValsCombined, buffer := range buffers {
			tags := makeLabel(label, labelValsCombined)
			for _, elapsedNs := range buffer.Values() {
				if err := sb.statsdClient.TimeInMilliseconds(k, float64(elapsedNs)/1000.0/1000.0, tags, sb.sampleRate); err != nil {
					log.Errorf("Failed to add TimeInMilliseconds %v for key %v", buffer.Values(), k)
				}
			}
		}
	case *stats.Histogram:
		labels := v.Labels()
		buckets := v.Buckets()
		for i := range labels {
			name := fmt.Sprintf("%s.%s", k, labels[i])
			if err := sb.statsdClient.Count(name, buckets[i], []string{}, sb.sampleRate); err != nil {
				log.Errorf("Failed to add Histogram %v for key %v", buckets[i], name)
			}
		}
	case expvar.Func:
		// Export memstats as gauge so that we don't need to call extra ReadMemStats
		if k == "memstats" {
			var obj map[string]interface{}
			if err := json.Unmarshal([]byte(v.String()), &obj); err != nil {
				return
			}
			for k, v := range obj {
				if k == "NumGC" {
					if err := sb.statsdClient.Gauge("NumGC", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export NumGC %v", v)
					}
				} else if k == "Frees" {
					if err := sb.statsdClient.Gauge("Frees", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export Frees %v", v)
					}
				} else if k == "GCCPUFraction" {
					if err := sb.statsdClient.Gauge("GCCPUFraction", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export GCCPUFraction %v", v)
					}
				} else if k == "PauseTotalNs" {
					if err := sb.statsdClient.Gauge("PauseTotalNs", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export PauseTotalNs %v", v)
					}
				} else if k == "HeapAlloc" {
					if err := sb.statsdClient.Gauge("HeapAlloc", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export HeapAlloc %v", v)
					}
				}
			}
		} else {
			log.Warningf("Silently ignore metrics with key %v [%T]", k, kv.Value)
		}
	default:
		log.Warningf("Silently ignore metrics with key %v [%T]", k, kv.Value)
	}
}

// PushAll flush out the pending metrics
func (sb StatsBackend) PushAll() error {
	expvar.Do(func(kv expvar.KeyValue) {
		sb.addExpVar(kv)
	})
	if err := sb.statsdClient.Flush(); err != nil {
		return err
	}
	return nil
}
<commit_msg>ignore metrics that does not make sense to statsd<commit_after>package statsd

import (
	"encoding/json"
	"expvar"
	"flag"
	"fmt"
	"strings"

	"github.com/DataDog/datadog-go/statsd"
	"vitess.io/vitess/go/stats"
	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/servenv"
)

var (
	statsdAddress    = flag.String("statsd_address", "", "Address for statsd client")
	statsdSampleRate = flag.Float64("statsd_sample_rate", 1.0, "")
)

// StatsBackend implements PullBackend using statsd
type StatsBackend struct {
	namespace    string
	statsdClient *statsd.Client
	sampleRate   float64
}

var (
	sb StatsBackend
)

// makeLabel builds a tag list with a single label + value.
func makeLabel(labelName string, labelVal string) []string {
	return []string{fmt.Sprintf("%s:%s", labelName, labelVal)}
}

// makeLabels takes the vitess stat representation of label values ("."-separated list) and breaks it
// apart into a map of label name -> label value.
func makeLabels(labelNames []string, labelValsCombined string) []string {
	tags := make([]string, len(labelNames))
	labelVals := strings.Split(labelValsCombined, ".")
	for i, v := range labelVals {
		tags[i] = fmt.Sprintf("%s:%s", labelNames[i], v)
	}
	return tags
}

// Init initializes the statsd with the given namespace.
func Init(namespace string) {
	servenv.OnRun(func() {
		if *statsdAddress == "" {
			return
		}
		statsdC, err := statsd.NewBuffered(*statsdAddress, 100)
		if err != nil {
			log.Errorf("Failed to create statsd client %v", err)
			return
		}
		statsdC.Namespace = namespace + "."
		sb.namespace = namespace
		sb.statsdClient = statsdC
		sb.sampleRate = *statsdSampleRate
		stats.RegisterPushBackend("statsd", sb)
	})
}

func (sb StatsBackend) addExpVar(kv expvar.KeyValue) {
	k := kv.Key
	switch v := kv.Value.(type) {
	case *stats.String:
		if err := sb.statsdClient.Set(k, v.Get(), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add String %v for key %v", v, k)
		}
	case *stats.Counter:
		if err := sb.statsdClient.Count(k, v.Get(), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add Counter %v for key %v", v, k)
		}
	case *stats.Gauge:
		if err := sb.statsdClient.Gauge(k, float64(v.Get()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add Gauge %v for key %v", v, k)
		}
	case *stats.GaugeFunc:
		if err := sb.statsdClient.Gauge(k, float64(v.F()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add GaugeFunc %v for key %v", v, k)
		}
	case *stats.CounterFunc:
		if err := sb.statsdClient.Gauge(k, float64(v.F()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add CounterFunc %v for key %v", v, k)
		}
	case *stats.CounterDuration:
		if err := sb.statsdClient.TimeInMilliseconds(k, float64(v.Get().Milliseconds()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add CounterDuration %v for key %v", v, k)
		}
	case *stats.CounterDurationFunc:
		if err := sb.statsdClient.TimeInMilliseconds(k, float64(v.F().Milliseconds()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add CounterDuration %v for key %v", v, k)
		}
	case *stats.GaugeDuration:
		if err := sb.statsdClient.TimeInMilliseconds(k, float64(v.Get().Milliseconds()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add GaugeDuration %v for key %v", v, k)
		}
	case *stats.GaugeDurationFunc:
		if err := sb.statsdClient.TimeInMilliseconds(k, float64(v.F().Milliseconds()), nil, sb.sampleRate); err != nil {
			log.Errorf("Failed to add GaugeDuration %v for key %v", v, k)
		}
	case *stats.CountersWithSingleLabel:
		for labelVal, val := range v.Counts() {
			if err := sb.statsdClient.Count(k, val, makeLabel(v.Label(), labelVal), sb.sampleRate); err != nil {
				log.Errorf("Failed to add CountersWithSingleLabel %v for key %v", v, k)
			}
		}
	case *stats.CountersWithMultiLabels:
		for labelVals, val := range v.Counts() {
			if err := sb.statsdClient.Count(k, val, makeLabels(v.Labels(), labelVals), sb.sampleRate); err != nil {
				log.Errorf("Failed to add CountersFuncWithMultiLabels %v for key %v", v, k)
			}
		}
	case *stats.CountersFuncWithMultiLabels:
		for labelVals, val := range v.Counts() {
			if err := sb.statsdClient.Count(k, val, makeLabels(v.Labels(), labelVals), sb.sampleRate); err != nil {
				log.Errorf("Failed to add CountersFuncWithMultiLabels %v for key %v", v, k)
			}
		}
	case *stats.GaugesWithMultiLabels:
		for labelVals, val := range v.Counts() {
			if err := sb.statsdClient.Gauge(k, float64(val), makeLabels(v.Labels(), labelVals), sb.sampleRate); err != nil {
				log.Errorf("Failed to add GaugesWithMultiLabels %v for key %v", v, k)
			}
		}
	case *stats.GaugesFuncWithMultiLabels:
		for labelVals, val := range v.Counts() {
			if err := sb.statsdClient.Gauge(k, float64(val), makeLabels(v.Labels(), labelVals), sb.sampleRate); err != nil {
				log.Errorf("Failed to add GaugesFuncWithMultiLabels %v for key %v", v, k)
			}
		}
	case *stats.GaugesWithSingleLabel:
		for labelVal, val := range v.Counts() {
			if err := sb.statsdClient.Gauge(k, float64(val), makeLabel(v.Label(), labelVal), sb.sampleRate); err != nil {
				log.Errorf("Failed to add GaugesWithSingleLabel %v for key %v", v, k)
			}
		}
	case *stats.MultiTimings:
		labels := v.Labels()
		buffers := v.Buffers()
		for labelValsCombined, buffer := range buffers {
			tags := makeLabels(labels, labelValsCombined)
			for _, elapsedNs := range buffer.Values() {
				if err := sb.statsdClient.TimeInMilliseconds(k, float64(elapsedNs)/1000.0/1000.0, tags, sb.sampleRate); err != nil {
					log.Errorf("Failed to add TimeInMilliseconds %v for key %v", buffer.Values(), k)
				}
			}
		}
	case *stats.Timings:
		label := v.Label()
		buffers := v.Buffers()
		for labelValsCombined, buffer := range buffers {
			tags := makeLabel(label, labelValsCombined)
			for _, elapsedNs := range buffer.Values() {
				if err := sb.statsdClient.TimeInMilliseconds(k, float64(elapsedNs)/1000.0/1000.0, tags, sb.sampleRate); err != nil {
					log.Errorf("Failed to add TimeInMilliseconds %v for key %v", buffer.Values(), k)
				}
			}
		}
	case *stats.Histogram:
		labels := v.Labels()
		buckets := v.Buckets()
		for i := range labels {
			name := fmt.Sprintf("%s.%s", k, labels[i])
			if err := sb.statsdClient.Count(name, buckets[i], []string{}, sb.sampleRate); err != nil {
				log.Errorf("Failed to add Histogram %v for key %v", buckets[i], name)
			}
		}
	case expvar.Func:
		// Export memstats as gauge so that we don't need to call extra ReadMemStats
		if k == "memstats" {
			var obj map[string]interface{}
			if err := json.Unmarshal([]byte(v.String()), &obj); err != nil {
				return
			}
			for k, v := range obj {
				if k == "NumGC" {
					if err := sb.statsdClient.Gauge("NumGC", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export NumGC %v", v)
					}
				} else if k == "Frees" {
					if err := sb.statsdClient.Gauge("Frees", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export Frees %v", v)
					}
				} else if k == "GCCPUFraction" {
					if err := sb.statsdClient.Gauge("GCCPUFraction", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export GCCPUFraction %v", v)
					}
				} else if k == "PauseTotalNs" {
					if err := sb.statsdClient.Gauge("PauseTotalNs", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export PauseTotalNs %v", v)
					}
				} else if k == "HeapAlloc" {
					if err := sb.statsdClient.Gauge("HeapAlloc", v.(float64), []string{}, sb.sampleRate); err != nil {
						log.Errorf("Failed to export HeapAlloc %v", v)
					}
				}
			}
		}
	case *stats.StringMapFunc, *stats.Rates, *stats.RatesFunc:
		// Silently ignore metrics that does not make sense to be exported to statsd
	default:
		log.Warningf("Silently ignore metrics with key %v [%T]", k, kv.Value)
	}
}

// PushAll flush out the pending metrics
func (sb StatsBackend) PushAll() error {
	expvar.Do(func(kv expvar.KeyValue) {
		sb.addExpVar(kv)
	})
	if err := sb.statsdClient.Flush(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"

	"github.com/ginuerzh/gost/gost"
)

var (
	laddr, faddr string
	quiet        bool
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	flag.StringVar(&laddr, "L", ":18080", "listen address")
	flag.StringVar(&faddr, "F", ":12222", "forward address")
	flag.BoolVar(&quiet, "q", false, "quiet mode")
	flag.BoolVar(&gost.Debug, "d", false, "debug mode")
	flag.Parse()

	if quiet {
		gost.SetLogger(&gost.NopLogger{})
	}
}

func main() {
	chain := gost.NewChain(
		gost.Node{
			Addr: faddr,
			Client: gost.NewClient(
				gost.HTTPConnector(nil),
				gost.SSHTunnelTransporter(),
			),
		},
	)

	s := &gost.Server{}
	s.Handle(gost.HTTPHandler(gost.ChainHandlerOption(chain)))
	ln, err := gost.TCPListener(laddr)
	if err != nil {
		log.Fatal(err)
	}
	log.Fatal(s.Serve(ln))
}
<commit_msg>change sshc to socks5<commit_after>package main

import (
	"flag"
	"log"

	"github.com/ginuerzh/gost/gost"
)

var (
	laddr, faddr string
	quiet        bool
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	flag.StringVar(&laddr, "L", ":18080", "listen address")
	flag.StringVar(&faddr, "F", ":12222", "forward address")
	flag.BoolVar(&quiet, "q", false, "quiet mode")
	flag.BoolVar(&gost.Debug, "d", false, "debug mode")
	flag.Parse()

	if quiet {
		gost.SetLogger(&gost.NopLogger{})
	}
}

func main() {
	chain := gost.NewChain(
		gost.Node{
			Addr: faddr,
			Client: gost.NewClient(
				gost.HTTPConnector(nil),
				gost.SSHTunnelTransporter(),
			),
		},
	)

	s := &gost.Server{}
	s.Handle(gost.SOCKS5Handler(gost.ChainHandlerOption(chain)))
	ln, err := gost.TCPListener(laddr)
	if err != nil {
		log.Fatal(err)
	}
	log.Fatal(s.Serve(ln))
}
<|endoftext|>


<commit_before>package slackRealtime

import (
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"

	"github.com/FogCreek/victor/pkg/chat"
	"github.com/abourget/slack"
)

// The Slack Websocket's registered adapter name for the victor framework.
const AdapterName = "slackRealtime"

// Prefix for the user's ID which is used when reading/writing from the bot's store
const userInfoPrefix = AdapterName + "."

const userIDRegexpString = "\\b<?@?(U[[:alnum:]]+)(?:(?:|\\S+)?>?)"

// Match "<@Userid>" and "<@UserID|fullname>"
var userIDRegexp = regexp.MustCompile(userIDRegexpString)

// Match "johndoe", "@johndoe",
// not needed?
// var userIDAndNameRegexp = regexp.MustCompile("\\A@?(\\w+)|" + userIDRegexpString)

// channelGroupInfo is used instead of the slack library's Channel struct since we
// are trying to consider channels and groups to be roughly the same while it
// considers them seperate and provides no way to consolidate them on its own.
//
// This also allows us to throw our information that we don't care about (members, etc.).
type channelGroupInfo struct {
	Name      string
	ID        string
	IsDM      bool
	UserID    string
	IsChannel bool
	// UserID is only stored for IM/DM's so we can then send a user a DM as a
	// response if needed
}

// init registers SlackAdapter to the victor chat framework.
func init() {
	chat.Register(AdapterName, func(r chat.Robot) chat.Adapter {
		config, configSet := r.AdapterConfig()
		if !configSet {
			log.Println("A configuration struct implementing the SlackConfig interface must be set.")
			os.Exit(1)
		}
		sConfig, ok := config.(Config)
		if !ok {
			log.Println("The bot's config must implement the SlackConfig interface.")
			os.Exit(1)
		}
		return &SlackAdapter{
			robot:           r,
			chReceiver:      make(chan slack.SlackEvent),
			token:           sConfig.Token(),
			channelInfo:     make(map[string]channelGroupInfo),
			directMessageID: make(map[string]string),
			userInfo:        make(map[string]slack.User),
		}
	})
}

// Config provides the slack adapter with the necessary
// information to open a websocket connection with the slack Real time API.
type Config interface {
	Token() string
}

// Config implements the SlackRealtimeConfig interface to provide a slack
// adapter with the information it needs to authenticate with slack.
type configImpl struct {
	token string
}

// NewConfig returns a new slack configuration instance using the given token.
func NewConfig(token string) configImpl {
	return configImpl{token: token}
}

// Token returns the slack token.
func (c configImpl) Token() string {
	return c.token
}

// SlackAdapter holds all information needed by the adapter to send/receive messages.
type SlackAdapter struct {
	robot           chat.Robot
	token           string
	instance        *slack.Client
	rtm             *slack.RTM
	chReceiver      chan slack.SlackEvent
	channelInfo     map[string]channelGroupInfo
	directMessageID map[string]string
	userInfo        map[string]slack.User
	domain          string
	botID           string
}

// GetUser will parse the given user ID string and then return the user's
// information as provided by the slack API. This will first try to get the
// user's information from a local cache and then will perform a slack API
// call if the user's information is not cached. Returns nil if the user does
// not exist or if an error occurrs during the slack API call.
func (adapter *SlackAdapter) GetUser(userIDStr string) chat.User {
	if !adapter.IsPotentialUser(userIDStr) {
		log.Printf("%s is not a potential user", userIDStr)
		return nil
	}
	userID := adapter.NormalizeUserID(userIDStr)
	userObj, err := adapter.getUserFromSlack(userID)
	if err != nil {
		log.Println("Error getting user: " + err.Error())
		return nil
	}
	return &chat.BaseUser{
		UserID:    userObj.Id,
		UserName:  userObj.Name,
		UserEmail: userObj.Profile.Email,
		UserIsBot: userObj.IsBot,
	}
}

// IsPotentialUser checks if a given string is potentially referring to a slack
// user. Strings given to this function should be trimmed of leading whitespace
// as it does not account for that (it is meant to be used with the fields
// method on the frameworks calls to handlers which are trimmed).
func (adapter *SlackAdapter) IsPotentialUser(userString string) bool {
	return userIDRegexp.MatchString(userString)
}

// normalizeUserID returns a user's ID without the extra formatting that slack
// might add. This will return "U01234567" for inputs: "U01234567",
// "@U01234567", "<@U01234567>", and "<@U01234567|name>"
func (adapter *SlackAdapter) NormalizeUserID(userID string) string {
	userIDArr := userIDRegexp.FindAllStringSubmatch(userID, 1)
	if len(userIDArr) == 0 {
		return userID
	}
	return userIDArr[0][1]
}

// Run starts the adapter and begins to listen for new messages to send/receive.
// At the moment this will crash the program and print the error messages to a
// log if the connection fails.
func (adapter *SlackAdapter) Run() error {
	adapter.instance = slack.New(adapter.token)
	adapter.instance.SetDebug(false)
	adapter.rtm = adapter.instance.NewRTM()
	go adapter.rtm.ManageConnection()
	go adapter.monitorEvents()
	return nil
}

func (adapter *SlackAdapter) initAdapterInfo(info *slack.Info) {
	// info := adapter.rtm.GetInfo()
	adapter.botID = info.User.Id
	adapter.domain = info.Team.Domain
	for _, channel := range info.Channels {
		if !channel.IsMember {
			continue
		}
		adapter.channelInfo[channel.Id] = channelGroupInfo{
			ID:        channel.Id,
			Name:      channel.Name,
			IsChannel: true,
		}
	}
	for _, group := range info.Groups {
		adapter.channelInfo[group.Id] = channelGroupInfo{
			ID:   group.Id,
			Name: group.Name,
		}
	}
	for _, im := range info.IMs {
		adapter.channelInfo[im.Id] = channelGroupInfo{
			ID:     im.Id,
			Name:   fmt.Sprintf("DM %s", im.Id),
			IsDM:   true,
			UserID: im.UserId,
		}
		adapter.directMessageID[im.UserId] = im.Id
	}
	for _, user := range info.Users {
		adapter.userInfo[user.Id] = user
	}
}

// Stop stops the adapter.
// TODO implement
func (adapter *SlackAdapter) Stop() {
}

// ID returns a unique ID for this adapter. At the moment this just returns
// the slack instance token but could be modified to return a uuid using a
// package such as https://godoc.org/code.google.com/p/go-uuid/uuid
func (adapter *SlackAdapter) ID() string {
	return adapter.token
}

func (adapter *SlackAdapter) getUserFromSlack(userID string) (*slack.User, error) {
	// try to get the stored user info
	user, exists := adapter.userInfo[userID]
	// if it hasn't been stored then perform a slack API call to get it and
	// store it
	if !exists {
		user, err := adapter.instance.GetUserInfo(userID)
		if err != nil {
			log.Println(err.Error())
			return nil, err
		}
		// try to encode it as a json string for storage
		adapter.userInfo[user.Id] = *user
		return user, nil
	}

	return &user, nil
}

func (adapter *SlackAdapter) handleMessage(event *slack.MessageEvent) {
	if len(event.SubType) > 0 {
		return
	}
	user, _ := adapter.getUserFromSlack(event.UserId)
	channel, exists := adapter.channelInfo[event.ChannelId]
	if !exists {
		log.Printf("Unrecognized channel with ID %s", event.Id)
		channel = channelGroupInfo{
			Name: "Unrecognized",
			ID:   event.ChannelId,
		}
	}
	// TODO use error
	if user != nil {
		// ignore any messages that are sent by any bot
		if user.IsBot {
			return
		}
		messageText := adapter.unescapeMessage(event.Text)
		var archiveLink string
		if !channel.IsDM {
			archiveLink = adapter.getArchiveLink(channel.Name, event.Timestamp)
		} else {
			archiveLink = "No archive link for Direct Messages"
		}
		msg := chat.BaseMessage{
			MsgUser: &chat.BaseUser{
				UserID:    user.Id,
				UserName:  user.Name,
				UserEmail: user.Profile.Email,
			},
			MsgText:        messageText,
			MsgChannelID:   channel.ID,
			MsgChannelName: channel.Name,
			MsgIsDirect:    channel.IsDM,
			MsgTimestamp:   strings.SplitN(event.Timestamp, ".", 2)[0],
			MsgArchiveLink: archiveLink,
		}
		adapter.robot.Receive(&msg)
	}
}

const archiveURLFormat = "http://%s.slack.com/archives/%s/p%s"

func (adapter *SlackAdapter) getArchiveLink(channelName, timestamp string) string {
	return fmt.Sprintf(archiveURLFormat, adapter.domain, channelName, strings.Replace(timestamp, ".", "", 1))
}

// Replace all instances of the bot's encoded name with it's actual name.
//
// TODO might want to handle unescaping emails and urls here
func (adapter *SlackAdapter) unescapeMessage(msg string) string {
	userID := getEncodedUserID(adapter.botID)
	if strings.HasPrefix(msg, userID) {
		return strings.Replace(msg, userID, adapter.robot.Name(), 1)
	}
	return msg
}

// Returns the encoded string version of a user's slack ID.
func getEncodedUserID(userID string) string {
	return fmt.Sprintf("<@%s>", userID)
}

// monitorEvents handles incoming events and filters them to only worry about
// incoming messages.
func (adapter *SlackAdapter) monitorEvents() {
	for {
		event := <-adapter.rtm.IncomingEvents
		switch e := event.Data.(type) {
		case *slack.ConnectingEvent:
			log.Println(adapter.token + " connecting")
		case *slack.ConnectedEvent:
			log.Println(adapter.token + " connected")
			adapter.initAdapterInfo(e.Info)
		case *slack.DisconnectedEvent:
			// TODO handle disconnect
			log.Println(adapter.token + " disconnected")
		case *slack.MessageEvent:
			go adapter.handleMessage(e)
		case *slack.ChannelJoinedEvent:
			go adapter.joinedChannel(e.Channel, true)
		case *slack.GroupJoinedEvent:
			go adapter.joinedChannel(e.Channel, false)
		case *slack.IMCreatedEvent:
			go adapter.joinedIM(e)
		case *slack.ChannelLeftEvent:
			go adapter.leftChannel(e.ChannelId)
		case *slack.GroupLeftEvent:
			go adapter.leftChannel(e.ChannelId)
		case *slack.IMCloseEvent:
			go adapter.leftIM(e)
		case *slack.TeamDomainChangeEvent:
			go adapter.domainChanged(e)
		case *slack.UserChangeEvent:
			go adapter.userChanged(e.User)
		case *slack.TeamJoinEvent:
			go adapter.userChanged(*e.User)
		}
	}
}

func (adapter *SlackAdapter) userChanged(user slack.User) {
	if user.IsBot {
		return
	}
	adapter.userInfo[user.Id] = user
}

func (adapter *SlackAdapter) domainChanged(event *slack.TeamDomainChangeEvent) {
	adapter.domain = event.Domain
}

func (adapter *SlackAdapter) joinedChannel(channel slack.Channel, isChannel bool) {
	adapter.channelInfo[channel.Id] = channelGroupInfo{
		Name:      channel.Name,
		ID:        channel.Id,
		IsChannel: isChannel,
	}
}

func (adapter *SlackAdapter) joinedIM(event *slack.IMCreatedEvent) {
	adapter.channelInfo[event.Channel.Id] = channelGroupInfo{
		Name:   event.Channel.Name,
		ID:     event.Channel.Id,
		IsDM:   true,
		UserID: event.UserId,
	}
	adapter.directMessageID[event.UserId] = event.Channel.Id
}

func (adapter *SlackAdapter) leftIM(event *slack.IMCloseEvent) {
	adapter.leftChannel(event.ChannelId)
	delete(adapter.directMessageID, event.UserId)
}

func (adapter *SlackAdapter) leftChannel(channelID string) {
	delete(adapter.channelInfo, channelID)
}

// Send sends a message to the given slack channel.
func (adapter *SlackAdapter) Send(channelID, msg string) {
	msgObj := adapter.rtm.NewOutgoingMessage(msg, channelID)
	adapter.rtm.SendMessage(msgObj)
}

// SendDirectMessage sends the given message to the given user in a direct
// (private) message.
func (adapter *SlackAdapter) SendDirectMessage(userID, msg string) {
	channelID, err := adapter.getDirectMessageID(userID)
	if err != nil {
		log.Printf("Error getting direct message channel ID for user \"%s\": %s", userID, err.Error())
		return
	}
	adapter.Send(channelID, msg)
}

func (adapter *SlackAdapter) getDirectMessageID(userID string) (string, error) {
	// need to figure out if the first two bool return values are important
	// https://github.com/nlopes/slack/blob/master/dm.go#L58
	channel, exists := adapter.channelInfo[userID]
	if !exists {
		_, _, channelID, err := adapter.instance.OpenIMChannel(userID)
		return channelID, err
	}
	return channel.ID, nil
}
<commit_msg>Switched to vendored slack api import<commit_after>package slackRealtime

import (
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"

	"github.com/FogCreek/slack"
	"github.com/FogCreek/victor/pkg/chat"
)

// The Slack Websocket's registered adapter name for the victor framework.
const AdapterName = "slackRealtime"

// Prefix for the user's ID which is used when reading/writing from the bot's store
const userInfoPrefix = AdapterName + "."

const userIDRegexpString = "\\b<?@?(U[[:alnum:]]+)(?:(?:|\\S+)?>?)"

// Match "<@Userid>" and "<@UserID|fullname>"
var userIDRegexp = regexp.MustCompile(userIDRegexpString)

// Match "johndoe", "@johndoe",
// not needed?
// var userIDAndNameRegexp = regexp.MustCompile("\\A@?(\\w+)|" + userIDRegexpString)

// channelGroupInfo is used instead of the slack library's Channel struct since we
// are trying to consider channels and groups to be roughly the same while it
// considers them seperate and provides no way to consolidate them on its own.
//
// This also allows us to throw our information that we don't care about (members, etc.).
type channelGroupInfo struct {
	Name      string
	ID        string
	IsDM      bool
	UserID    string
	IsChannel bool
	// UserID is only stored for IM/DM's so we can then send a user a DM as a
	// response if needed
}

// init registers SlackAdapter to the victor chat framework.
func init() {
	chat.Register(AdapterName, func(r chat.Robot) chat.Adapter {
		config, configSet := r.AdapterConfig()
		if !configSet {
			log.Println("A configuration struct implementing the SlackConfig interface must be set.")
			os.Exit(1)
		}
		sConfig, ok := config.(Config)
		if !ok {
			log.Println("The bot's config must implement the SlackConfig interface.")
			os.Exit(1)
		}
		return &SlackAdapter{
			robot:           r,
			chReceiver:      make(chan slack.SlackEvent),
			token:           sConfig.Token(),
			channelInfo:     make(map[string]channelGroupInfo),
			directMessageID: make(map[string]string),
			userInfo:        make(map[string]slack.User),
		}
	})
}

// Config provides the slack adapter with the necessary
// information to open a websocket connection with the slack Real time API.
type Config interface {
	Token() string
}

// Config implements the SlackRealtimeConfig interface to provide a slack
// adapter with the information it needs to authenticate with slack.
type configImpl struct {
	token string
}

// NewConfig returns a new slack configuration instance using the given token.
func NewConfig(token string) configImpl {
	return configImpl{token: token}
}

// Token returns the slack token.
func (c configImpl) Token() string {
	return c.token
}

// SlackAdapter holds all information needed by the adapter to send/receive messages.
type SlackAdapter struct {
	robot           chat.Robot
	token           string
	instance        *slack.Client
	rtm             *slack.RTM
	chReceiver      chan slack.SlackEvent
	channelInfo     map[string]channelGroupInfo
	directMessageID map[string]string
	userInfo        map[string]slack.User
	domain          string
	botID           string
}

// GetUser will parse the given user ID string and then return the user's
// information as provided by the slack API. This will first try to get the
// user's information from a local cache and then will perform a slack API
// call if the user's information is not cached. Returns nil if the user does
// not exist or if an error occurrs during the slack API call.
func (adapter *SlackAdapter) GetUser(userIDStr string) chat.User {
	if !adapter.IsPotentialUser(userIDStr) {
		log.Printf("%s is not a potential user", userIDStr)
		return nil
	}
	userID := adapter.NormalizeUserID(userIDStr)
	userObj, err := adapter.getUserFromSlack(userID)
	if err != nil {
		log.Println("Error getting user: " + err.Error())
		return nil
	}
	return &chat.BaseUser{
		UserID:    userObj.Id,
		UserName:  userObj.Name,
		UserEmail: userObj.Profile.Email,
		UserIsBot: userObj.IsBot,
	}
}

// IsPotentialUser checks if a given string is potentially referring to a slack
// user. Strings given to this function should be trimmed of leading whitespace
// as it does not account for that (it is meant to be used with the fields
// method on the frameworks calls to handlers which are trimmed).
func (adapter *SlackAdapter) IsPotentialUser(userString string) bool {
	return userIDRegexp.MatchString(userString)
}

// normalizeUserID returns a user's ID without the extra formatting that slack
// might add. This will return "U01234567" for inputs: "U01234567",
// "@U01234567", "<@U01234567>", and "<@U01234567|name>"
func (adapter *SlackAdapter) NormalizeUserID(userID string) string {
	userIDArr := userIDRegexp.FindAllStringSubmatch(userID, 1)
	if len(userIDArr) == 0 {
		return userID
	}
	return userIDArr[0][1]
}

// Run starts the adapter and begins to listen for new messages to send/receive.
// At the moment this will crash the program and print the error messages to a
// log if the connection fails.
func (adapter *SlackAdapter) Run() error {
	adapter.instance = slack.New(adapter.token)
	adapter.instance.SetDebug(false)
	adapter.rtm = adapter.instance.NewRTM()
	go adapter.monitorEvents()
	go adapter.rtm.ManageConnection()
	return nil
}

func (adapter *SlackAdapter) initAdapterInfo(info *slack.Info) {
	// info := adapter.rtm.GetInfo()
	adapter.botID = info.User.Id
	adapter.domain = info.Team.Domain
	for _, channel := range info.Channels {
		if !channel.IsMember {
			continue
		}
		adapter.channelInfo[channel.Id] = channelGroupInfo{
			ID:        channel.Id,
			Name:      channel.Name,
			IsChannel: true,
		}
	}
	for _, group := range info.Groups {
		adapter.channelInfo[group.Id] = channelGroupInfo{
			ID:   group.Id,
			Name: group.Name,
		}
	}
	for _, im := range info.IMs {
		adapter.channelInfo[im.Id] = channelGroupInfo{
			ID:     im.Id,
			Name:   fmt.Sprintf("DM %s", im.Id),
			IsDM:   true,
			UserID: im.UserId,
		}
		adapter.directMessageID[im.UserId] = im.Id
	}
	for _, user := range info.Users {
		adapter.userInfo[user.Id] = user
	}
}

// Stop stops the adapter.
// TODO implement
func (adapter *SlackAdapter) Stop() {
}

// ID returns a unique ID for this adapter. At the moment this just returns
// the slack instance token but could be modified to return a uuid using a
// package such as https://godoc.org/code.google.com/p/go-uuid/uuid
func (adapter *SlackAdapter) ID() string {
	return adapter.token
}

func (adapter *SlackAdapter) getUserFromSlack(userID string) (*slack.User, error) {
	// try to get the stored user info
	user, exists := adapter.userInfo[userID]
	// if it hasn't been stored then perform a slack API call to get it and
	// store it
	if !exists {
		user, err := adapter.instance.GetUserInfo(userID)
		if err != nil {
			log.Println(err.Error())
			return nil, err
		}
		// try to encode it as a json string for storage
		adapter.userInfo[user.Id] = *user
		return user, nil
	}

	return &user, nil
}

func (adapter *SlackAdapter) handleMessage(event *slack.MessageEvent) {
	if len(event.SubType) > 0 {
		return
	}
	user, _ := adapter.getUserFromSlack(event.UserId)
	channel, exists := adapter.channelInfo[event.ChannelId]
	if !exists {
		log.Printf("Unrecognized channel with ID %s", event.Id)
		channel = channelGroupInfo{
			Name: "Unrecognized",
			ID:   event.ChannelId,
		}
	}
	// TODO use error
	if user != nil {
		// ignore any messages that are sent by any bot
		if user.IsBot {
			return
		}
		messageText := adapter.unescapeMessage(event.Text)
		var archiveLink string
		if !channel.IsDM {
			archiveLink = adapter.getArchiveLink(channel.Name, event.Timestamp)
		} else {
			archiveLink = "No archive link for Direct Messages"
		}
		msg := chat.BaseMessage{
			MsgUser: &chat.BaseUser{
				UserID:    user.Id,
				UserName:  user.Name,
				UserEmail: user.Profile.Email,
			},
			MsgText:        messageText,
			MsgChannelID:   channel.ID,
			MsgChannelName: channel.Name,
			MsgIsDirect:    channel.IsDM,
			MsgTimestamp:   strings.SplitN(event.Timestamp, ".", 2)[0],
			MsgArchiveLink: archiveLink,
		}
		adapter.robot.Receive(&msg)
	}
}

const archiveURLFormat = "http://%s.slack.com/archives/%s/p%s"

func (adapter *SlackAdapter) getArchiveLink(channelName, timestamp string) string {
	return fmt.Sprintf(archiveURLFormat, adapter.domain, channelName, strings.Replace(timestamp, ".", "", 1))
}

// Replace all instances of the bot's encoded name with it's actual name.
//
// TODO might want to handle unescaping emails and urls here
func (adapter *SlackAdapter) unescapeMessage(msg string) string {
	userID := getEncodedUserID(adapter.botID)
	if strings.HasPrefix(msg, userID) {
		return strings.Replace(msg, userID, adapter.robot.Name(), 1)
	}
	return msg
}

// Returns the encoded string version of a user's slack ID.
func getEncodedUserID(userID string) string {
	return fmt.Sprintf("<@%s>", userID)
}

// monitorEvents handles incoming events and filters them to only worry about
// incoming messages.
func (adapter *SlackAdapter) monitorEvents() {
	for {
		event := <-adapter.rtm.IncomingEvents
		switch e := event.Data.(type) {
		case *slack.InvalidAuthEvent:
			log.Println(adapter.token + " invalid")
		case *slack.ConnectingEvent:
			log.Println(adapter.token + " connecting")
		case *slack.ConnectedEvent:
			log.Println(adapter.token + " connected")
			adapter.initAdapterInfo(e.Info)
		case *slack.DisconnectedEvent:
			// TODO handle disconnect
			log.Println(adapter.token + " disconnected")
		case *slack.MessageEvent:
			go adapter.handleMessage(e)
		case *slack.ChannelJoinedEvent:
			go adapter.joinedChannel(e.Channel, true)
		case *slack.GroupJoinedEvent:
			go adapter.joinedChannel(e.Channel, false)
		case *slack.IMCreatedEvent:
			go adapter.joinedIM(e)
		case *slack.ChannelLeftEvent:
			go adapter.leftChannel(e.ChannelId)
		case *slack.GroupLeftEvent:
			go adapter.leftChannel(e.ChannelId)
		case *slack.IMCloseEvent:
			go adapter.leftIM(e)
		case *slack.TeamDomainChangeEvent:
			go adapter.domainChanged(e)
		case *slack.UserChangeEvent:
			go adapter.userChanged(e.User)
		case *slack.TeamJoinEvent:
			go adapter.userChanged(*e.User)
		}
	}
}

func (adapter *SlackAdapter) userChanged(user slack.User) {
	if user.IsBot {
		return
	}
	adapter.userInfo[user.Id] = user
}

func (adapter *SlackAdapter) domainChanged(event *slack.TeamDomainChangeEvent) {
	adapter.domain = event.Domain
}

func (adapter *SlackAdapter) joinedChannel(channel slack.Channel, isChannel bool) {
	adapter.channelInfo[channel.Id] = channelGroupInfo{
		Name:      channel.Name,
		ID:        channel.Id,
		IsChannel: isChannel,
	}
}

func (adapter *SlackAdapter) joinedIM(event *slack.IMCreatedEvent) {
	adapter.channelInfo[event.Channel.Id] = channelGroupInfo{
		Name:   event.Channel.Name,
		ID:     event.Channel.Id,
		IsDM:   true,
		UserID: event.UserId,
	}
	adapter.directMessageID[event.UserId] = event.Channel.Id
}

func (adapter *SlackAdapter) leftIM(event *slack.IMCloseEvent) {
	adapter.leftChannel(event.ChannelId)
	delete(adapter.directMessageID, event.UserId)
}

func (adapter *SlackAdapter) leftChannel(channelID string) {
	delete(adapter.channelInfo, channelID)
}

// Send sends a message to the given slack channel.
func (adapter *SlackAdapter) Send(channelID, msg string) {
	msgObj := adapter.rtm.NewOutgoingMessage(msg, channelID)
	adapter.rtm.SendMessage(msgObj)
}

// SendDirectMessage sends the given message to the given user in a direct
// (private) message.
func (adapter *SlackAdapter) SendDirectMessage(userID, msg string) {
	channelID, err := adapter.getDirectMessageID(userID)
	if err != nil {
		log.Printf("Error getting direct message channel ID for user \"%s\": %s", userID, err.Error())
		return
	}
	adapter.Send(channelID, msg)
}

func (adapter *SlackAdapter) getDirectMessageID(userID string) (string, error) {
	// need to figure out if the first two bool return values are important
	// https://github.com/nlopes/slack/blob/master/dm.go#L58
	channel, exists := adapter.channelInfo[userID]
	if !exists {
		_, _, channelID, err := adapter.instance.OpenIMChannel(userID)
		return channelID, err
	}
	return channel.ID, nil
}
<|endoftext|>


<commit_before>// +build linux

/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package empty_dir

import (
	"fmt"

	"github.com/golang/glog"
	"golang.org/x/sys/unix"
	"k8s.io/kubernetes/pkg/util/mount"
)

// Defined by Linux - the type number for tmpfs mounts.
const (
	linuxTmpfsMagic     = 0x01021994
	linuxHugetlbfsMagic = 0x958458f6
)

// realMountDetector implements mountDetector in terms of syscalls.
type realMountDetector struct {
	mounter mount.Interface
}

func (m *realMountDetector) GetMountMedium(path string) (storageMedium, bool, error) {
	glog.V(5).Infof("Determining mount medium of %v", path)
	notMnt, err := m.mounter.IsLikelyNotMountPoint(path)
	if err != nil {
		return 0, false, fmt.Errorf("IsLikelyNotMountPoint(%q): %v", path, err)
	}
	buf := unix.Statfs_t{}
	if err := unix.Statfs(path, &buf); err != nil {
		return 0, false, fmt.Errorf("statfs(%q): %v", path, err)
	}

	glog.V(5).Infof("Statfs_t of %v: %+v", path, buf)
	if buf.Type == linuxTmpfsMagic {
		return mediumMemory, !notMnt, nil
	} else if buf.Type == linuxHugetlbfsMagic {
		return mediumHugepages, !notMnt, nil
	}
	return mediumUnknown, !notMnt, nil
}
<commit_msg>Make hugepages comparison work on 32-bit platforms<commit_after>// +build linux

/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package empty_dir

import (
	"fmt"

	"github.com/golang/glog"
	"golang.org/x/sys/unix"
	"k8s.io/kubernetes/pkg/util/mount"
)

// Defined by Linux - the type number for tmpfs mounts.
const (
	linuxTmpfsMagic     = 0x01021994
	linuxHugetlbfsMagic = 0x958458f6
)

// realMountDetector implements mountDetector in terms of syscalls.
type realMountDetector struct {
	mounter mount.Interface
}

func (m *realMountDetector) GetMountMedium(path string) (storageMedium, bool, error) {
	glog.V(5).Infof("Determining mount medium of %v", path)
	notMnt, err := m.mounter.IsLikelyNotMountPoint(path)
	if err != nil {
		return 0, false, fmt.Errorf("IsLikelyNotMountPoint(%q): %v", path, err)
	}
	buf := unix.Statfs_t{}
	if err := unix.Statfs(path, &buf); err != nil {
		return 0, false, fmt.Errorf("statfs(%q): %v", path, err)
	}

	glog.V(5).Infof("Statfs_t of %v: %+v", path, buf)
	if buf.Type == linuxTmpfsMagic {
		return mediumMemory, !notMnt, nil
	} else if int64(buf.Type) == linuxHugetlbfsMagic {
		return mediumHugepages, !notMnt, nil
	}
	return mediumUnknown, !notMnt, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// UTF-8 support.

package utf8

const (
	RuneError = 0xFFFD;
	RuneSelf = 0x80;
	RuneMax = 0x10FFFF;
	UTFMax = 4;
)

const (
	_T1 = 0x00;	// 0000 0000
	_Tx = 0x80;	// 1000 0000
	_T2 = 0xC0;	// 1100 0000
	_T3 = 0xE0;	// 1110 0000
	_T4 = 0xF0;	// 1111 0000
	_T5 = 0xF8;	// 1111 1000

	_Maskx = 0x3F;	// 0011 1111
	_Mask2 = 0x1F;	// 0001 1111
	_Mask3 = 0x0F;	// 0000 1111
	_Mask4 = 0x07;	// 0000 0111

	_Rune1Max = 1<<7 - 1;
	_Rune2Max = 1<<11 - 1;
	_Rune3Max = 1<<16 - 1;
	_Rune4Max = 1<<21 - 1;
)

func decodeRuneInternal(p []byte) (rune, size int, short bool) {
	n := len(p);
	if n < 1 {
		return RuneError, 0, true;
	}
	c0 := p[0];

	// 1-byte, 7-bit sequence?
	if c0 < _Tx {
		return int(c0), 1, false
	}

	// unexpected continuation byte?
	if c0 < _T2 {
		return RuneError, 1, false
	}

	// need first continuation byte
	if n < 2 {
		return RuneError, 1, true
	}
	c1 := p[1];
	if c1 < _Tx || _T2 <= c1 {
		return RuneError, 1, false
	}

	// 2-byte, 11-bit sequence?
	if c0 < _T3 {
		rune = int(c0&_Mask2)<<6 | int(c1&_Maskx);
		if rune <= _Rune1Max {
			return RuneError, 1, false
		}
		return rune, 2, false
	}

	// need second continuation byte
	if n < 3 {
		return RuneError, 1, true
	}
	c2 := p[2];
	if c2 < _Tx || _T2 <= c2 {
		return RuneError, 1, false
	}

	// 3-byte, 16-bit sequence?
	if c0 < _T4 {
		rune = int(c0&_Mask3)<<12 | int(c1&_Maskx)<<6 | int(c2&_Maskx);
		if rune <= _Rune2Max {
			return RuneError, 1, false
		}
		return rune, 3, false
	}

	// need third continuation byte
	if n < 4 {
		return RuneError, 1, true
	}
	c3 := p[3];
	if c3 < _Tx || _T2 <= c3 {
		return RuneError, 1, false
	}

	// 4-byte, 21-bit sequence?
	if c0 < _T5 {
		rune = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx);
		if rune <= _Rune3Max {
			return RuneError, 1, false
		}
		return rune, 4, false
	}

	// error
	return RuneError, 1, false
}

func decodeRuneInStringInternal(s string, i int, n int) (rune, size int, short bool) {
	if n < 1 {
		return RuneError, 0, true;
	}
	c0 := s[i];

	// 1-byte, 7-bit sequence?
	if c0 < _Tx {
		return int(c0), 1, false
	}

	// unexpected continuation byte?
	if c0 < _T2 {
		return RuneError, 1, false
	}

	// need first continuation byte
	if n < 2 {
		return RuneError, 1, true
	}
	c1 := s[i+1];
	if c1 < _Tx || _T2 <= c1 {
		return RuneError, 1, false
	}

	// 2-byte, 11-bit sequence?
	if c0 < _T3 {
		rune = int(c0&_Mask2)<<6 | int(c1&_Maskx);
		if rune <= _Rune1Max {
			return RuneError, 1, false
		}
		return rune, 2, false
	}

	// need second continuation byte
	if n < 3 {
		return RuneError, 1, true
	}
	c2 := s[i+2];
	if c2 < _Tx || _T2 <= c2 {
		return RuneError, 1, false
	}

	// 3-byte, 16-bit sequence?
	if c0 < _T4 {
		rune = int(c0&_Mask3)<<12 | int(c1&_Maskx)<<6 | int(c2&_Maskx);
		if rune <= _Rune2Max {
			return RuneError, 1, false
		}
		return rune, 3, false
	}

	// need third continuation byte
	if n < 4 {
		return RuneError, 1, true
	}
	c3 := s[i+3];
	if c3 < _Tx || _T2 <= c3 {
		return RuneError, 1, false
	}

	// 4-byte, 21-bit sequence?
	if c0 < _T5 {
		rune = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx);
		if rune <= _Rune3Max {
			return RuneError, 1, false
		}
		return rune, 4, false
	}

	// error
	return RuneError, 1, false
}

func FullRune(p []byte) bool {
	rune, size, short := decodeRuneInternal(p);
	return !short
}

func FullRuneInString(s string, i int) bool {
	rune, size, short := decodeRuneInStringInternal(s, i, len(s) - i);
	return !short
}

func DecodeRune(p []byte) (rune, size int) {
	var short bool;
	rune, size, short = decodeRuneInternal(p);
	return;
}

func DecodeRuneInString(s string, i int) (rune, size int) {
	var short bool;
	rune, size, short = decodeRuneInStringInternal(s, i, len(s) - i);
	return;
}

func RuneLen(rune int) int {
	switch {
	case rune <= _Rune1Max:
		return 1;
	case rune <= _Rune2Max:
		return 2;
	case rune <= _Rune3Max:
		return 3;
	case rune <= _Rune4Max:
		return 4;
	}
	return -1;
}

func EncodeRune(rune int, p []byte) int {
	if rune <= _Rune1Max {
		p[0] = byte(rune);
		return 1;
	}

	if rune <= _Rune2Max {
		p[0] = _T2 | byte(rune>>6);
		p[1] = _Tx | byte(rune)&_Maskx;
		return 2;
	}

	if rune > RuneMax {
		rune = RuneError
	}

	if rune <= _Rune3Max {
		p[0] = _T3 | byte(rune>>12);
		p[1] = _Tx | byte(rune>>6)&_Maskx;
		p[2] = _Tx | byte(rune)&_Maskx;
		return 3;
	}

	p[0] = _T4 | byte(rune>>18);
	p[1] = _Tx | byte(rune>>12)&_Maskx;
	p[2] = _Tx | byte(rune>>6)&_Maskx;
	p[3] = _Tx | byte(rune)&_Maskx;
	return 4;
}

func RuneCount(p []byte) int {
	i := 0;
	var n int;
	for n = 0; i < len(p); n++ {
		if p[i] < RuneSelf {
			i++;
		} else {
			rune, size := DecodeRune(p[i:len(p)]);
			i += size;
		}
	}
	return n;
}

func RuneCountInString(s string, i int, l int) int {
	ei := i + l;
	n := 0;
	for n = 0; i < ei; n++ {
		if s[i] < RuneSelf {
			i++;
		} else {
			rune, size, short := decodeRuneInStringInternal(s, i, ei - i);
			i += size;
		}
	}
	return n;
}

<commit_msg>document utf8<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Functions and constants to support text encoded in UTF-8.
// This package calls a Unicode character a rune for brevity.
package utf8

// Numbers fundamental to the encoding.
const (
	RuneError = 0xFFFD;	// the "error" Rune or "replacement character".
	RuneSelf = 0x80;	// characters below Runeself are represented as themselves in a single byte.
	RuneMax = 0x10FFFF;	// maximum Unicode code point.
	UTFMax = 4;	// maximum number of bytes of a UTF-8 encoded Unicode character.
)

const (
	_T1 = 0x00;	// 0000 0000
	_Tx = 0x80;	// 1000 0000
	_T2 = 0xC0;	// 1100 0000
	_T3 = 0xE0;	// 1110 0000
	_T4 = 0xF0;	// 1111 0000
	_T5 = 0xF8;	// 1111 1000

	_Maskx = 0x3F;	// 0011 1111
	_Mask2 = 0x1F;	// 0001 1111
	_Mask3 = 0x0F;	// 0000 1111
	_Mask4 = 0x07;	// 0000 0111

	_Rune1Max = 1<<7 - 1;
	_Rune2Max = 1<<11 - 1;
	_Rune3Max = 1<<16 - 1;
	_Rune4Max = 1<<21 - 1;
)

func decodeRuneInternal(p []byte) (rune, size int, short bool) {
	n := len(p);
	if n < 1 {
		return RuneError, 0, true;
	}
	c0 := p[0];

	// 1-byte, 7-bit sequence?
	if c0 < _Tx {
		return int(c0), 1, false
	}

	// unexpected continuation byte?
	if c0 < _T2 {
		return RuneError, 1, false
	}

	// need first continuation byte
	if n < 2 {
		return RuneError, 1, true
	}
	c1 := p[1];
	if c1 < _Tx || _T2 <= c1 {
		return RuneError, 1, false
	}

	// 2-byte, 11-bit sequence?
	if c0 < _T3 {
		rune = int(c0&_Mask2)<<6 | int(c1&_Maskx);
		if rune <= _Rune1Max {
			return RuneError, 1, false
		}
		return rune, 2, false
	}

	// need second continuation byte
	if n < 3 {
		return RuneError, 1, true
	}
	c2 := p[2];
	if c2 < _Tx || _T2 <= c2 {
		return RuneError, 1, false
	}

	// 3-byte, 16-bit sequence?
	if c0 < _T4 {
		rune = int(c0&_Mask3)<<12 | int(c1&_Maskx)<<6 | int(c2&_Maskx);
		if rune <= _Rune2Max {
			return RuneError, 1, false
		}
		return rune, 3, false
	}

	// need third continuation byte
	if n < 4 {
		return RuneError, 1, true
	}
	c3 := p[3];
	if c3 < _Tx || _T2 <= c3 {
		return RuneError, 1, false
	}

	// 4-byte, 21-bit sequence?
	if c0 < _T5 {
		rune = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx);
		if rune <= _Rune3Max {
			return RuneError, 1, false
		}
		return rune, 4, false
	}

	// error
	return RuneError, 1, false
}

func decodeRuneInStringInternal(s string, i int, n int) (rune, size int, short bool) {
	if n < 1 {
		return RuneError, 0, true;
	}
	c0 := s[i];

	// 1-byte, 7-bit sequence?
	if c0 < _Tx {
		return int(c0), 1, false
	}

	// unexpected continuation byte?
	if c0 < _T2 {
		return RuneError, 1, false
	}

	// need first continuation byte
	if n < 2 {
		return RuneError, 1, true
	}
	c1 := s[i+1];
	if c1 < _Tx || _T2 <= c1 {
		return RuneError, 1, false
	}

	// 2-byte, 11-bit sequence?
	if c0 < _T3 {
		rune = int(c0&_Mask2)<<6 | int(c1&_Maskx);
		if rune <= _Rune1Max {
			return RuneError, 1, false
		}
		return rune, 2, false
	}

	// need second continuation byte
	if n < 3 {
		return RuneError, 1, true
	}
	c2 := s[i+2];
	if c2 < _Tx || _T2 <= c2 {
		return RuneError, 1, false
	}

	// 3-byte, 16-bit sequence?
	if c0 < _T4 {
		rune = int(c0&_Mask3)<<12 | int(c1&_Maskx)<<6 | int(c2&_Maskx);
		if rune <= _Rune2Max {
			return RuneError, 1, false
		}
		return rune, 3, false
	}

	// need third continuation byte
	if n < 4 {
		return RuneError, 1, true
	}
	c3 := s[i+3];
	if c3 < _Tx || _T2 <= c3 {
		return RuneError, 1, false
	}

	// 4-byte, 21-bit sequence?
	if c0 < _T5 {
		rune = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx);
		if rune <= _Rune3Max {
			return RuneError, 1, false
		}
		return rune, 4, false
	}

	// error
	return RuneError, 1, false
}

// FullRune reports whether the bytes in p begin with a full UTF-8 encoding of a rune.
// An invalid encoding is considered a full Rune since it will convert as a width-1 error rune.
func FullRune(p []byte) bool {
	rune, size, short := decodeRuneInternal(p);
	return !short
}

// FullRuneInString is like FullRune but its input is a string.
func FullRuneInString(s string, i int) bool {
	rune, size, short := decodeRuneInStringInternal(s, i, len(s) - i);
	return !short
}

// DecodeRune unpacks the first UTF-8 encoding in p and returns the rune and its width in bytes.
func DecodeRune(p []byte) (rune, size int) {
	var short bool;
	rune, size, short = decodeRuneInternal(p);
	return;
}

// DecodeRuneInString is like DecodeRune but its input is a string.
func DecodeRuneInString(s string, i int) (rune, size int) {
	var short bool;
	rune, size, short = decodeRuneInStringInternal(s, i, len(s) - i);
	return;
}

// RuneLen returns the number of bytes required to encode the rune.
func RuneLen(rune int) int {
	switch {
	case rune <= _Rune1Max:
		return 1;
	case rune <= _Rune2Max:
		return 2;
	case rune <= _Rune3Max:
		return 3;
	case rune <= _Rune4Max:
		return 4;
	}
	return -1;
}

// EncodeRune writes into p (which must be large enough) the UTF-8 encoding of the rune.
// It returns the number of bytes written.
func EncodeRune(rune int, p []byte) int {
	if rune <= _Rune1Max {
		p[0] = byte(rune);
		return 1;
	}

	if rune <= _Rune2Max {
		p[0] = _T2 | byte(rune>>6);
		p[1] = _Tx | byte(rune)&_Maskx;
		return 2;
	}

	if rune > RuneMax {
		rune = RuneError
	}

	if rune <= _Rune3Max {
		p[0] = _T3 | byte(rune>>12);
		p[1] = _Tx | byte(rune>>6)&_Maskx;
		p[2] = _Tx | byte(rune)&_Maskx;
		return 3;
	}

	p[0] = _T4 | byte(rune>>18);
	p[1] = _Tx | byte(rune>>12)&_Maskx;
	p[2] = _Tx | byte(rune>>6)&_Maskx;
	p[3] = _Tx | byte(rune)&_Maskx;
	return 4;
}

// RuneCount returns the number of runes in p.  Erroneous and short
// encodings are treated as single runes of width 1 byte.
func RuneCount(p []byte) int {
	i := 0;
	var n int;
	for n = 0; i < len(p); n++ {
		if p[i] < RuneSelf {
			i++;
		} else {
			rune, size := DecodeRune(p[i:len(p)]);
			i += size;
		}
	}
	return n;
}

// RuneCountInString is like RuneCount but its input is a string.
func RuneCountInString(s string, i int, l int) int {
	ei := i + l;
	n := 0;
	for n = 0; i < ei; n++ {
		if s[i] < RuneSelf {
			i++;
		} else {
			rune, size, short := decodeRuneInStringInternal(s, i, ei - i);
			i += size;
		}
	}
	return n;
}

<|endoftext|>


<commit_before>package Data
import (
  "fmt"
  //"log"
  "../config"
  "gopkg.in/olivere/elastic.v3"
)

type Item struct {
  Id        int    `json:"Id"`
  Title     string `json:"Title"`
  Body      string `json:"Body"`
  Category  int    `json:"Category"`
  Created   string `json:"CreatedAt"`
  Updated   string `json:"UpdatedAt"`
}
type User struct {
  SID       string `json:"SystemId"`
  Device    string `json:"Device"`
  Sub       string `json:"Sub"`
  Created   string `json:"CreatedAt"`
  Updated   string `json:"UpdatedAt"`
}
type Calendar struct {
  Date      string  `json:"Date"`
  Item      string  `json:"ItemId"`
  Users     string  `json:"Users"`
  Device    string  `json:"Device"`
  Created   string  `json:"CreatedAt"`
  Updated   string  `jsno:"UpdatedAt"`
}
type Category struct {
  Name      string  `json:"Name"`
  Created   string  `json:"CreatedAt"`
  Updated   string  `json:"UpdatedAt"`
}

var con Config.Base = Config.Get()
var Index string = "Push"
func Client()(client *elastic.Client) {
  client, err := elastic.NewClient(elastic.SetURL(fmt.Sprintf("%s:%s", con.Elasticsearch.URL, con.Elasticsearch.BASE_PORT)))
  if err != nil {panic(err)}
  return client
}

func Alive()(version string){
  client := Client()
  ver, err := client.ElasticsearchVersion(fmt.Sprintf("%s:%s", con.Elasticsearch.URL, con.Elasticsearch.BASE_PORT))
  if err != nil { panic(err) }
  return ver
}
func ExistsIndex(Name string)(result bool) {
  client := Client()
  exists, err := client.IndexExists(Name).Do()
  if err != nil {panic(err)}
  return exists
}
func CreateIndex(Index string)(result bool) {
  return true
}
<commit_msg>modify DB Setting<commit_after>package Data
import (
  "fmt"
  //"log"
  "../config"
  "gopkg.in/olivere/elastic.v3"
)

type Item struct {
  Title     string `json:"Title"`
  Body      string `json:"Body"`
  Category  int    `json:"Category"`
  Created   string `json:"CreatedAt"`
  Updated   string `json:"UpdatedAt"`
}
type User struct {
  SID       string `json:"SystemId"`
  Device    string `json:"Device"`
  Sub       string `json:"Sub"`
  Created   string `json:"CreatedAt"`
  Updated   string `json:"UpdatedAt"`
}
type Calendar struct {
  Date      string  `json:"Date"`
  Item      string  `json:"ItemId"`
  Users     string  `json:"Users"`
  Device    string  `json:"Device"`
  Created   string  `json:"CreatedAt"`
  Updated   string  `jsno:"UpdatedAt"`
}
type Category struct {
  Name      string  `json:"Name"`
  Created   string  `json:"CreatedAt"`
  Updated   string  `json:"UpdatedAt"`
}

var con Config.Base = Config.Get()
var Index string = "Push"
func Client()(client *elastic.Client) {
  client, err := elastic.NewClient(elastic.SetURL(fmt.Sprintf("%s:%s", con.Elasticsearch.URL, con.Elasticsearch.BASE_PORT)))
  if err != nil {panic(err)}
  return client
}

func Alive()(version string){
  client := Client()
  ver, err := client.ElasticsearchVersion(fmt.Sprintf("%s:%s", con.Elasticsearch.URL, con.Elasticsearch.BASE_PORT))
  if err != nil { panic(err) }
  return ver
}
func ExistsIndex(Name string)(result bool) {
  client := Client()
  exists, err := client.IndexExists(Name).Do()
  if err != nil {panic(err)}
  return exists
}
func CreateIndex(Index string)(result bool) {
  return true
}
<|endoftext|>