<commit_before>package app

import (
	"golang.org/x/crypto/ssh"
	"log"
	"time"
	"io/ioutil"
)

const MAX_RETRIES_SERVER = 16

type GetServerReq struct {
	reply          chan *ssh.Client
	returnDirectly bool
}

type ConnectionDone struct {
	client *ssh.Client
	err    error
}

const (
	SSH_SERVER_DISCONNECTED = iota
	SSH_SERVER_CONNECTING = iota
	SSH_SERVER_CONNECTED = iota
)

func (b *backendStruct)sshServerConnector() {
	var client *ssh.Client
	state := SSH_SERVER_DISCONNECTED
	waitq := make([]chan *ssh.Client, 0)

	wd := watchdog(b)

	connectionDone := make(chan *ssh.Client)
	for {
		select {
		case req := <-b.getServerChan:
			if req.returnDirectly || client != nil {
				req.reply <- client
			} else {
				waitq = append(waitq, req.reply)
			}
			if client == nil && state == SSH_SERVER_DISCONNECTED && b.info.SSHTunnel != nil {
				state = SSH_SERVER_CONNECTING
				go connectSSH(b.info, connectionDone, b.progressChan)
			}
		case c := <-connectionDone:
			client = c
			if c != nil {
				state = SSH_SERVER_CONNECTED
				for _, reply := range waitq {
					reply <- c
				}
				waitq = nil
			} else {
				state = SSH_SERVER_DISCONNECTED
			}
		case reply := <-b.reconnectServerChan:
			waitq = append(waitq, reply)
			if state != SSH_SERVER_CONNECTING {
				client = nil
				state = SSH_SERVER_CONNECTING
				go connectSSH(b.info, connectionDone, b.progressChan)
			}
		case bark := <-wd:
			bark <- true
		}
	}
}

func dialSSH(info *SSHTunnel, config *ssh.ClientConfig, proxyCommand string) (*ssh.Client, error) {
	if proxyCommand == "" {
		return ssh.Dial(`tcp`, info.Address, config)
	} else {
		conn, err := connectProxy(proxyCommand, info.Address)
		if err != nil {
			return nil, err
		}
		c, chans, reqs, err := ssh.NewClientConn(conn, info.Address, config)
		if err != nil {
			return nil, err
		}
		return ssh.NewClient(c, chans, reqs), nil
	}
}

func connectSSH(info PathInfo, resp chan <- *ssh.Client, progress chan <- ProgressCmd) {
	var err error
	log.Printf("SSH-connecting to %s\n", info.SSHTunnel.Address)

	progress <- ProgressCmd{"connection_start", nil}
	sshKey := []byte(info.SSHTunnel.SSHKeyContents)
	if info.SSHTunnel.SSHKeyFileName != "" {
		sshKey, err = ioutil.ReadFile(info.SSHTunnel.SSHKeyFileName)
		if err != nil {
			progress <- ProgressCmd{"connection_failed", "Failed to read SSH key"}
			resp <- nil
			return
		}
	}

	key, err := ssh.ParsePrivateKey(sshKey)
	if err != nil {
		progress <- ProgressCmd{"connection_failed", "Failed to parse SSH key"}
		resp <- nil
		return
	}

	config := &ssh.ClientConfig{
		User: info.SSHTunnel.Username,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(key),
		},
	}

	currentRetriesServer := 0
	var sshClientConn *ssh.Client

	for {
		progress <- ProgressCmd{"connection_try", nil}
		if sshClientConn, err = dialSSH(info.SSHTunnel, config, proxyCommand); err == nil {
			break
		}

		currentRetriesServer++
		log.Printf("SSH Connection failed %s: %s\n", info.SSHTunnel.Address, err.Error())

		if currentRetriesServer < MAX_RETRIES_SERVER {
			log.Println(`Retry...`)
			progress <- ProgressCmd{"connection_retry", nil}
			time.Sleep(1 * time.Second)
		} else {
			progress <- ProgressCmd{"connection_failed", "Connection retry limit reached"}
			resp <- nil
			return
		}
	}
	progress <- ProgressCmd{"connection_established", nil}

	runBootstrap(sshClientConn, info, progress)

	if info.SSHTunnel.Run != nil {
		session, _ := sshClientConn.NewSession()

		modes := ssh.TerminalModes{
			ssh.ECHO:          0,
		}

		if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
			log.Fatalf("request for pseudo terminal failed: %s", err)
		}

		session.Start(info.SSHTunnel.Run.Command)
		time.Sleep(500 * time.Millisecond)
	}
	log.Printf("SSH-connection OK\n")
	progress <- ProgressCmd{"connection_success", nil}
	resp <- sshClientConn
}
<commit_msg>Wait for backend server to be ready<commit_after>package app

import (
	"golang.org/x/crypto/ssh"
	"log"
	"time"
	"io/ioutil"
)

const MAX_RETRIES_SERVER = 5 * 60
const MAX_RETRIES_CLIENT = 5 * 60

type GetServerReq struct {
	reply          chan *ssh.Client
	returnDirectly bool
}

type ConnectionDone struct {
	client *ssh.Client
	err    error
}

const (
	SSH_SERVER_DISCONNECTED = iota
	SSH_SERVER_CONNECTING = iota
	SSH_SERVER_CONNECTED = iota
)

func (b *backendStruct)sshServerConnector() {
	var client *ssh.Client
	state := SSH_SERVER_DISCONNECTED
	waitq := make([]chan *ssh.Client, 0)

	wd := watchdog(b)

	connectionDone := make(chan *ssh.Client)
	for {
		select {
		case req := <-b.getServerChan:
			if req.returnDirectly || client != nil {
				req.reply <- client
			} else {
				waitq = append(waitq, req.reply)
			}
			if client == nil && state == SSH_SERVER_DISCONNECTED && b.info.SSHTunnel != nil {
				state = SSH_SERVER_CONNECTING
				go connectSSH(b.info, connectionDone, b.progressChan)
			}
		case c := <-connectionDone:
			client = c
			if c != nil {
				state = SSH_SERVER_CONNECTED
				for _, reply := range waitq {
					reply <- c
				}
				waitq = nil
			} else {
				state = SSH_SERVER_DISCONNECTED
			}
		case reply := <-b.reconnectServerChan:
			waitq = append(waitq, reply)
			if state != SSH_SERVER_CONNECTING {
				client = nil
				state = SSH_SERVER_CONNECTING
				go connectSSH(b.info, connectionDone, b.progressChan)
			}
		case bark := <-wd:
			bark <- true
		}
	}
}

func dialSSH(info *SSHTunnel, config *ssh.ClientConfig, proxyCommand string) (*ssh.Client, error) {
	if proxyCommand == "" {
		return ssh.Dial(`tcp`, info.Address, config)
	} else {
		conn, err := connectProxy(proxyCommand, info.Address)
		if err != nil {
			return nil, err
		}
		c, chans, reqs, err := ssh.NewClientConn(conn, info.Address, config)
		if err != nil {
			return nil, err
		}
		return ssh.NewClient(c, chans, reqs), nil
	}
}

func connectSSH(info PathInfo, resp chan <- *ssh.Client, progress chan <- ProgressCmd) {
	var err error
	log.Printf("SSH-connecting to %s\n", info.SSHTunnel.Address)

	progress <- ProgressCmd{"connection_start", nil}
	sshKey := []byte(info.SSHTunnel.SSHKeyContents)
	if info.SSHTunnel.SSHKeyFileName != "" {
		sshKey, err = ioutil.ReadFile(info.SSHTunnel.SSHKeyFileName)
		if err != nil {
			progress <- ProgressCmd{"connection_failed", "Failed to read SSH key"}
			resp <- nil
			return
		}
	}

	key, err := ssh.ParsePrivateKey(sshKey)
	if err != nil {
		progress <- ProgressCmd{"connection_failed", "Failed to parse SSH key"}
		resp <- nil
		return
	}

	config := &ssh.ClientConfig{
		User: info.SSHTunnel.Username,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(key),
		},
	}

	currentRetriesServer := 0
	var sshClientConn *ssh.Client

	for {
		progress <- ProgressCmd{"connection_try", nil}
		if sshClientConn, err = dialSSH(info.SSHTunnel, config, proxyCommand); err == nil {
			break
		}

		currentRetriesServer++
		log.Printf("SSH Connection failed %s: %s\n", info.SSHTunnel.Address, err.Error())

		if currentRetriesServer < (MAX_RETRIES_SERVER / 1) {
			log.Println(`Retry...`)
			progress <- ProgressCmd{"connection_retry", nil}
			time.Sleep(1 * time.Second)
		} else {
			progress <- ProgressCmd{"connection_failed", "Connection retry limit reached"}
			resp <- nil
			return
		}
	}
	progress <- ProgressCmd{"connection_established", nil}

	runBootstrap(sshClientConn, info, progress)

	if info.SSHTunnel.Run != nil {
		session, _ := sshClientConn.NewSession()

		modes := ssh.TerminalModes{
			ssh.ECHO:          0,
		}

		if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
			log.Fatalf("request for pseudo terminal failed: %s", err)
		}

		session.Start(info.SSHTunnel.Run.Command)
		time.Sleep(500 * time.Millisecond)
	}
	log.Printf("SSH-connection OK. Waiting for %s to be ready...\n", info.Backend.Address)

	progress <- ProgressCmd{"waiting_backend", nil}
	currentRetriesClient := 0
	for {
		if conn, err := sshClientConn.Dial("tcp", info.Backend.Address); err == nil {
			conn.Close()
			break
		}
		currentRetriesClient++

		if currentRetriesClient < (MAX_RETRIES_CLIENT / 5) {
			log.Println(`Retry...`)
			progress <- ProgressCmd{"waiting_backend_retry", nil}
			time.Sleep(5 * time.Second)
		} else {
			progress <- ProgressCmd{"waiting_backend_timeout", "Connection retry limit reached"}
			resp <- nil
			return
		}
	}

	progress <- ProgressCmd{"connection_success", nil}
	resp <- sshClientConn
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package impl

import (
	"os"
	"path/filepath"
	"sync"
	"syscall"
	"time"

	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/services/device"
	"v.io/v23/services/stats"
	"v.io/v23/vdl"
	"v.io/v23/verror"
)

const (
	AppcycleReconciliation = "V23_APPCYCLE_RECONCILIATION"
)

var (
	errPIDIsNotInteger = verror.Register(pkgPath+".errPIDIsNotInteger", verror.NoRetry, "{1:}{2:} __debug/stats/system/pid isn't an integer{:_}")

	v23PIDMgmt = true
)

func init() {
	// TODO(rjkroege): Environment variables do not survive device manager updates.
	// Use an alternative mechanism.
	if os.Getenv(AppcycleReconciliation) != "" {
		v23PIDMgmt = false
	}
}

type pidInstanceDirPair struct {
	instanceDir string
	pid         int
}

type reaper struct {
	c   chan pidInstanceDirPair
	ctx *context.T
}

var stashedPidMap map[string]int

func newReaper(ctx *context.T, root string, appRunner *appRunner) (*reaper, error) {
	pidMap, restartCandidates, err := findAllTheInstances(ctx, root)

	// Used only by the testing code that verifies that all processes
	// have been shutdown.
	stashedPidMap = pidMap
	if err != nil {
		return nil, err
	}

	r := &reaper{
		c:   make(chan pidInstanceDirPair),
		ctx: ctx,
	}
	go r.processStatusPolling(ctx, pidMap, appRunner)

	// Restart daemon jobs if they're not running (say because the machine crashed.)
	for _, idir := range restartCandidates {
		go appRunner.restartAppIfNecessary(ctx, idir)
	}
	return r, nil
}

func markNotRunning(ctx *context.T, idir string) {
	if err := transitionInstance(idir, device.InstanceStateRunning, device.InstanceStateNotRunning); err != nil {
		// This may fail under two circumstances.
		// 1. The app has crashed between where startCmd invokes
		// startWatching and where the invoker sets the state to running.
		// 2. Remove experiences a failure (e.g. filesystem becoming R/O)
		// 3. The app is in the process of being Kill'ed when the reaper poll
		// finds the process dead and attempts a restart.
		ctx.Errorf("reaper transitionInstance(%v, %v, %v) failed: %v\n", idir, device.InstanceStateRunning, device.InstanceStateNotRunning, err)
	}
}

// processStatusPolling polls for the continued existence of a set of
// tracked pids. TODO(rjkroege): There are nicer ways to provide this
// functionality. For example, use the kevent facility in darwin or
// replace init. See http://www.incenp.org/dvlpt/wait4.html for
// inspiration.
func (r *reaper) processStatusPolling(ctx *context.T, trackedPids map[string]int, appRunner *appRunner) {
	poll := func(ctx *context.T) {
		for idir, pid := range trackedPids {
			switch err := syscall.Kill(pid, 0); err {
			case syscall.ESRCH:
				// No such PID.
				go appRunner.restartAppIfNecessary(ctx, idir)
				ctx.VI(2).Infof("processStatusPolling discovered pid %d ended", pid)
				markNotRunning(ctx, idir)
				go appRunner.restartAppIfNecessary(ctx, idir)
				delete(trackedPids, idir)
			case nil, syscall.EPERM:
				ctx.VI(2).Infof("processStatusPolling saw live pid: %d", pid)
				// The task exists and is running under the same uid as
				// the device manager or the task exists and is running
				// under a different uid as would be the case if invoked
				// via suidhelper. In this case do, nothing.

				// This implementation cannot detect if a process exited
				// and was replaced by an arbitrary non-Vanadium process
				// within the polling interval.
				// TODO(rjkroege): Probe the appcycle service of the app
				// to confirm that its pid is valid iff v23PIDMgmt
				// is false.

				// TODO(rjkroege): if we can't connect to the app here via
				// the appcycle manager, the app was probably started under
				// a different agent and cannot be managed. Perhaps we should
				// then kill the app and restart it?
			default:
				// The kill system call manpage says that this can only happen
				// if the kernel claims that 0 is an invalid signal.
				// Only a deeply confused kernel would say this so just give
				// up.
				ctx.Panicf("processStatusPolling: unanticpated result from sys.Kill: %v", err)
			}
		}
	}

	for {
		select {
		case p, ok := <-r.c:
			switch {
			case !ok:
				return
			case p.pid == -1: // stop watching this instance
				delete(trackedPids, p.instanceDir)
				poll(ctx)
			case p.pid == -2: // kill the process
				if pid, ok := trackedPids[p.instanceDir]; ok {
					if err := suidHelper.terminatePid(ctx, pid, nil, nil); err != nil {
						ctx.Errorf("Failure to kill: %v", err)
					}
				}
			case p.pid < 0:
				ctx.Panicf("invalid pid %v", p.pid)
			default:
				trackedPids[p.instanceDir] = p.pid
				poll(ctx)
			}
		case <-time.After(time.Second):
			// Poll once / second.
			poll(ctx)
		}
	}
}

// startWatching begins watching process pid's state. This routine
// assumes that pid already exists. Since pid is delivered to the device
// manager by RPC callback, this seems reasonable.
func (r *reaper) startWatching(idir string, pid int) {
	r.c <- pidInstanceDirPair{instanceDir: idir, pid: pid}
}

// stopWatching stops watching process pid's state.
func (r *reaper) stopWatching(idir string) {
	r.c <- pidInstanceDirPair{instanceDir: idir, pid: -1}
}

// forciblySuspend terminates the process pid
func (r *reaper) forciblySuspend(idir string) {
	r.c <- pidInstanceDirPair{instanceDir: idir, pid: -2}
}

func (r *reaper) shutdown() {
	close(r.c)
}

type pidErrorTuple struct {
	ipath        string
	pid          int
	err          error
	mightRestart bool
}

// In seconds.
const appCycleTimeout = 5

// processStatusViaAppCycleMgr updates the status based on getting the
// pid from the AppCycleMgr because the data in the instance info might
// be outdated: the app may have exited and an arbitrary non-Vanadium
// process may have been executed with the same pid.
func processStatusViaAppCycleMgr(ctx *context.T, c chan<- pidErrorTuple, instancePath string, info *instanceInfo, state device.InstanceState) {
	nctx, _ := context.WithTimeout(ctx, appCycleTimeout*time.Second)

	name := naming.Join(info.AppCycleMgrName, "__debug/stats/system/pid")
	sclient := stats.StatsClient(name)
	v, err := sclient.Value(nctx)
	if err != nil {
		ctx.Infof("Instance: %v error: %v", instancePath, err)
		// No process is actually running for this instance.
		ctx.VI(2).Infof("perinstance stats fetching failed: %v", err)
		if err := transitionInstance(instancePath, state, device.InstanceStateNotRunning); err != nil {
			ctx.Errorf("transitionInstance(%s,%s,%s) failed: %v", instancePath, state, device.InstanceStateNotRunning, err)
		}
		// We only want to restart apps that were Running or Launching.
		if state == device.InstanceStateLaunching || state == device.InstanceStateRunning {
			c <- pidErrorTuple{ipath: instancePath, err: err, mightRestart: true}
		} else {
			c <- pidErrorTuple{ipath: instancePath, err: err}
		}
		return
	}
	// Convert the stat value from *vdl.Value into an int pid.
	var pid int
	if err := vdl.Convert(&pid, v); err != nil {
		err = verror.New(errPIDIsNotInteger, ctx, err)
		ctx.Errorf(err.Error())
		c <- pidErrorTuple{ipath: instancePath, err: err}
		return
	}

	ptuple := pidErrorTuple{ipath: instancePath, pid: pid}

	// Update the instance info.
	if info.Pid != pid {
		info.Pid = pid
		ptuple.err = saveInstanceInfo(ctx, instancePath, info)
	}

	// The instance was found to be running, so update its state accordingly
	// (in case the device restarted while the instance was in one of the
	// transitional states like launching, dying, etc).
	if err := transitionInstance(instancePath, state, device.InstanceStateRunning); err != nil {
		ctx.Errorf("transitionInstance(%s,%v,%s) failed: %v", instancePath, state, device.InstanceStateRunning, err)
	}

	ctx.VI(0).Infof("perInstance go routine for %v ending", instancePath)
	c <- ptuple
}

// processStatusViaKill updates the status based on sending a kill signal
// to the process. This assumes that most processes on the system are
// likely to be managed by the device manager and a live process is not
// responsive because the agent has been restarted rather than being
// created through a different means.
func processStatusViaKill(ctx *context.T, c chan<- pidErrorTuple, instancePath string, info *instanceInfo, state device.InstanceState) {
	pid := info.Pid

	switch err := syscall.Kill(pid, 0); err {
	case syscall.ESRCH:
		// No such PID.
		if err := transitionInstance(instancePath, state, device.InstanceStateNotRunning); err != nil {
			ctx.Errorf("transitionInstance(%s,%s,%s) failed: %v", instancePath, state, device.InstanceStateNotRunning, err)
		}
		// We only want to restart apps that were Running or Launching.
		if state == device.InstanceStateLaunching || state == device.InstanceStateRunning {
			c <- pidErrorTuple{ipath: instancePath, err: err, pid: pid, mightRestart: true}
		} else {
			c <- pidErrorTuple{ipath: instancePath, err: err, pid: pid}
		}
	case nil, syscall.EPERM:
		// The instance was found to be running, so update its state.
		if err := transitionInstance(instancePath, state, device.InstanceStateRunning); err != nil {
			ctx.Errorf("transitionInstance(%s,%v, %v) failed: %v", instancePath, state, device.InstanceStateRunning, err)
		}
		ctx.VI(0).Infof("perInstance go routine for %v ending", instancePath)
		c <- pidErrorTuple{ipath: instancePath, pid: pid}
	}
}

func perInstance(ctx *context.T, instancePath string, c chan<- pidErrorTuple, wg *sync.WaitGroup) {
	defer wg.Done()
	ctx.Infof("Instance: %v", instancePath)
	state, err := getInstanceState(instancePath)
	switch state {
	// Ignore apps already in deleted and not running states.
	case device.InstanceStateNotRunning:
		c <- pidErrorTuple{ipath: instancePath}
		return
	case device.InstanceStateDeleted:
		return
	// If the app was updating, it means it was already not running, so just
	// update its state back to not running.
	case device.InstanceStateUpdating:
		if err := transitionInstance(instancePath, state, device.InstanceStateNotRunning); err != nil {
			ctx.Errorf("transitionInstance(%s,%s,%s) failed: %v", instancePath, state, device.InstanceStateNotRunning, err)
		}
		return
	}
	ctx.VI(2).Infof("perInstance firing up on %s", instancePath)

	// Read the instance data.
	info, err := loadInstanceInfo(ctx, instancePath)
	if err != nil {
		ctx.Errorf("loadInstanceInfo failed: %v", err)
		// Something has gone badly wrong.
		// TODO(rjkroege,caprita): Consider removing the instance or at
		// least set its state to something indicating error?
		c <- pidErrorTuple{err: err, ipath: instancePath}
		return
	}

	// Remaining states: Launching, Running, Dying. Of these,
	// daemon mode will restart Launching and Running if the process
	// is not alive.
	if !v23PIDMgmt {
		processStatusViaAppCycleMgr(ctx, c, instancePath, info, state)
		return
	}
	processStatusViaKill(ctx, c, instancePath, info, state)
}

// Digs through the directory hierarchy
func findAllTheInstances(ctx *context.T, root string) (map[string]int, []string, error) {
	paths, err := filepath.Glob(filepath.Join(root, "app*", "installation*", "instances", "instance*"))
	if err != nil {
		return nil, nil, err
	}

	pidmap := make(map[string]int)
	pidchan := make(chan pidErrorTuple, len(paths))
	var wg sync.WaitGroup

	for _, pth := range paths {
		wg.Add(1)
		go perInstance(ctx, pth, pidchan, &wg)
	}
	wg.Wait()
	close(pidchan)

	restartCandidates := make([]string, len(paths))
	for p := range pidchan {
		if p.err != nil {
			ctx.Errorf("instance at %s had an error: %v", p.ipath, p.err)
		}
		if p.pid > 0 {
			pidmap[p.ipath] = p.pid
		}
		if p.mightRestart {
			restartCandidates = append(restartCandidates, p.ipath)
		}
	}
	return pidmap, restartCandidates, nil
}

// RunningChildrenProcesses uses the reaper to verify that a test has
// successfully shut down all processes.
func RunningChildrenProcesses() bool {
	return len(stashedPidMap) > 0
}
<commit_msg>x/ref/services/device/deviced/internal/impl: remove extra restartAppIfNecessary<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package impl

import (
	"os"
	"path/filepath"
	"sync"
	"syscall"
	"time"

	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/services/device"
	"v.io/v23/services/stats"
	"v.io/v23/vdl"
	"v.io/v23/verror"
)

const (
	AppcycleReconciliation = "V23_APPCYCLE_RECONCILIATION"
)

var (
	errPIDIsNotInteger = verror.Register(pkgPath+".errPIDIsNotInteger", verror.NoRetry, "{1:}{2:} __debug/stats/system/pid isn't an integer{:_}")

	v23PIDMgmt = true
)

func init() {
	// TODO(rjkroege): Environment variables do not survive device manager updates.
	// Use an alternative mechanism.
	if os.Getenv(AppcycleReconciliation) != "" {
		v23PIDMgmt = false
	}
}

type pidInstanceDirPair struct {
	instanceDir string
	pid         int
}

type reaper struct {
	c   chan pidInstanceDirPair
	ctx *context.T
}

var stashedPidMap map[string]int

func newReaper(ctx *context.T, root string, appRunner *appRunner) (*reaper, error) {
	pidMap, restartCandidates, err := findAllTheInstances(ctx, root)

	// Used only by the testing code that verifies that all processes
	// have been shutdown.
	stashedPidMap = pidMap
	if err != nil {
		return nil, err
	}

	r := &reaper{
		c:   make(chan pidInstanceDirPair),
		ctx: ctx,
	}
	go r.processStatusPolling(ctx, pidMap, appRunner)

	// Restart daemon jobs if they're not running (say because the machine crashed.)
	for _, idir := range restartCandidates {
		go appRunner.restartAppIfNecessary(ctx, idir)
	}
	return r, nil
}

func markNotRunning(ctx *context.T, idir string) {
	if err := transitionInstance(idir, device.InstanceStateRunning, device.InstanceStateNotRunning); err != nil {
		// This may fail under two circumstances.
		// 1. The app has crashed between where startCmd invokes
		// startWatching and where the invoker sets the state to running.
		// 2. Remove experiences a failure (e.g. filesystem becoming R/O)
		// 3. The app is in the process of being Kill'ed when the reaper poll
		// finds the process dead and attempts a restart.
		ctx.Errorf("reaper transitionInstance(%v, %v, %v) failed: %v\n", idir, device.InstanceStateRunning, device.InstanceStateNotRunning, err)
	}
}

// processStatusPolling polls for the continued existence of a set of
// tracked pids. TODO(rjkroege): There are nicer ways to provide this
// functionality. For example, use the kevent facility in darwin or
// replace init. See http://www.incenp.org/dvlpt/wait4.html for
// inspiration.
func (r *reaper) processStatusPolling(ctx *context.T, trackedPids map[string]int, appRunner *appRunner) {
	poll := func(ctx *context.T) {
		for idir, pid := range trackedPids {
			switch err := syscall.Kill(pid, 0); err {
			case syscall.ESRCH:
				// No such PID.
				ctx.VI(2).Infof("processStatusPolling discovered pid %d ended", pid)
				markNotRunning(ctx, idir)
				go appRunner.restartAppIfNecessary(ctx, idir)
				delete(trackedPids, idir)
			case nil, syscall.EPERM:
				ctx.VI(2).Infof("processStatusPolling saw live pid: %d", pid)
				// The task exists and is running under the same uid as
				// the device manager or the task exists and is running
				// under a different uid as would be the case if invoked
				// via suidhelper. In this case do, nothing.

				// This implementation cannot detect if a process exited
				// and was replaced by an arbitrary non-Vanadium process
				// within the polling interval.
				// TODO(rjkroege): Probe the appcycle service of the app
				// to confirm that its pid is valid iff v23PIDMgmt
				// is false.

				// TODO(rjkroege): if we can't connect to the app here via
				// the appcycle manager, the app was probably started under
				// a different agent and cannot be managed. Perhaps we should
				// then kill the app and restart it?
			default:
				// The kill system call manpage says that this can only happen
				// if the kernel claims that 0 is an invalid signal.
				// Only a deeply confused kernel would say this so just give
				// up.
				ctx.Panicf("processStatusPolling: unanticpated result from sys.Kill: %v", err)
			}
		}
	}

	for {
		select {
		case p, ok := <-r.c:
			switch {
			case !ok:
				return
			case p.pid == -1: // stop watching this instance
				delete(trackedPids, p.instanceDir)
				poll(ctx)
			case p.pid == -2: // kill the process
				if pid, ok := trackedPids[p.instanceDir]; ok {
					if err := suidHelper.terminatePid(ctx, pid, nil, nil); err != nil {
						ctx.Errorf("Failure to kill: %v", err)
					}
				}
			case p.pid < 0:
				ctx.Panicf("invalid pid %v", p.pid)
			default:
				trackedPids[p.instanceDir] = p.pid
				poll(ctx)
			}
		case <-time.After(time.Second):
			// Poll once / second.
			poll(ctx)
		}
	}
}

// startWatching begins watching process pid's state. This routine
// assumes that pid already exists. Since pid is delivered to the device
// manager by RPC callback, this seems reasonable.
func (r *reaper) startWatching(idir string, pid int) {
	r.c <- pidInstanceDirPair{instanceDir: idir, pid: pid}
}

// stopWatching stops watching process pid's state.
func (r *reaper) stopWatching(idir string) {
	r.c <- pidInstanceDirPair{instanceDir: idir, pid: -1}
}

// forciblySuspend terminates the process pid
func (r *reaper) forciblySuspend(idir string) {
	r.c <- pidInstanceDirPair{instanceDir: idir, pid: -2}
}

func (r *reaper) shutdown() {
	close(r.c)
}

type pidErrorTuple struct {
	ipath        string
	pid          int
	err          error
	mightRestart bool
}

// In seconds.
const appCycleTimeout = 5

// processStatusViaAppCycleMgr updates the status based on getting the
// pid from the AppCycleMgr because the data in the instance info might
// be outdated: the app may have exited and an arbitrary non-Vanadium
// process may have been executed with the same pid.
func processStatusViaAppCycleMgr(ctx *context.T, c chan<- pidErrorTuple, instancePath string, info *instanceInfo, state device.InstanceState) {
	nctx, _ := context.WithTimeout(ctx, appCycleTimeout*time.Second)

	name := naming.Join(info.AppCycleMgrName, "__debug/stats/system/pid")
	sclient := stats.StatsClient(name)
	v, err := sclient.Value(nctx)
	if err != nil {
		ctx.Infof("Instance: %v error: %v", instancePath, err)
		// No process is actually running for this instance.
		ctx.VI(2).Infof("perinstance stats fetching failed: %v", err)
		if err := transitionInstance(instancePath, state, device.InstanceStateNotRunning); err != nil {
			ctx.Errorf("transitionInstance(%s,%s,%s) failed: %v", instancePath, state, device.InstanceStateNotRunning, err)
		}
		// We only want to restart apps that were Running or Launching.
		if state == device.InstanceStateLaunching || state == device.InstanceStateRunning {
			c <- pidErrorTuple{ipath: instancePath, err: err, mightRestart: true}
		} else {
			c <- pidErrorTuple{ipath: instancePath, err: err}
		}
		return
	}
	// Convert the stat value from *vdl.Value into an int pid.
	var pid int
	if err := vdl.Convert(&pid, v); err != nil {
		err = verror.New(errPIDIsNotInteger, ctx, err)
		ctx.Errorf(err.Error())
		c <- pidErrorTuple{ipath: instancePath, err: err}
		return
	}

	ptuple := pidErrorTuple{ipath: instancePath, pid: pid}

	// Update the instance info.
	if info.Pid != pid {
		info.Pid = pid
		ptuple.err = saveInstanceInfo(ctx, instancePath, info)
	}

	// The instance was found to be running, so update its state accordingly
	// (in case the device restarted while the instance was in one of the
	// transitional states like launching, dying, etc).
	if err := transitionInstance(instancePath, state, device.InstanceStateRunning); err != nil {
		ctx.Errorf("transitionInstance(%s,%v,%s) failed: %v", instancePath, state, device.InstanceStateRunning, err)
	}

	ctx.VI(0).Infof("perInstance go routine for %v ending", instancePath)
	c <- ptuple
}

// processStatusViaKill updates the status based on sending a kill signal
// to the process. This assumes that most processes on the system are
// likely to be managed by the device manager and a live process is not
// responsive because the agent has been restarted rather than being
// created through a different means.
func processStatusViaKill(ctx *context.T, c chan<- pidErrorTuple, instancePath string, info *instanceInfo, state device.InstanceState) {
	pid := info.Pid

	switch err := syscall.Kill(pid, 0); err {
	case syscall.ESRCH:
		// No such PID.
		if err := transitionInstance(instancePath, state, device.InstanceStateNotRunning); err != nil {
			ctx.Errorf("transitionInstance(%s,%s,%s) failed: %v", instancePath, state, device.InstanceStateNotRunning, err)
		}
		// We only want to restart apps that were Running or Launching.
		if state == device.InstanceStateLaunching || state == device.InstanceStateRunning {
			c <- pidErrorTuple{ipath: instancePath, err: err, pid: pid, mightRestart: true}
		} else {
			c <- pidErrorTuple{ipath: instancePath, err: err, pid: pid}
		}
	case nil, syscall.EPERM:
		// The instance was found to be running, so update its state.
		if err := transitionInstance(instancePath, state, device.InstanceStateRunning); err != nil {
			ctx.Errorf("transitionInstance(%s,%v, %v) failed: %v", instancePath, state, device.InstanceStateRunning, err)
		}
		ctx.VI(0).Infof("perInstance go routine for %v ending", instancePath)
		c <- pidErrorTuple{ipath: instancePath, pid: pid}
	}
}

func perInstance(ctx *context.T, instancePath string, c chan<- pidErrorTuple, wg *sync.WaitGroup) {
	defer wg.Done()
	ctx.Infof("Instance: %v", instancePath)
	state, err := getInstanceState(instancePath)
	switch state {
	// Ignore apps already in deleted and not running states.
	case device.InstanceStateNotRunning:
		c <- pidErrorTuple{ipath: instancePath}
		return
	case device.InstanceStateDeleted:
		return
	// If the app was updating, it means it was already not running, so just
	// update its state back to not running.
	case device.InstanceStateUpdating:
		if err := transitionInstance(instancePath, state, device.InstanceStateNotRunning); err != nil {
			ctx.Errorf("transitionInstance(%s,%s,%s) failed: %v", instancePath, state, device.InstanceStateNotRunning, err)
		}
		return
	}
	ctx.VI(2).Infof("perInstance firing up on %s", instancePath)

	// Read the instance data.
	info, err := loadInstanceInfo(ctx, instancePath)
	if err != nil {
		ctx.Errorf("loadInstanceInfo failed: %v", err)
		// Something has gone badly wrong.
		// TODO(rjkroege,caprita): Consider removing the instance or at
		// least set its state to something indicating error?
		c <- pidErrorTuple{err: err, ipath: instancePath}
		return
	}

	// Remaining states: Launching, Running, Dying. Of these,
	// daemon mode will restart Launching and Running if the process
	// is not alive.
	if !v23PIDMgmt {
		processStatusViaAppCycleMgr(ctx, c, instancePath, info, state)
		return
	}
	processStatusViaKill(ctx, c, instancePath, info, state)
}

// Digs through the directory hierarchy
func findAllTheInstances(ctx *context.T, root string) (map[string]int, []string, error) {
	paths, err := filepath.Glob(filepath.Join(root, "app*", "installation*", "instances", "instance*"))
	if err != nil {
		return nil, nil, err
	}

	pidmap := make(map[string]int)
	pidchan := make(chan pidErrorTuple, len(paths))
	var wg sync.WaitGroup

	for _, pth := range paths {
		wg.Add(1)
		go perInstance(ctx, pth, pidchan, &wg)
	}
	wg.Wait()
	close(pidchan)

	restartCandidates := make([]string, len(paths))
	for p := range pidchan {
		if p.err != nil {
			ctx.Errorf("instance at %s had an error: %v", p.ipath, p.err)
		}
		if p.pid > 0 {
			pidmap[p.ipath] = p.pid
		}
		if p.mightRestart {
			restartCandidates = append(restartCandidates, p.ipath)
		}
	}
	return pidmap, restartCandidates, nil
}

// RunningChildrenProcesses uses the reaper to verify that a test has
// successfully shut down all processes.
func RunningChildrenProcesses() bool {
	return len(stashedPidMap) > 0
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package uuid

import (
	"errors"
	"fmt"
)

// Scan implements sql.Scanner so UUIDs can be read from databases transparently
// Currently, database types that map to string and []byte are supported. Please
// consult database-specific driver documentation for matching types.
func (uuid *UUID) Scan(src interface{}) error {
	switch src.(type) {
	case string:
		// see uuid.Parse for required string format
		parsed := Parse(src.(string))

		if parsed == nil {
			return errors.New("Scan: invalid UUID format")
		}

		*uuid = parsed
	case []byte:
		b := src.([]byte)

		// assumes a simple slice of bytes if 16 bytes
		// otherwise attempts to parse
		if len(b) == 16 {
			*uuid = UUID(b)
		} else {
			u := Parse(string(b))

			if u == nil {
				return errors.New("Scan: invalid UUID format")
			}

			*uuid = u
		}

	default:
		return fmt.Errorf("Scan: unable to scan type %T into UUID", src)
	}

	return nil
}
<commit_msg>altered uuid.Scan() so that it allows for empty UUIDs to be read properly (returning a null UUID value)<commit_after>// Copyright 2015 Google Inc.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package uuid

import (
	"errors"
	"fmt"
)

// Scan implements sql.Scanner so UUIDs can be read from databases transparently
// Currently, database types that map to string and []byte are supported. Please
// consult database-specific driver documentation for matching types.
func (uuid *UUID) Scan(src interface{}) error {
	switch src.(type) {
	case string:
		// if an empty UUID comes from a table, we return a null UUID
		if src.(string) == "" {
			return nil
		}

		// see uuid.Parse for required string format
		parsed := Parse(src.(string))

		if parsed == nil {
			return errors.New("Scan: invalid UUID format")
		}

		*uuid = parsed
	case []byte:
		b := src.([]byte)

		// if an empty UUID comes from a table, we return a null UUID
		if len(b) == 0 {
			return nil
		}

		// assumes a simple slice of bytes if 16 bytes
		// otherwise attempts to parse
		if len(b) == 16 {
			*uuid = UUID(b)
		} else {
			u := Parse(string(b))

			if u == nil {
				return errors.New("Scan: invalid UUID format")
			}

			*uuid = u
		}

	default:
		return fmt.Errorf("Scan: unable to scan type %T into UUID", src)
	}

	return nil
}
<|endoftext|>


<commit_before>package veneur

import (
	"fmt"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
)

// Worker is the doodad that does work.
type Worker struct {
	id         int
	PacketChan chan UDPMetric
	ImportChan chan []JSONMetric
	QuitChan   chan struct{}
	processed  int64
	imported   int64
	mutex      *sync.Mutex
	stats      *statsd.Client
	logger     *logrus.Logger
	wm         WorkerMetrics
}

// WorkerMetrics is just a plain struct bundling together the flushed contents of a worker
type WorkerMetrics struct {
	// we do not want to key on the metric's Digest here, because those could
	// collide, and then we'd have to implement a hashtable on top of go maps,
	// which would be silly
	counters   map[MetricKey]*Counter
	gauges     map[MetricKey]*Gauge
	histograms map[MetricKey]*Histo
	sets       map[MetricKey]*Set
	timers     map[MetricKey]*Histo

	// these are used for metrics that shouldn't be forwarded
	localHistograms map[MetricKey]*Histo
	localSets       map[MetricKey]*Set
	localTimers     map[MetricKey]*Histo
}

// NewWorkerMetrics initializes a WorkerMetrics struct
func NewWorkerMetrics() WorkerMetrics {
	return WorkerMetrics{
		counters:        make(map[MetricKey]*Counter),
		gauges:          make(map[MetricKey]*Gauge),
		histograms:      make(map[MetricKey]*Histo),
		sets:            make(map[MetricKey]*Set),
		timers:          make(map[MetricKey]*Histo),
		localHistograms: make(map[MetricKey]*Histo),
		localSets:       make(map[MetricKey]*Set),
		localTimers:     make(map[MetricKey]*Histo),
	}
}

// Upsert creates an entry on the WorkerMetrics struct for the given metrickey, if it does not already exist.
// Returns true if the metric entry was created and false otherwise.
func (wm WorkerMetrics) Upsert(mk MetricKey, localOnly bool, tags []string) bool {
	present := false
	switch mk.Type {
	case "counter":
		if _, present = wm.counters[mk]; !present {
			wm.counters[mk] = NewCounter(mk.Name, tags)
		}
	case "gauge":
		if _, present = wm.gauges[mk]; !present {
			wm.gauges[mk] = NewGauge(mk.Name, tags)
		}
	case "histogram":
		if localOnly {
			if _, present = wm.localHistograms[mk]; !present {
				wm.localHistograms[mk] = NewHist(mk.Name, tags)
			}
		} else {
			if _, present = wm.histograms[mk]; !present {
				wm.histograms[mk] = NewHist(mk.Name, tags)
			}
		}
	case "set":
		if localOnly {
			if _, present = wm.localSets[mk]; !present {
				wm.localSets[mk] = NewSet(mk.Name, tags)
			}
		} else {
			if _, present = wm.sets[mk]; !present {
				wm.sets[mk] = NewSet(mk.Name, tags)
			}
		}
	case "timer":
		if localOnly {
			if _, present = wm.localTimers[mk]; !present {
				wm.localTimers[mk] = NewHist(mk.Name, tags)
			}
		} else {
			if _, present = wm.timers[mk]; !present {
				wm.timers[mk] = NewHist(mk.Name, tags)
			}
		}
		// no need to raise errors on unknown types
		// the caller will probably end up doing that themselves
	}
	return !present
}

// NewWorker creates, and returns a new Worker object.
func NewWorker(id int, stats *statsd.Client, logger *logrus.Logger) *Worker {
	return &Worker{
		id:         id,
		PacketChan: make(chan UDPMetric),
		ImportChan: make(chan []JSONMetric),
		QuitChan:   make(chan struct{}),
		processed:  0,
		imported:   0,
		mutex:      &sync.Mutex{},
		stats:      stats,
		logger:     logger,
		wm:         NewWorkerMetrics(),
	}
}

// Work will start the worker listening for metrics to process or import.
// It will not return until the worker is sent a message to terminate using Stop()
func (w *Worker) Work() {
	for {
		select {
		case m := <-w.PacketChan:
			w.ProcessMetric(&m)
		case m := <-w.ImportChan:
			for _, j := range m {
				w.ImportMetric(j)
			}
		case <-w.QuitChan:
			// We have been asked to stop.
			w.logger.WithField("worker", w.id).Error("Stopping")
			return
		}
	}
}

// ProcessMetric takes a Metric and samples it
//
// This is standalone to facilitate testing
func (w *Worker) ProcessMetric(m *UDPMetric) {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	w.processed++
	w.wm.Upsert(m.MetricKey, m.LocalOnly, m.Tags)

	switch m.Type {
	case "counter":
		w.wm.counters[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
	case "gauge":
		w.wm.gauges[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
	case "histogram":
		if m.LocalOnly {
			w.wm.localHistograms[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
		} else {
			w.wm.histograms[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
		}
	case "set":
		if m.LocalOnly {
			w.wm.localSets[m.MetricKey].Sample(m.Value.(string), m.SampleRate)
		} else {
			w.wm.sets[m.MetricKey].Sample(m.Value.(string), m.SampleRate)
		}
	case "timer":
		if m.LocalOnly {
			w.wm.localTimers[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
		} else {
			w.wm.timers[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
		}
	default:
		w.logger.WithField("type", m.Type).Error("Unknown metric type for processing")
	}
}

// ImportMetric receives a metric from another veneur instance
func (w *Worker) ImportMetric(other JSONMetric) {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	// we don't increment the processed metric counter here, it was already
	// counted by the original veneur that sent this to us
	w.imported++
	w.wm.Upsert(other.MetricKey, false, other.Tags)

	switch other.Type {
	case "set":
		if err := w.wm.sets[other.MetricKey].Combine(other.Value); err != nil {
			w.logger.WithError(err).Error("Could not merge sets")
		}
	case "histogram":
		if err := w.wm.histograms[other.MetricKey].Combine(other.Value); err != nil {
			w.logger.WithError(err).Error("Could not merge histograms")
		}
	case "timer":
		if err := w.wm.timers[other.MetricKey].Combine(other.Value); err != nil {
			w.logger.WithError(err).Error("Could not merge timers")
		}
	default:
		w.logger.WithField("type", other.Type).Error("Unknown metric type for importing")
	}
}

// Flush resets the worker's internal metrics and returns their contents.
func (w *Worker) Flush() WorkerMetrics {
	start := time.Now()
	// This is a critical spot. The worker can't process metrics while this
	// mutex is held! So we try and minimize it by copying the maps of values
	// and assigning new ones.
	w.mutex.Lock()
	ret := w.wm
	processed := w.processed
	imported := w.imported

	w.wm = NewWorkerMetrics()
	w.processed = 0
	w.imported = 0
	w.mutex.Unlock()

	// Track how much time each worker takes to flush.
	w.stats.TimeInMilliseconds(
		"flush.worker_duration_ns",
		float64(time.Now().Sub(start).Nanoseconds()),
		nil,
		1.0,
	)

	w.stats.Count("worker.metrics_processed_total", processed, []string{fmt.Sprintf("worker:%d", w.id)}, 1.0)
	w.stats.Count("worker.metrics_imported_total", imported, []string{fmt.Sprintf("worker:%d", w.id)}, 1.0)

	return ret
}

// Stop tells the worker to stop listening for work requests.
//
// Note that the worker will only stop *after* it has finished its work.
func (w *Worker) Stop() {
	close(w.QuitChan)
}

// EventWorker is similar to a Worker but it collects events and service checks instead of metrics.
type EventWorker struct {
	EventChan        chan UDPEvent
	ServiceCheckChan chan UDPServiceCheck
	mutex            *sync.Mutex
	events           []UDPEvent
	checks           []UDPServiceCheck
	stats            *statsd.Client
}

// NewEventWorker creates an EventWorker ready to collect events and service checks.
func NewEventWorker(stats *statsd.Client) *EventWorker {
	return &EventWorker{
		EventChan:        make(chan UDPEvent),
		ServiceCheckChan: make(chan UDPServiceCheck),
		mutex:            &sync.Mutex{},
		stats:            stats,
	}
}

// Work will start the EventWorker listening for events and service checks.
// This function will never return.
func (ew *EventWorker) Work() {
	for {
		select {
		case evt := <-ew.EventChan:
			ew.mutex.Lock()
			ew.events = append(ew.events, evt)
			ew.mutex.Unlock()
		case svcheck := <-ew.ServiceCheckChan:
			ew.mutex.Lock()
			ew.checks = append(ew.checks, svcheck)
			ew.mutex.Unlock()
		}
	}
}

// Flush returns the EventWorker's stored events and service checks and
// resets the stored contents.
func (ew *EventWorker) Flush() ([]UDPEvent, []UDPServiceCheck) {
	start := time.Now()
	ew.mutex.Lock()

	retevts := ew.events
	retsvchecks := ew.checks
	// these slices will be allocated again at append time
	ew.events = nil
	ew.checks = nil

	ew.mutex.Unlock()
	ew.stats.TimeInMilliseconds("flush.event_worker_duration_ns", float64(time.Now().Sub(start).Nanoseconds()), nil, 1.0)
	return retevts, retsvchecks
}
<commit_msg>Gofmt<commit_after>package veneur

import (
	"fmt"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
)

// Worker is the doodad that does work.
type Worker struct {
	id         int
	PacketChan chan UDPMetric
	ImportChan chan []JSONMetric
	QuitChan   chan struct{}
	processed  int64
	imported   int64
	mutex      *sync.Mutex
	stats      *statsd.Client
	logger     *logrus.Logger
	wm         WorkerMetrics
}

// WorkerMetrics is just a plain struct bundling together the flushed contents of a worker
type WorkerMetrics struct {
	// we do not want to key on the metric's Digest here, because those could
	// collide, and then we'd have to implement a hashtable on top of go maps,
	// which would be silly
	counters   map[MetricKey]*Counter
	gauges     map[MetricKey]*Gauge
	histograms map[MetricKey]*Histo
	sets       map[MetricKey]*Set
	timers     map[MetricKey]*Histo

	// these are used for metrics that shouldn't be forwarded
	localHistograms map[MetricKey]*Histo
	localSets       map[MetricKey]*Set
	localTimers     map[MetricKey]*Histo
}

// NewWorkerMetrics initializes a WorkerMetrics struct
func NewWorkerMetrics() WorkerMetrics {
	return WorkerMetrics{
		counters:        make(map[MetricKey]*Counter),
		gauges:          make(map[MetricKey]*Gauge),
		histograms:      make(map[MetricKey]*Histo),
		sets:            make(map[MetricKey]*Set),
		timers:          make(map[MetricKey]*Histo),
		localHistograms: make(map[MetricKey]*Histo),
		localSets:       make(map[MetricKey]*Set),
		localTimers:     make(map[MetricKey]*Histo),
	}
}

// Upsert creates an entry on the WorkerMetrics struct for the given metrickey, if it does not already exist.
// Returns true if the metric entry was created and false otherwise.
func (wm WorkerMetrics) Upsert(mk MetricKey, localOnly bool, tags []string) bool {
	present := false
	switch mk.Type {
	case "counter":
		if _, present = wm.counters[mk]; !present {
			wm.counters[mk] = NewCounter(mk.Name, tags)
		}
	case "gauge":
		if _, present = wm.gauges[mk]; !present {
			wm.gauges[mk] = NewGauge(mk.Name, tags)
		}
	case "histogram":
		if localOnly {
			if _, present = wm.localHistograms[mk]; !present {
				wm.localHistograms[mk] = NewHist(mk.Name, tags)
			}
		} else {
			if _, present = wm.histograms[mk]; !present {
				wm.histograms[mk] = NewHist(mk.Name, tags)
			}
		}
	case "set":
		if localOnly {
			if _, present = wm.localSets[mk]; !present {
				wm.localSets[mk] = NewSet(mk.Name, tags)
			}
		} else {
			if _, present = wm.sets[mk]; !present {
				wm.sets[mk] = NewSet(mk.Name, tags)
			}
		}
	case "timer":
		if localOnly {
			if _, present = wm.localTimers[mk]; !present {
				wm.localTimers[mk] = NewHist(mk.Name, tags)
			}
		} else {
			if _, present = wm.timers[mk]; !present {
				wm.timers[mk] = NewHist(mk.Name, tags)
			}
		}
		// no need to raise errors on unknown types
		// the caller will probably end up doing that themselves
	}
	return !present
}

// NewWorker creates, and returns a new Worker object.
func NewWorker(id int, stats *statsd.Client, logger *logrus.Logger) *Worker {
	return &Worker{
		id:         id,
		PacketChan: make(chan UDPMetric),
		ImportChan: make(chan []JSONMetric),
		QuitChan:   make(chan struct{}),
		processed:  0,
		imported:   0,
		mutex:      &sync.Mutex{},
		stats:      stats,
		logger:     logger,
		wm:         NewWorkerMetrics(),
	}
}

// Work will start the worker listening for metrics to process or import.
// It will not return until the worker is sent a message to terminate using Stop()
func (w *Worker) Work() {
	for {
		select {
		case m := <-w.PacketChan:
			w.ProcessMetric(&m)
		case m := <-w.ImportChan:
			for _, j := range m {
				w.ImportMetric(j)
			}
		case <-w.QuitChan:
			// We have been asked to stop.
			w.logger.WithField("worker", w.id).Error("Stopping")
			return
		}
	}
}

// ProcessMetric takes a Metric and samples it
//
// This is standalone to facilitate testing
func (w *Worker) ProcessMetric(m *UDPMetric) {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	w.processed++
	w.wm.Upsert(m.MetricKey, m.LocalOnly, m.Tags)

	switch m.Type {
	case "counter":
		w.wm.counters[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
	case "gauge":
		w.wm.gauges[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
	case "histogram":
		if m.LocalOnly {
			w.wm.localHistograms[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
		} else {
			w.wm.histograms[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
		}
	case "set":
		if m.LocalOnly {
			w.wm.localSets[m.MetricKey].Sample(m.Value.(string), m.SampleRate)
		} else {
			w.wm.sets[m.MetricKey].Sample(m.Value.(string), m.SampleRate)
		}
	case "timer":
		if m.LocalOnly {
			w.wm.localTimers[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
		} else {
			w.wm.timers[m.MetricKey].Sample(m.Value.(float64), m.SampleRate)
		}
	default:
		w.logger.WithField("type", m.Type).Error("Unknown metric type for processing")
	}
}

// ImportMetric receives a metric from another veneur instance
func (w *Worker) ImportMetric(other JSONMetric) {
	w.mutex.Lock()
	defer w.mutex.Unlock()

	// we don't increment the processed metric counter here, it was already
	// counted by the original veneur that sent this to us
	w.imported++
	w.wm.Upsert(other.MetricKey, false, other.Tags)

	switch other.Type {
	case "set":
		if err := w.wm.sets[other.MetricKey].Combine(other.Value); err != nil {
			w.logger.WithError(err).Error("Could not merge sets")
		}
	case "histogram":
		if err := w.wm.histograms[other.MetricKey].Combine(other.Value); err != nil {
			w.logger.WithError(err).Error("Could not merge histograms")
		}
	case "timer":
		if err := w.wm.timers[other.MetricKey].Combine(other.Value); err != nil {
			w.logger.WithError(err).Error("Could not merge timers")
		}
	default:
		w.logger.WithField("type", other.Type).Error("Unknown metric type for importing")
	}
}

// Flush resets the worker's internal metrics and returns their contents.
func (w *Worker) Flush() WorkerMetrics {
	start := time.Now()
	// This is a critical spot. The worker can't process metrics while this
	// mutex is held! So we try and minimize it by copying the maps of values
	// and assigning new ones.
	w.mutex.Lock()
	ret := w.wm
	processed := w.processed
	imported := w.imported

	w.wm = NewWorkerMetrics()
	w.processed = 0
	w.imported = 0
	w.mutex.Unlock()

	// Track how much time each worker takes to flush.
	w.stats.TimeInMilliseconds(
		"flush.worker_duration_ns",
		float64(time.Now().Sub(start).Nanoseconds()),
		nil,
		1.0,
	)

	w.stats.Count("worker.metrics_processed_total", processed, []string{fmt.Sprintf("worker:%d", w.id)}, 1.0)
	w.stats.Count("worker.metrics_imported_total", imported, []string{fmt.Sprintf("worker:%d", w.id)}, 1.0)

	return ret
}

// Stop tells the worker to stop listening for work requests.
//
// Note that the worker will only stop *after* it has finished its work.
func (w *Worker) Stop() {
	close(w.QuitChan)
}

// EventWorker is similar to a Worker but it collects events and service checks instead of metrics.
type EventWorker struct {
	EventChan        chan UDPEvent
	ServiceCheckChan chan UDPServiceCheck
	mutex            *sync.Mutex
	events           []UDPEvent
	checks           []UDPServiceCheck
	stats            *statsd.Client
}

// NewEventWorker creates an EventWorker ready to collect events and service checks.
func NewEventWorker(stats *statsd.Client) *EventWorker {
	return &EventWorker{
		EventChan:        make(chan UDPEvent),
		ServiceCheckChan: make(chan UDPServiceCheck),
		mutex:            &sync.Mutex{},
		stats:            stats,
	}
}

// Work will start the EventWorker listening for events and service checks.
// This function will never return.
func (ew *EventWorker) Work() {
	for {
		select {
		case evt := <-ew.EventChan:
			ew.mutex.Lock()
			ew.events = append(ew.events, evt)
			ew.mutex.Unlock()
		case svcheck := <-ew.ServiceCheckChan:
			ew.mutex.Lock()
			ew.checks = append(ew.checks, svcheck)
			ew.mutex.Unlock()
		}
	}
}

// Flush returns the EventWorker's stored events and service checks and
// resets the stored contents.
func (ew *EventWorker) Flush() ([]UDPEvent, []UDPServiceCheck) {
	start := time.Now()
	ew.mutex.Lock()

	retevts := ew.events
	retsvchecks := ew.checks
	// these slices will be allocated again at append time
	ew.events = nil
	ew.checks = nil

	ew.mutex.Unlock()
	ew.stats.TimeInMilliseconds("flush.event_worker_duration_ns", float64(time.Now().Sub(start).Nanoseconds()), nil, 1.0)
	return retevts, retsvchecks
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"go/importer"
	gotypes "go/types"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	outputFile := flag.String("o", "-", "File to output to. Blank or - for stdin")
	templateFile := flag.String("t", "", "File to use as template for sprintf. if blank, just list the types")
	fmtStr := flag.String("f", "%s", "Format string to use on each type before sending to the template")

	flag.Parse()
	var err error

	tmpl := "%s"
	if *templateFile != "" {
		var bytes []byte
		bytes, err = ioutil.ReadFile(*templateFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		tmpl = string(bytes)
	}

	var wr io.WriteCloser = os.Stdout
	if *outputFile != "" && *outputFile != "-" {
		wr, err = os.Create(*outputFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}
	importer := importer.Default()

	// aaaallllrighty that's all the flag stuff outta the way
	// now we read all the packages and fmt.Fprintf(wr, tmpl, types)
	var types []string
	hasFailed := false

	for _, p := range flag.Args() {
		var pkg *gotypes.Package
		pkg, err = importer.Import(p)
		if err != nil {
			fmt.Println(err)
			hasFailed = true
			continue
		}
		pkgName := pkg.Name()
		pkgPath := pkg.Path()
		scope := pkg.Scope()
		names := scope.Names()
		for _, name := range names {
			obj := scope.Lookup(name)
			inScopeRef := fmt.Sprintf("%s.%s", pkgName, name)
			fullNameWithPath := fmt.Sprintf("%s.%s", pkgPath, name)
			if obj.Exported() && obj.Type().String() == fullNameWithPath {
				types = append(types, fmt.Sprintf(*fmtStr, inScopeRef))
			}
		}

	}

	_, err = fmt.Fprintf(wr, tmpl, strings.Join(types, "\n"))
	if err != nil {
		fmt.Printf("Couldn't write types\n", err)
	}
	err = wr.Close()
	if err != nil {
		fmt.Println(err)
		hasFailed = true
	}
	if hasFailed {
		os.Exit(1)
	}

}
<commit_msg>oh yeah need a formatting directive.<commit_after>package main

import (
	"flag"
	"fmt"
	"go/importer"
	gotypes "go/types"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	outputFile := flag.String("o", "-", "File to output to. Blank or - for stdin")
	templateFile := flag.String("t", "", "File to use as template for sprintf. if blank, just list the types")
	fmtStr := flag.String("f", "%s", "Format string to use on each type before sending to the template")

	flag.Parse()
	var err error

	tmpl := "%s"
	if *templateFile != "" {
		var bytes []byte
		bytes, err = ioutil.ReadFile(*templateFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		tmpl = string(bytes)
	}

	var wr io.WriteCloser = os.Stdout
	if *outputFile != "" && *outputFile != "-" {
		wr, err = os.Create(*outputFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}
	importer := importer.Default()

	// aaaallllrighty that's all the flag stuff outta the way
	// now we read all the packages and fmt.Fprintf(wr, tmpl, types)
	var types []string
	hasFailed := false

	for _, p := range flag.Args() {
		var pkg *gotypes.Package
		pkg, err = importer.Import(p)
		if err != nil {
			fmt.Println(err)
			hasFailed = true
			continue
		}
		pkgName := pkg.Name()
		pkgPath := pkg.Path()
		scope := pkg.Scope()
		names := scope.Names()
		for _, name := range names {
			obj := scope.Lookup(name)
			inScopeRef := fmt.Sprintf("%s.%s", pkgName, name)
			fullNameWithPath := fmt.Sprintf("%s.%s", pkgPath, name)
			if obj.Exported() && obj.Type().String() == fullNameWithPath {
				types = append(types, fmt.Sprintf(*fmtStr, inScopeRef))
			}
		}

	}

	_, err = fmt.Fprintf(wr, tmpl, strings.Join(types, "\n"))
	if err != nil {
		fmt.Printf("Couldn't write types: %s\n", err)
	}
	err = wr.Close()
	if err != nil {
		fmt.Println(err)
		hasFailed = true
	}
	if hasFailed {
		os.Exit(1)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer"
	"github.com/blevesearch/bleve/analysis/char_filters/html_char_filter"
	"github.com/blevesearch/bleve/analysis/language/fr"
	"github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter"
	bleveuni "github.com/blevesearch/bleve/analysis/tokenizers/unicode"
	"github.com/blevesearch/bleve/index/store/boltdb"
	"github.com/blevesearch/bleve/index/upside_down"
)

func loadOffer(store *Store, id string) (*jsonOffer, error) {
	data, err := store.Get(id)
	if err != nil {
		return nil, err
	}
	offer := &jsonOffer{}
	err = json.Unmarshal(data, offer)
	return offer, err
}

type offerResult struct {
	Id    string
	Offer *jsonOffer
	Err   error
}

func loadOffers(store *Store) ([]*jsonOffer, error) {
	ids, err := store.List()
	if err != nil {
		return nil, err
	}
	sort.Strings(ids)
	pending := make(chan string, len(ids))
	for _, id := range ids {
		pending <- id
	}
	close(pending)

	results := make(chan offerResult, len(ids))
	running := &sync.WaitGroup{}
	jobs := 4
	for i := 0; i < jobs; i++ {
		running.Add(1)
		go func() {
			defer running.Done()
			for id := range pending {
				offer, err := loadOffer(store, id)
				results <- offerResult{
					Id:    id,
					Offer: offer,
					Err:   err,
				}
			}
		}()
	}
	go func() {
		running.Wait()
		close(results)
	}()

	offers := []*jsonOffer{}
	for r := range results {
		if r.Err != nil {
			fmt.Printf("loading error for %s: %s\n", r.Id, r.Err)
			continue
		}
		offers = append(offers, r.Offer)
	}
	return offers, nil
}

type Offer struct {
	Account   string
	Id        string    `json:"id"`
	HTML      string    `json:"html"`
	Title     string    `json:"title"`
	MinSalary int       `json:"min_salary"`
	MaxSalary int       `json:"max_salary"`
	Date      time.Time `json:"date"`
	URL       string
	Location  string `json:"location"`
	City      string `json:"city"`
	County    string `json:"county"`
	State     string `json:"state"`
	Country   string `json:"country"`
}

const (
	ApecURL = "https://cadres.apec.fr/offres-emploi-cadres/offre.html?numIdOffre="
)

func convertOffer(offer *jsonOffer) (*Offer, error) {
	r := &Offer{
		Account:  offer.Account,
		Id:       offer.Id,
		HTML:     offer.HTML,
		Title:    offer.Title,
		URL:      ApecURL + offer.Id,
		Location: offer.Location,
	}
	min, max, err := parseSalary(offer.Salary)
	if err != nil {
		return nil, fmt.Errorf("cannot parse salary %q: %s", offer.Salary, err)
	}
	d, err := time.Parse("2006-01-02T15:04:05.000+0000", offer.Date)
	if err != nil {
		return nil, err
	}
	r.Date = d
	r.MinSalary = min
	r.MaxSalary = max
	return r, nil
}

func convertOffers(offers []*jsonOffer) ([]*Offer, error) {
	result := make([]*Offer, 0, len(offers))
	for _, o := range offers {
		r, err := convertOffer(o)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

func NewOfferIndex(dir string) (bleve.Index, error) {
	err := os.RemoveAll(dir)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	frTokens := []string{
		lower_case_filter.Name,
		fr.ElisionName,
		fr.StopName,
		fr.LightStemmerName,
	}
	fr := map[string]interface{}{
		"type":          custom_analyzer.Name,
		"tokenizer":     bleveuni.Name,
		"token_filters": frTokens,
	}
	frHtml := map[string]interface{}{
		"type": custom_analyzer.Name,
		"char_filters": []string{
			html_char_filter.Name,
		},
		"tokenizer":     bleveuni.Name,
		"token_filters": frTokens,
	}
	m := bleve.NewIndexMapping()
	err = m.AddCustomAnalyzer("fr", fr)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr: %s", err)
	}
	err = m.AddCustomAnalyzer("fr_html", frHtml)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr_html: %s", err)
	}

	html := bleve.NewTextFieldMapping()
	html.Store = false
	html.IncludeTermVectors = false
	html.Analyzer = "fr_html"

	textFr := bleve.NewTextFieldMapping()
	textFr.Store = false
	textFr.IncludeTermVectors = false
	textFr.Analyzer = "fr"

	text := bleve.NewTextFieldMapping()
	text.Store = false
	text.IncludeInAll = false
	text.IncludeTermVectors = false

	date := bleve.NewDateTimeFieldMapping()
	date.Index = true
	date.Store = false
	date.IncludeTermVectors = false
	date.IncludeInAll = false

	offer := bleve.NewDocumentStaticMapping()
	offer.Dynamic = false
	offer.AddFieldMappingsAt("html", textFr)
	offer.AddFieldMappingsAt("title", textFr)
	offer.AddFieldMappingsAt("city", text)
	offer.AddFieldMappingsAt("county", text)
	offer.AddFieldMappingsAt("state", text)
	offer.AddFieldMappingsAt("country", text)
	offer.AddFieldMappingsAt("date", date)

	m.AddDocumentMapping("offer", offer)
	m.DefaultMapping = offer

	index, err := bleve.NewUsing(dir, m, upside_down.Name, boltdb.Name,
		map[string]interface{}{
			"nosync": true,
		})
	if err != nil {
		return nil, err
	}
	return index, nil
}

var (
	indexCmd     = app.Command("index", "index APEC offers")
	indexMaxSize = indexCmd.Flag("max-count", "maximum number of items to index").
			Short('n').Default("0").Int()
	indexNoIndex = indexCmd.Flag("no-index", "disable indexing").Bool()
	indexVerbose = indexCmd.Flag("verbose", "verbose mode").Short('v').Bool()
)

func indexOffers(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return err
	}
	rawOffers, err := loadOffers(store)
	if err != nil {
		return err
	}
	offers, err := convertOffers(rawOffers)
	if err != nil {
		return err
	}
	if *indexMaxSize > 0 && len(offers) > *indexMaxSize {
		offers = offers[:*indexMaxSize]
	}

	rejected := 0
	geocodingKey := cfg.GeocodingKey()
	if geocodingKey != "" {
		geocoder, err := NewGeocoder(geocodingKey, cfg.Geocoder())
		if err != nil {
			return err
		}
		defer geocoder.Close()
		rejected, err = geocodeOffers(geocoder, offers, *indexVerbose)
		if err != nil {
			return err
		}
		fmt.Printf("%d rejected geocoding\n", rejected)
	}
	if !*indexNoIndex {
		index, err := NewOfferIndex(cfg.Index())
		if err != nil {
			return err
		}
		start := time.Now()
		indexed := 0
		for i, offer := range offers {
			if (i+1)%500 == 0 {
				now := time.Now()
				elapsed := float64(now.Sub(start)) / float64(time.Second)
				fmt.Printf("%d indexed, %.1f/s\n", i+1, float64(i+1)/elapsed)
			}
			err = index.Index(offer.Id, offer)
			if err != nil {
				return err
			}
			indexed += 1
		}
		err = index.Close()
		if err != nil {
			return err
		}
		end := time.Now()
		fmt.Printf("%d/%d documents indexed in %.2fs\n", indexed, len(offers),
			float64(end.Sub(start))/float64(time.Second))
	}
	return nil
}
<commit_msg>index: fix publication date storage/indexing<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer"
	"github.com/blevesearch/bleve/analysis/char_filters/html_char_filter"
	"github.com/blevesearch/bleve/analysis/language/fr"
	"github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter"
	bleveuni "github.com/blevesearch/bleve/analysis/tokenizers/unicode"
	"github.com/blevesearch/bleve/index/store/boltdb"
	"github.com/blevesearch/bleve/index/upside_down"
)

func loadOffer(store *Store, id string) (*jsonOffer, error) {
	data, err := store.Get(id)
	if err != nil {
		return nil, err
	}
	offer := &jsonOffer{}
	err = json.Unmarshal(data, offer)
	return offer, err
}

type offerResult struct {
	Id    string
	Offer *jsonOffer
	Err   error
}

func loadOffers(store *Store) ([]*jsonOffer, error) {
	ids, err := store.List()
	if err != nil {
		return nil, err
	}
	sort.Strings(ids)
	pending := make(chan string, len(ids))
	for _, id := range ids {
		pending <- id
	}
	close(pending)

	results := make(chan offerResult, len(ids))
	running := &sync.WaitGroup{}
	jobs := 4
	for i := 0; i < jobs; i++ {
		running.Add(1)
		go func() {
			defer running.Done()
			for id := range pending {
				offer, err := loadOffer(store, id)
				results <- offerResult{
					Id:    id,
					Offer: offer,
					Err:   err,
				}
			}
		}()
	}
	go func() {
		running.Wait()
		close(results)
	}()

	offers := []*jsonOffer{}
	for r := range results {
		if r.Err != nil {
			fmt.Printf("loading error for %s: %s\n", r.Id, r.Err)
			continue
		}
		offers = append(offers, r.Offer)
	}
	return offers, nil
}

type Offer struct {
	Account   string
	Id        string    `json:"id"`
	HTML      string    `json:"html"`
	Title     string    `json:"title"`
	MinSalary int       `json:"min_salary"`
	MaxSalary int       `json:"max_salary"`
	Date      time.Time `json:"date"`
	URL       string
	Location  string `json:"location"`
	City      string `json:"city"`
	County    string `json:"county"`
	State     string `json:"state"`
	Country   string `json:"country"`
}

const (
	ApecURL = "https://cadres.apec.fr/offres-emploi-cadres/offre.html?numIdOffre="
)

func convertOffer(offer *jsonOffer) (*Offer, error) {
	r := &Offer{
		Account:  offer.Account,
		Id:       offer.Id,
		HTML:     offer.HTML,
		Title:    offer.Title,
		URL:      ApecURL + offer.Id,
		Location: offer.Location,
	}
	min, max, err := parseSalary(offer.Salary)
	if err != nil {
		return nil, fmt.Errorf("cannot parse salary %q: %s", offer.Salary, err)
	}
	d, err := time.Parse("2006-01-02T15:04:05.000+0000", offer.Date)
	if err != nil {
		return nil, err
	}
	r.Date = d
	r.MinSalary = min
	r.MaxSalary = max
	return r, nil
}

func convertOffers(offers []*jsonOffer) ([]*Offer, error) {
	result := make([]*Offer, 0, len(offers))
	for _, o := range offers {
		r, err := convertOffer(o)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

func NewOfferIndex(dir string) (bleve.Index, error) {
	err := os.RemoveAll(dir)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	frTokens := []string{
		lower_case_filter.Name,
		fr.ElisionName,
		fr.StopName,
		fr.LightStemmerName,
	}
	fr := map[string]interface{}{
		"type":          custom_analyzer.Name,
		"tokenizer":     bleveuni.Name,
		"token_filters": frTokens,
	}
	frHtml := map[string]interface{}{
		"type": custom_analyzer.Name,
		"char_filters": []string{
			html_char_filter.Name,
		},
		"tokenizer":     bleveuni.Name,
		"token_filters": frTokens,
	}
	m := bleve.NewIndexMapping()
	err = m.AddCustomAnalyzer("fr", fr)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr: %s", err)
	}
	err = m.AddCustomAnalyzer("fr_html", frHtml)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr_html: %s", err)
	}

	html := bleve.NewTextFieldMapping()
	html.Store = false
	html.IncludeTermVectors = false
	html.Analyzer = "fr_html"

	textFr := bleve.NewTextFieldMapping()
	textFr.Store = false
	textFr.IncludeTermVectors = false
	textFr.Analyzer = "fr"

	text := bleve.NewTextFieldMapping()
	text.Store = false
	text.IncludeInAll = false
	text.IncludeTermVectors = false

	date := bleve.NewDateTimeFieldMapping()
	date.Index = false
	date.Store = true
	date.IncludeTermVectors = false
	date.IncludeInAll = false

	offer := bleve.NewDocumentStaticMapping()
	offer.Dynamic = false
	offer.AddFieldMappingsAt("html", textFr)
	offer.AddFieldMappingsAt("title", textFr)
	offer.AddFieldMappingsAt("city", text)
	offer.AddFieldMappingsAt("county", text)
	offer.AddFieldMappingsAt("state", text)
	offer.AddFieldMappingsAt("country", text)
	offer.AddFieldMappingsAt("date", date)

	m.AddDocumentMapping("offer", offer)
	m.DefaultMapping = offer

	index, err := bleve.NewUsing(dir, m, upside_down.Name, boltdb.Name,
		map[string]interface{}{
			"nosync": true,
		})
	if err != nil {
		return nil, err
	}
	return index, nil
}

var (
	indexCmd     = app.Command("index", "index APEC offers")
	indexMaxSize = indexCmd.Flag("max-count", "maximum number of items to index").
			Short('n').Default("0").Int()
	indexNoIndex = indexCmd.Flag("no-index", "disable indexing").Bool()
	indexVerbose = indexCmd.Flag("verbose", "verbose mode").Short('v').Bool()
)

func indexOffers(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return err
	}
	rawOffers, err := loadOffers(store)
	if err != nil {
		return err
	}
	offers, err := convertOffers(rawOffers)
	if err != nil {
		return err
	}
	if *indexMaxSize > 0 && len(offers) > *indexMaxSize {
		offers = offers[:*indexMaxSize]
	}

	rejected := 0
	geocodingKey := cfg.GeocodingKey()
	if geocodingKey != "" {
		geocoder, err := NewGeocoder(geocodingKey, cfg.Geocoder())
		if err != nil {
			return err
		}
		defer geocoder.Close()
		rejected, err = geocodeOffers(geocoder, offers, *indexVerbose)
		if err != nil {
			return err
		}
		fmt.Printf("%d rejected geocoding\n", rejected)
	}
	if !*indexNoIndex {
		index, err := NewOfferIndex(cfg.Index())
		if err != nil {
			return err
		}
		start := time.Now()
		indexed := 0
		for i, offer := range offers {
			if (i+1)%500 == 0 {
				now := time.Now()
				elapsed := float64(now.Sub(start)) / float64(time.Second)
				fmt.Printf("%d indexed, %.1f/s\n", i+1, float64(i+1)/elapsed)
			}
			err = index.Index(offer.Id, offer)
			if err != nil {
				return err
			}
			indexed += 1
		}
		err = index.Close()
		if err != nil {
			return err
		}
		end := time.Now()
		fmt.Printf("%d/%d documents indexed in %.2fs\n", indexed, len(offers),
			float64(end.Sub(start))/float64(time.Second))
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>Change default port to 21313 to allow clients to run on 8080<commit_after><|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
)

type fileinfo struct {
	name  string
	data  []byte
	index slice
}

type (
	filedata []*fileinfo
	slice    []int
	slices   map[string]slice
)

type config struct {
	pkg  string
	out  string
	in   []string
	data filedata
}

func (d filedata) Len() int           { return len(d) }
func (d filedata) Less(i, j int) bool { return d[i].name < d[j].name }
func (d filedata) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }

func main() {
	defer handlePanic()
	c := &config{
		out: "assets.go",
		pkg: "main",
	}
	c.run()
}

func handlePanic() {
	if err := recover(); err != nil {
		fmt.Fprintf(os.Stderr, "%s", err)
		os.Exit(-1)
	}
}

func (c *config) run() {
	c.parseConfig()
	c.validateConfig()
	c.read()
	c.validateInput()
	c.write()
}

func (c *config) parseConfig() {
	flag.Usage = func() {
		fmt.Printf("Usage: %s [options] <file patterns>\n\n", os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&c.pkg, "pkg", c.pkg, "Package name for generated code.")
	flag.StringVar(&c.out, "out", c.out, "Output file to be generated.")
	flag.Parse()

	c.in = make([]string, flag.NArg())
	for i := range c.in {
		c.in[i] = flag.Arg(i)
	}
}

func (c *config) validateConfig() {
	if flag.NArg() == 0 {
		fmt.Fprintf(os.Stderr, "Missing <file pattern>\n\n")
		flag.Usage()
		os.Exit(1)
	}
}

func (c *config) validateInput() {
	if len(c.data) == 0 {
		fmt.Fprintf(os.Stderr, "No assets to bundle\n\n")
		flag.Usage()
		os.Exit(3)
	}
}

func (c *config) read() {
	d := filedata{}
	for _, pattern := range c.in {
		d = append(d, readPattern(pattern)...)
	}
	d.sortFiles()
	c.data = d
}

func (c *config) write() {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf(header, c.pkg))
	buf.WriteString(fmt.Sprintf("\n\n%s", c.data.String()))

	if err := os.MkdirAll(path.Dir(c.out), os.ModePerm); err != nil {
		panic(err)
	}

	if err := ioutil.WriteFile(c.out, buf.Bytes(), os.ModePerm); err != nil {
		panic(err)
	}
}

func (d filedata) sortFiles() {
	sort.Sort(d)
}

func (d filedata) String() string {
	var buf bytes.Buffer
	buf.WriteString(d.compressedImage())
	buf.WriteString(d.decompressor())
	return buf.String()
}

func (d filedata) compressedImage() string {
	var db bytes.Buffer
	o := 0
	for _, f := range d {
		db.Write(f.data)
		f.index = slice{o, db.Len()}
		o = db.Len()
	}
	c := compress(db.Bytes())
	w := wrap(c, 48, 64)
	return fmt.Sprintf("var compressed = []byte(%s)\n\n", w)
}

func wrap(s string, f, r int) string {
	if f > len(s) {
		f = len(s)
	}
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("\"%s\" +\n", s[0:f]))
	for s = s[f:]; len(s) > r; s = s[r:] {
		buf.WriteString(fmt.Sprintf("\t\"%s\"", s[0:r]))
		if len(s) != r {
			buf.WriteString(" +\n")
		}
	}
	if s != "" {
		buf.WriteString(fmt.Sprintf("\t\"%s\"", s))
	}
	return buf.String()
}

func (d filedata) decompressor() string {
	var buf bytes.Buffer

	sl := len(d)
	buf.WriteString(fmt.Sprintf("var data = make(map[string][]byte, %d)\n", sl))
	buf.WriteString(`
func init() {
	uc := uncompress(compressed)
	compressed = nil
`)

	for _, f := range d {
		s := f.index
		e := fmt.Sprintf("data[%q] = uc[%d:%d]", f.name, s[0], s[1])
		buf.WriteString(fmt.Sprintf("\t%s\n", e))
	}
	buf.WriteString("}\n")
	return buf.String()
}

func readPattern(pattern string) filedata {
	matches, err := filepath.Glob(pattern)
	if err != nil {
		panic(fmt.Errorf("couldn't resolve pattern %s", pattern))
	}
	d := filedata{}
	for _, filename := range matches {
		d = append(d, readFile(filename))
	}
	return d
}

func readFile(filename string) *fileinfo {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(fmt.Errorf("couldn't read from %s", filename))
	}
	return &fileinfo{
		name: filename,
		data: b,
	}
}

func compress(b []byte) string {
	var buf bytes.Buffer
	w := gzip.NewWriter(&buf)
	w.Write(b)
	w.Close()

	s := fmt.Sprintf("%x", buf.Bytes())
	p := make([]string, len(s)/2)
	for i, j := 0, 0; i < len(s); i += 2 {
		p[j] = s[i : i+2]
		j++
	}
	return `\x` + strings.Join(p, `\x`)
}

// here because it's too ugly to go anywhere else
const header = `package %s

import (
	"bytes"
	"compress/gzip"
	"errors"
	"io"
	"sort"
)

// AssetNames returns a list of all assets
func AssetNames() []string {
	an := make([]string, len(data))
	i := 0
	for k := range data {
		an[i] = k
		i++
	}
	sort.Strings(an)
	return an
}

// Get returns an asset by name
func Get(an string) ([]byte, bool) {
	if d, ok := data[an]; ok {
		return d, true
	}
	return nil, false
}

// MustGet returns an asset by name or explodes
func MustGet(an string) []byte {
	if r, ok := Get(an); ok {
		return r
	}
	panic(errors.New("could not find asset: "+an))
}

func uncompress(b []byte) []byte {
	r, err := gzip.NewReader(bytes.NewBuffer(b))
	if err != nil {
		panic(err)
	}
	defer r.Close()
	var buf bytes.Buffer
	io.Copy(&buf, r)
	return buf.Bytes()
}`
<commit_msg>order by modified date<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

type fileinfo struct {
	name  string
	data  []byte
	index slice
	mod   time.Time
}

type (
	filedata []*fileinfo
	slice    []int
	slices   map[string]slice
)

type config struct {
	pkg  string
	out  string
	in   []string
	data filedata
}

func (d filedata) Len() int           { return len(d) }
func (d filedata) Less(i, j int) bool { return d[i].mod.Before(d[j].mod) }
func (d filedata) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }

func main() {
	defer handlePanic()
	c := &config{
		out: "assets.go",
		pkg: "main",
	}
	c.run()
}

func handlePanic() {
	if err := recover(); err != nil {
		fmt.Fprintf(os.Stderr, "%s", err)
		os.Exit(-1)
	}
}

func (c *config) run() {
	c.parseConfig()
	c.validateConfig()
	c.read()
	c.validateInput()
	c.write()
}

func (c *config) parseConfig() {
	flag.Usage = func() {
		fmt.Printf("Usage: %s [options] <file patterns>\n\n", os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&c.pkg, "pkg", c.pkg, "Package name for generated code.")
	flag.StringVar(&c.out, "out", c.out, "Output file to be generated.")
	flag.Parse()

	c.in = make([]string, flag.NArg())
	for i := range c.in {
		c.in[i] = flag.Arg(i)
	}
}

func (c *config) validateConfig() {
	if flag.NArg() == 0 {
		fmt.Fprintf(os.Stderr, "Missing <file pattern>\n\n")
		flag.Usage()
		os.Exit(1)
	}
}

func (c *config) validateInput() {
	if len(c.data) == 0 {
		fmt.Fprintf(os.Stderr, "No assets to bundle\n\n")
		flag.Usage()
		os.Exit(3)
	}
}

func (c *config) read() {
	d := filedata{}
	for _, pattern := range c.in {
		d = append(d, readPattern(pattern)...)
	}
	d.sortFiles()
	c.data = d
}

func (c *config) write() {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf(header, c.pkg))
	buf.WriteString(fmt.Sprintf("\n\n%s", c.data.String()))

	if err := os.MkdirAll(path.Dir(c.out), os.ModePerm); err != nil {
		panic(err)
	}

	if err := ioutil.WriteFile(c.out, buf.Bytes(), os.ModePerm); err != nil {
		panic(err)
	}
}

func (d filedata) sortFiles() {
	sort.Sort(d)
}

func (d filedata) String() string {
	var buf bytes.Buffer
	buf.WriteString(d.compressedImage())
	buf.WriteString(d.decompressor())
	return buf.String()
}

func (d filedata) compressedImage() string {
	var db bytes.Buffer
	o := 0
	for _, f := range d {
		db.Write(f.data)
		f.index = slice{o, db.Len()}
		o = db.Len()
	}
	c := compress(db.Bytes())
	w := wrap(c, 48, 64)
	return fmt.Sprintf("var compressed = []byte(%s)\n\n", w)
}

func wrap(s string, f, r int) string {
	if f > len(s) {
		f = len(s)
	}
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("\"%s\" +\n", s[0:f]))
	for s = s[f:]; len(s) > r; s = s[r:] {
		buf.WriteString(fmt.Sprintf("\t\"%s\"", s[0:r]))
		if len(s) != r {
			buf.WriteString(" +\n")
		}
	}
	if s != "" {
		buf.WriteString(fmt.Sprintf("\t\"%s\"", s))
	}
	return buf.String()
}

func (d filedata) decompressor() string {
	var buf bytes.Buffer

	sl := len(d)
	buf.WriteString(fmt.Sprintf("var data = make(map[string][]byte, %d)\n", sl))
	buf.WriteString(`
func init() {
	uc := uncompress(compressed)
	compressed = nil
`)

	for _, f := range d {
		s := f.index
		e := fmt.Sprintf("data[%q] = uc[%d:%d]", f.name, s[0], s[1])
		buf.WriteString(fmt.Sprintf("\t%s\n", e))
	}
	buf.WriteString("}\n")
	return buf.String()
}

func readPattern(pattern string) filedata {
	matches, err := filepath.Glob(pattern)
	if err != nil {
		panic(fmt.Errorf("couldn't resolve pattern %s", pattern))
	}
	d := filedata{}
	for _, filename := range matches {
		d = append(d, readFile(filename))
	}
	return d
}

func readFile(filename string) *fileinfo {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(fmt.Errorf("couldn't read from %s", filename))
	}
	fi, err := os.Stat(filename)
	if err != nil {
		panic(fmt.Errorf("couldn't stat %s", filename))
	}
	return &fileinfo{
		name: filename,
		data: b,
		mod:  fi.ModTime(),
	}
}

func compress(b []byte) string {
	var buf bytes.Buffer
	w := gzip.NewWriter(&buf)
	w.Write(b)
	w.Close()

	s := fmt.Sprintf("%x", buf.Bytes())
	p := make([]string, len(s)/2)
	for i, j := 0, 0; i < len(s); i += 2 {
		p[j] = s[i : i+2]
		j++
	}
	return `\x` + strings.Join(p, `\x`)
}

// here because it's too ugly to go anywhere else
const header = `package %s

import (
	"bytes"
	"compress/gzip"
	"errors"
	"io"
	"sort"
)

// AssetNames returns a list of all assets
func AssetNames() []string {
	an := make([]string, len(data))
	i := 0
	for k := range data {
		an[i] = k
		i++
	}
	sort.Strings(an)
	return an
}

// Get returns an asset by name
func Get(an string) ([]byte, bool) {
	if d, ok := data[an]; ok {
		return d, true
	}
	return nil, false
}

// MustGet returns an asset by name or explodes
func MustGet(an string) []byte {
	if r, ok := Get(an); ok {
		return r
	}
	panic(errors.New("could not find asset: "+an))
}

func uncompress(b []byte) []byte {
	r, err := gzip.NewReader(bytes.NewBuffer(b))
	if err != nil {
		panic(err)
	}
	defer r.Close()
	var buf bytes.Buffer
	io.Copy(&buf, r)
	return buf.Bytes()
}`
<|endoftext|>


<commit_before>// Command hello is a bunch of simple snippets from the tour of Go.
package main

import (
	"fmt"
	"math"
	"math/cmplx"
	"math/rand"
	"runtime"
	"time"
)

var c, python, java bool

var (
	c2, python2, java2 bool
	num                int
)

func main() {
	rand.Seed(time.Now().UTC().UnixNano())

	fmt.Println("Hello world")
	fmt.Println("My favorite number is", rand.Intn(10))
	fmt.Println(math.Pi) // Pi is an exported varialbe, pi is not (package private).

	printSection("time formatting")
	timeFormatting()

	printSection("simple functions")
	fmt.Println("add(3, 4) =", add(3, 4))

	s1, s2 := swap("apple", "boy")
	fmt.Println("swap(\"apple\", \"boy\") =", s1, s2)

	fmt.Println("nakedReturn(100) // 100 degree celsius to fahrenheit")
	fmt.Println(nakedReturn(100))

	printSection("variable declaractions")
	var i int
	fmt.Println(i, c, java, python, c2, java2, python2, num)

	c3, python3, java3 := true, false, "no!"
	fmt.Println(c3, java3, python3)

	var c4, python4, java4 = true, false, "no!"
	fmt.Println(c4, java4, python4)

	printSection("different types")

	// bool
	//
	// string
	//
	// int  int8  int16  int32  int64
	// uint uint8 uint16 uint32 uint64 uintptr
	//
	// byte // alias for uint8
	//
	// rune // alias for int32
	//      // represents a Unicode code point
	//
	// float32 float64
	//
	// complex64 complex128
	var (
		ToBe     bool       = false
		MaxInt32 uint64     = 1<<64 - 1
		MaxInt64 uint64     = 1<<32 - 1
		z        complex128 = cmplx.Sqrt(-5 + 12i)
	)
	fmt.Printf("Type: %T Value: %v\n", ToBe, ToBe)
	fmt.Printf("Type: %T Value: %v\n", MaxInt32, MaxInt32)
	fmt.Printf("Type: %T Value: %v\n", MaxInt64, MaxInt64)
	fmt.Printf("Type: %T Value: %v\n", z, z)

	printSection("Type conversions")

	int1 := 42
	float1 := float64(int1)
	uint1 := uint(float1)
	float2 := math.Sqrt(float64(30))
	uint2 := uint(float2)
	fmt.Println(int1, float1, uint1, float2, uint2)

	printSection("Type inference")
	int3 := 42
	float3 := 3.142
	complex3 := 0.867 + 0.5i
	fmt.Printf("Type %T = %v\n", int3, int3)
	fmt.Printf("Type %T = %v\n", float3, float3)
	fmt.Printf("Type %T = %v\n", complex3, complex3)

	const Pi = 3.14
	fmt.Printf("Type %T = %v\n", Pi, Pi)

	printSection("Constant types based on context")
	constantsDependsOnContext()

	printSection("For loop")
	for i := 0; i < 10; i++ {
		fmt.Println(i)
	}

	// while loop
	sum := 0
	for sum < 10 {
		fmt.Println("Sum is", sum)
		sum++
	}

	printSection("If")
	fmt.Println(sqrt(2), sqrt(-4))

	fmt.Println(pow(3, 2, 10), pow(3, 3, 20))

	printSection("exercise with loops")
	fmt.Println(mySqrt(2))

	printSection("switch")
	fmt.Print("Go runs on ")
	switch os := runtime.GOOS; os {
	case "darwin":
		fmt.Println("OS X.")
	case "linux":
		fmt.Println("Linux.")
	default:
		fmt.Printf("%s.\n", os)
	}
}

// end of main

// From: https://gobyexample.com/time-formatting-parsing
func timeFormatting() {
	p := fmt.Println

	t := time.Now()
	p("Current time:", t.Format(time.RFC3339))

	p("Using this format to format time: Mon Jan 2 15:04:05 MST 2006")

	printTimeFormatAndFormattedTime(t, "3:04PM")
	printTimeFormatAndFormattedTime(t, "Mon Jan _2 15:04:05 2006")
	printTimeFormatAndFormattedTime(t, "2006-01-02T15:04:05.999999-07:00")

	printSubsectionSeparator()

	fmt.Println("Show time's fields:")
	fmt.Printf("%d-%02d-%02dT%02d:%02d:%02d\n",
		t.Year(), t.Month(), t.Day(),
		t.Hour(), t.Minute(), t.Second())

	printSubsectionSeparator()

	fmt.Println("Error is like this:")
	ansic := "Mon Jan _2 15:04:05 2006"
	_, e := time.Parse(ansic, "8:41PM")
	p(e)
}

func printTimeFormatAndFormattedTime(t time.Time, format string) {
	fmt.Println(format, "=>", t.Format(format))
}

func printSectionSeparator() {
	fmt.Println("\n-----\n")
}

func printSection(header string) {
	fmt.Printf("\n-- %s --\n\n", header)
}

func printSubsectionSeparator() {
	fmt.Println("\n")
}

func add(x, y int) int {
	return x + y
}

func swap(a, b string) (string, string) {
	return b, a
}

// Go's return values may be named. If so, they are treated as variables
// defined at the top of the function.
//
// These names should be used to document the meaning of the return values.
//
// A return statement without arguments returns the named return values. This
// is known as a "naked" return.
//
// Naked return statements should be used only in short functions, as with the
// example shown here. They can harm readability in longer functions.
func nakedReturn(c int) (cdegree, fdegree int) {
	cdegree = c
	fdegree = c*9/5 + 32
	return
}

func constantsDependsOnContext() {
	const (
		// Create a huge number by shifting a 1 bit left 100 places.
		// In other words, the binary number that is 1 followed by 100 zeroes.
		Big = 1 << 100
		// Shift it right again 99 places, so we end up with 1<<1, or 2.
		Small = Big >> 99
	)
	fmt.Println(needInt(Small))
	fmt.Println(needFloat(Small))
	fmt.Println(needFloat(Big))
	fmt.Printf("Type %T = %v\n", Small, Small)
	// Error if uncomment the following:
	// fmt.Printf("Type %T = %v\n", Big, Big)
}
func needInt(x int) int { return x*10 + 1 }
func needFloat(x float64) float64 {
	return x * 0.1
}

func sqrt(x float64) string {
	if x < 0 {
		return sqrt(-x) + "i"
	}
	return fmt.Sprint(math.Sqrt(x))
}

// if with a short statement
func pow(x, n, lim float64) float64 {
	if v := math.Pow(x, n); v < lim {
		// v is only in scope until the end of if/else
		return v
	} else {
		fmt.Printf("%g >= %g\n", v, lim)
	}
	return lim
}

func mySqrt(x float64) float64 {
	z := 1.0
	for i := 0; i < 10; i++ {
		fmt.Println(z)
		z -= (z*z - x) / (2 * z)
	}
	return z
}
<commit_msg>[go][tour] Finished "Struct Literals"<commit_after>// Command hello is a bunch of simple snippets from the tour of Go.
package main

import (
	"fmt"
	"math"
	"math/cmplx"
	"math/rand"
	"runtime"
	"time"
)

var c, python, java bool

var (
	c2, python2, java2 bool
	num                int
)

func main() {
	// defer's are executed in LIFO
	defer fmt.Println("The End!")
	defer printSectionSeparator()

	rand.Seed(time.Now().UTC().UnixNano())

	fmt.Println("Hello world")
	fmt.Println("My favorite number is", rand.Intn(10))
	fmt.Println(math.Pi) // Pi is an exported varialbe, pi is not (package private).

	printSection("time formatting")
	timeFormatting()

	printSection("simple functions")
	fmt.Println("add(3, 4) =", add(3, 4))

	s1, s2 := swap("apple", "boy")
	fmt.Println("swap(\"apple\", \"boy\") =", s1, s2)

	fmt.Println("nakedReturn(100) // 100 degree celsius to fahrenheit")
	fmt.Println(nakedReturn(100))

	printSection("variable declaractions")
	var i int
	fmt.Println(i, c, java, python, c2, java2, python2, num)

	c3, python3, java3 := true, false, "no!"
	fmt.Println(c3, java3, python3)

	var c4, python4, java4 = true, false, "no!"
	fmt.Println(c4, java4, python4)

	printSection("different types")

	// bool
	//
	// string
	//
	// int  int8  int16  int32  int64
	// uint uint8 uint16 uint32 uint64 uintptr
	//
	// byte // alias for uint8
	//
	// rune // alias for int32
	//      // represents a Unicode code point
	//
	// float32 float64
	//
	// complex64 complex128
	var (
		ToBe     bool       = false
		MaxInt32 uint64     = 1<<64 - 1
		MaxInt64 uint64     = 1<<32 - 1
		z        complex128 = cmplx.Sqrt(-5 + 12i)
	)
	fmt.Printf("Type: %T Value: %v\n", ToBe, ToBe)
	fmt.Printf("Type: %T Value: %v\n", MaxInt32, MaxInt32)
	fmt.Printf("Type: %T Value: %v\n", MaxInt64, MaxInt64)
	fmt.Printf("Type: %T Value: %v\n", z, z)

	printSection("Type conversions")

	int1 := 42
	float1 := float64(int1)
	uint1 := uint(float1)
	float2 := math.Sqrt(float64(30))
	uint2 := uint(float2)
	fmt.Println(int1, float1, uint1, float2, uint2)

	printSection("Type inference")
	int3 := 42
	float3 := 3.142
	complex3 := 0.867 + 0.5i
	fmt.Printf("Type %T = %v\n", int3, int3)
	fmt.Printf("Type %T = %v\n", float3, float3)
	fmt.Printf("Type %T = %v\n", complex3, complex3)

	const Pi = 3.14
	fmt.Printf("Type %T = %v\n", Pi, Pi)

	printSection("Constant types based on context")
	constantsDependsOnContext()

	printSection("For loop")
	for i := 0; i < 10; i++ {
		fmt.Println(i)
	}

	// while loop
	sum := 0
	for sum < 10 {
		fmt.Println("Sum is", sum)
		sum++
	}

	printSection("If")
	fmt.Println(sqrt(2), sqrt(-4))

	fmt.Println(pow(3, 2, 10), pow(3, 3, 20))

	greetingWithTime()

	printSection("exercise with loops")
	fmt.Println(mySqrt(2))

	printSection("switch")
	fmt.Print("Go runs on ")
	switch os := runtime.GOOS; os {
	case "darwin":
		fmt.Println("OS X.")
	case "linux":
		fmt.Println("Linux.")
	default:
		fmt.Printf("%s.\n", os)
	}

	greetingWithTimeUsingSwitch()

	printSection("Pointers")
	pointersSample()

	printSection("Structs")
	structsSample()
}

// end of main

// From: https://gobyexample.com/time-formatting-parsing
func timeFormatting() {
	p := fmt.Println

	t := time.Now()
	p("Current time:", t.Format(time.RFC3339))

	p("Using this format to format time: Mon Jan 2 15:04:05 MST 2006")

	printTimeFormatAndFormattedTime(t, "3:04PM")
	printTimeFormatAndFormattedTime(t, "Mon Jan _2 15:04:05 2006")
	printTimeFormatAndFormattedTime(t, "2006-01-02T15:04:05.999999-07:00")

	printSubsectionSeparator()

	fmt.Println("Show time's fields:")
	fmt.Printf("%d-%02d-%02dT%02d:%02d:%02d\n",
		t.Year(), t.Month(), t.Day(),
		t.Hour(), t.Minute(), t.Second())

	printSubsectionSeparator()

	fmt.Println("Error is like this:")
	ansic := "Mon Jan _2 15:04:05 2006"
	_, e := time.Parse(ansic, "8:41PM")
	p(e)
}

func printTimeFormatAndFormattedTime(t time.Time, format string) {
	fmt.Println(format, "=>", t.Format(format))
}

func printSectionSeparator() {
	fmt.Println("\n-----\n")
}

func printSection(header string) {
	fmt.Printf("\n-- %s --\n\n", header)
}

func printSubsectionSeparator() {
	fmt.Println("\n")
}

func printSubsection(subheader string) {
	fmt.Printf("\n%s\n\n", subheader)
}

func add(x, y int) int {
	return x + y
}

func swap(a, b string) (string, string) {
	return b, a
}

// Go's return values may be named. If so, they are treated as variables
// defined at the top of the function.
//
// These names should be used to document the meaning of the return values.
//
// A return statement without arguments returns the named return values. This
// is known as a "naked" return.
//
// Naked return statements should be used only in short functions, as with the
// example shown here. They can harm readability in longer functions.
func nakedReturn(c int) (cdegree, fdegree int) {
	cdegree = c
	fdegree = c*9/5 + 32
	return
}

func constantsDependsOnContext() {
	const (
		// Create a huge number by shifting a 1 bit left 100 places.
		// In other words, the binary number that is 1 followed by 100 zeroes.
		Big = 1 << 100
		// Shift it right again 99 places, so we end up with 1<<1, or 2.
		Small = Big >> 99
	)
	fmt.Println(needInt(Small))
	fmt.Println(needFloat(Small))
	fmt.Println(needFloat(Big))
	fmt.Printf("Type %T = %v\n", Small, Small)
	// Error if uncomment the following:
	// fmt.Printf("Type %T = %v\n", Big, Big)
}
func needInt(x int) int { return x*10 + 1 }
func needFloat(x float64) float64 {
	return x * 0.1
}

func sqrt(x float64) string {
	if x < 0 {
		return sqrt(-x) + "i"
	}
	return fmt.Sprint(math.Sqrt(x))
}

// if with a short statement
func pow(x, n, lim float64) float64 {
	if v := math.Pow(x, n); v < lim {
		// v is only in scope until the end of if/else
		return v
	} else {
		fmt.Printf("%g >= %g\n", v, lim)
	}
	return lim
}

func mySqrt(x float64) float64 {
	z := 1.0
	for i := 0; i < 10; i++ {
		fmt.Println(z)
		z -= (z*z - x) / (2 * z)
	}
	return z
}

func greetingWithTime() {
	if t := time.Now(); t.Hour() < 12 {
		fmt.Println("Good morning!")
	} else if t.Hour() < 17 {
		fmt.Println("Good afternoon!")
	} else {
		fmt.Println("Good evening!")
	}
}

func greetingWithTimeUsingSwitch() {
	t := time.Now()
	switch {
	case t.Hour() < 12:
		fmt.Println("Good morning!")
	case t.Hour() < 17:
		fmt.Println("Good afternoon!")
	default:
		fmt.Println("Good evening!")
	}
}

func pointersSample() {
	var p *int
	i := 42

	p = &i
	fmt.Println(p)
	fmt.Println(*p)

	*p = 21
	fmt.Println("After assigning *p:", *p)

	fmt.Println("Unlike C, Go has no pointer arithmetic.")
}

func structsSample() {
	v := Vertex{1, 2}
	fmt.Println(v)
	fmt.Printf("Type %T: %v\n", v, v)
	fmt.Printf("X = %d \n", v.X)

	p := &v
	// p.X is a shortcut for (*p).X
	fmt.Printf("p.X = %d\n", p.X)
	p.X = 101
	fmt.Printf("After changing p.X, v = %v\n", v)

	printSubsection("struct literals")
	v1 := Vertex{X: 102}
	v2 := Vertex{}
	p2 := &Vertex{5, 7}
	fmt.Println(v1, v2, p2)
}

type Vertex struct {
	X, Y int
}
<|endoftext|>