<commit_before>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/romana/core/agent"
	"github.com/romana/core/agent/enforcer"
	utilexec "github.com/romana/core/agent/exec"
	"github.com/romana/core/agent/policycache"
	"github.com/romana/core/agent/policycontroller"
	"github.com/romana/core/agent/rtable"
	"github.com/romana/core/agent/sysctl"
	"github.com/romana/core/common"
	"github.com/romana/core/common/api"
	"github.com/romana/core/common/client"

	log "github.com/romana/rlog"
	"github.com/vishvananda/netlink"
)

const (
	DefaultRouteTableId = 10
	DefaultGwIP         = "172.142.0.1"
)

var (
	kernelParameter = []string{
		"/proc/sys/net/ipv4/conf/default/proxy_arp",
		"/proc/sys/net/ipv4/conf/all/proxy_arp",
		"/proc/sys/net/ipv4/ip_forward",
	}
)

func main() {
	var err error

	etcdEndpoints := flag.String("endpoints", "", "csv list of etcd endpoints to romana storage")
	etcdPrefix := flag.String("prefix", "", "string that prefixes all romana keys in etcd")
	hostname := flag.String("hostname", "", "name of the host in romana database")
	provisionIface := flag.Bool("provision-iface", false, "create romana-gw interface and ip")
	provisionIfaceGwIp := flag.String("provision-iface-gw-ip", DefaultGwIP, "specifies ip address for gateway interface")
	provisionSysctls := flag.Bool("provision-sysctls", false, "configure routing sysctls")
	romanaRouteTableId := flag.Int("route-table-id", DefaultRouteTableId,
		"id that romana route table should have in /etc/iproute2/rt_tables")
	multihop := flag.Bool("multihop-blocks", false, "allows multihop blocks")
	policyEnforcer := flag.Bool("policy", false, "enable romana policies")
	metricsPort := flag.Int("metrics", 9607, "tcp port to expose prometheus metrics, -1 means disable")
	flag.Parse()

	fmt.Println(common.BuildInfo())

	if err := agent.MetricStart(*metricsPort); err != nil {
		log.Errorf("Failed to start metrics collector")
		os.Exit(2)
	}

	romanaConfig := common.Config{
		EtcdEndpoints: strings.Split(*etcdEndpoints, ","),
		EtcdPrefix:    *etcdPrefix,
	}

	if *hostname == "" {
		*hostname, err = os.Hostname()
		if err != nil {
			panic(err)
		}
	}

	romanaClient, err := client.NewClient(&romanaConfig)
	if err != nil {
		log.Errorf("Failed to initialize romana client: %v", err)
		os.Exit(2)
	}

	if *provisionIface {
		err := agent.CreateRomanaGW()
		if err != nil {
			log.Errorf("Failed to create romana-gw interface. %s", err)
			os.Exit(2)
		}

		err = agent.SetRomanaGwIP(*provisionIfaceGwIp)
		if err != nil {
			log.Errorf("Failed to install ip address on romana-gw interface. %s", err)
			os.Exit(2)
		}

	}

	if *provisionSysctls {
		err := setSysctls()
		if err != nil {
			log.Errorf("Failed to set sysctls %s", err)
			os.Exit(2)
		}
	}

	ok, err := checkSysctls()
	if err != nil {
		log.Errorf("Failed verify the state of essential systls %s", err)
		os.Exit(2)
	}
	if !ok {
		log.Errorf("Essential sysctls not set, consider using -provision-sysctls flag and ensure you are running from root")
		os.Exit(2)
	}

	err = rtable.EnsureRouteTableExist(*romanaRouteTableId)
	if err != nil {
		log.Errorf("Failed to make `romana` alias for route table=%d, %s. Unable to continue", *romanaRouteTableId, err)
		os.Exit(2)
	}

	nlHandle, err := netlink.NewHandle()
	if err != nil {
		log.Errorf("Failed to create netlink handle %s", err)
		os.Exit(2)
	}
	defer nlHandle.Delete()

	err = rtable.EnsureRomanaRouteRule(*romanaRouteTableId, nlHandle)
	if err != nil {
		log.Errorf("Failed to install route rule for romana routing table, %s", err)
		os.Exit(2)
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	defaultLink, err := agent.GetDefaultLink()
	if err != nil {
		log.Errorf("failed to get default link: %s\n", err)
		os.Exit(3)
	}
	err = agent.StartRomanaIPSync(ctx, romanaClient.Store, defaultLink)
	if err != nil {
		log.Errorf("failed to start romanaIP syncing mechanism: %s\n", err)
		os.Exit(4)
	}

	blocksChannel, err := romanaClient.WatchBlocks(ctx.Done())
	if err != nil {
		log.Errorf("Failed to subscribe to Romana blocks updates, %s", err)
		os.Exit(2)
	}

	if *policyEnforcer {
		ctx := context.Background()
		policyCache := policycache.New()
		var policyEtcdKey = "/romana/policies"
		policies, err := policycontroller.Run(ctx, policyEtcdKey, romanaClient, policyCache)
		if err != nil {
			log.Errorf("Failed to start policy controller, %s", err)
			os.Exit(2)
		}

		blocksList := romanaClient.IPAM.ListAllBlocks()

		// blocks are needed in both, route agent and policy agent
		// this duplicates blocks channel into the 2 new channels, one
		// used here for policies and another one passed down for routes.
		var extraBlocksChannel <-chan api.IPAMBlocksResponse
		blocksChannel, extraBlocksChannel = fanOut(ctx, blocksChannel)

		enforcer, err := enforcer.New(policyCache, policies, *blocksList, extraBlocksChannel, *hostname, new(utilexec.DefaultExecutor), 10)
		if err != nil {
			log.Errorf("Failed to create policy enforcer, %s", err)
			os.Exit(2)
		}

		enforcer.Run(ctx)

	}

	hostsChannel, err := romanaClient.WatchHosts(ctx.Done())
	if err != nil {
		log.Errorf("Failed to start watching for blocks, %s", err)
		os.Exit(2)
	}

	hosts := agent.IpamHosts{}
	for {
		select {
		case blocks := <-blocksChannel:
			startTime := time.Now()
			err := rtable.FlushRomanaTable()
			if err != nil {
				log.Errorf("failed to flush romana route table err=(%s)", err)
				continue
			}

			agent.CreateRouteToBlocks(blocks.Blocks, hosts, *romanaRouteTableId, *hostname, *multihop, nlHandle)
			runTime := time.Now().Sub(startTime)
			log.Tracef(4, "Time between route table flush and route table rebuild %s", runTime)

		case newHosts := <-hostsChannel:
			// TODO need mutex for this.
			hosts = agent.IpamHosts(newHosts.Hosts)
		}
	}
}

// fanOut duplicates data from one channel into 2 identical channels.
func fanOut(ctx context.Context, in <-chan api.IPAMBlocksResponse) (<-chan api.IPAMBlocksResponse, <-chan api.IPAMBlocksResponse) {
	out1 := make(chan api.IPAMBlocksResponse, 1)
	out2 := make(chan api.IPAMBlocksResponse, 1)

	go func() {
		for {
			select {
			case b := <-in:
				log.Trace(5, "Blocks fan out tick")
				out1 <- b
				out2 <- b
			case <-ctx.Done():
				close(out1)
				close(out2)
				return
			}

		}

	}()

	return out1, out2
}

// checkSysctls checks that esseantial sysctl options are set.
func checkSysctls() (ok bool, err error) {
	for _, path := range kernelParameter {
		ok, err = sysctl.Check(path)
		if !ok || err != nil {
			break
		}
	}

	return ok, err
}

// setSysctls sets essential sysctl options.
func setSysctls() (err error) {
	for _, path := range kernelParameter {
		err = sysctl.Set(path)
		if err != nil {
			break
		}
	}
	return err
}
<commit_msg>Wait for list of hosts before trying to create routes<commit_after>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/romana/core/agent"
	"github.com/romana/core/agent/enforcer"
	utilexec "github.com/romana/core/agent/exec"
	"github.com/romana/core/agent/policycache"
	"github.com/romana/core/agent/policycontroller"
	"github.com/romana/core/agent/rtable"
	"github.com/romana/core/agent/sysctl"
	"github.com/romana/core/common"
	"github.com/romana/core/common/api"
	"github.com/romana/core/common/client"

	log "github.com/romana/rlog"
	"github.com/vishvananda/netlink"
)

const (
	DefaultRouteTableId = 10
	DefaultGwIP         = "172.142.0.1"
)

var (
	kernelParameter = []string{
		"/proc/sys/net/ipv4/conf/default/proxy_arp",
		"/proc/sys/net/ipv4/conf/all/proxy_arp",
		"/proc/sys/net/ipv4/ip_forward",
	}
)

func main() {
	var err error

	etcdEndpoints := flag.String("endpoints", "", "csv list of etcd endpoints to romana storage")
	etcdPrefix := flag.String("prefix", "", "string that prefixes all romana keys in etcd")
	hostname := flag.String("hostname", "", "name of the host in romana database")
	provisionIface := flag.Bool("provision-iface", false, "create romana-gw interface and ip")
	provisionIfaceGwIp := flag.String("provision-iface-gw-ip", DefaultGwIP, "specifies ip address for gateway interface")
	provisionSysctls := flag.Bool("provision-sysctls", false, "configure routing sysctls")
	romanaRouteTableId := flag.Int("route-table-id", DefaultRouteTableId,
		"id that romana route table should have in /etc/iproute2/rt_tables")
	multihop := flag.Bool("multihop-blocks", false, "allows multihop blocks")
	policyEnforcer := flag.Bool("policy", false, "enable romana policies")
	metricsPort := flag.Int("metrics", 9607, "tcp port to expose prometheus metrics, -1 means disable")
	flag.Parse()

	fmt.Println(common.BuildInfo())

	if err := agent.MetricStart(*metricsPort); err != nil {
		log.Errorf("Failed to start metrics collector")
		os.Exit(2)
	}

	romanaConfig := common.Config{
		EtcdEndpoints: strings.Split(*etcdEndpoints, ","),
		EtcdPrefix:    *etcdPrefix,
	}

	if *hostname == "" {
		*hostname, err = os.Hostname()
		if err != nil {
			panic(err)
		}
	}

	romanaClient, err := client.NewClient(&romanaConfig)
	if err != nil {
		log.Errorf("Failed to initialize romana client: %v", err)
		os.Exit(2)
	}

	if *provisionIface {
		err := agent.CreateRomanaGW()
		if err != nil {
			log.Errorf("Failed to create romana-gw interface. %s", err)
			os.Exit(2)
		}

		err = agent.SetRomanaGwIP(*provisionIfaceGwIp)
		if err != nil {
			log.Errorf("Failed to install ip address on romana-gw interface. %s", err)
			os.Exit(2)
		}

	}

	if *provisionSysctls {
		err := setSysctls()
		if err != nil {
			log.Errorf("Failed to set sysctls %s", err)
			os.Exit(2)
		}
	}

	ok, err := checkSysctls()
	if err != nil {
		log.Errorf("Failed verify the state of essential systls %s", err)
		os.Exit(2)
	}
	if !ok {
		log.Errorf("Essential sysctls not set, consider using -provision-sysctls flag and ensure you are running from root")
		os.Exit(2)
	}

	err = rtable.EnsureRouteTableExist(*romanaRouteTableId)
	if err != nil {
		log.Errorf("Failed to make `romana` alias for route table=%d, %s. Unable to continue", *romanaRouteTableId, err)
		os.Exit(2)
	}

	nlHandle, err := netlink.NewHandle()
	if err != nil {
		log.Errorf("Failed to create netlink handle %s", err)
		os.Exit(2)
	}
	defer nlHandle.Delete()

	err = rtable.EnsureRomanaRouteRule(*romanaRouteTableId, nlHandle)
	if err != nil {
		log.Errorf("Failed to install route rule for romana routing table, %s", err)
		os.Exit(2)
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	defaultLink, err := agent.GetDefaultLink()
	if err != nil {
		log.Errorf("failed to get default link: %s\n", err)
		os.Exit(3)
	}
	err = agent.StartRomanaIPSync(ctx, romanaClient.Store, defaultLink)
	if err != nil {
		log.Errorf("failed to start romanaIP syncing mechanism: %s\n", err)
		os.Exit(4)
	}

	blocksChannel, err := romanaClient.WatchBlocks(ctx.Done())
	if err != nil {
		log.Errorf("Failed to subscribe to Romana blocks updates, %s", err)
		os.Exit(2)
	}

	if *policyEnforcer {
		ctx := context.Background()
		policyCache := policycache.New()
		var policyEtcdKey = "/romana/policies"
		policies, err := policycontroller.Run(ctx, policyEtcdKey, romanaClient, policyCache)
		if err != nil {
			log.Errorf("Failed to start policy controller, %s", err)
			os.Exit(2)
		}

		blocksList := romanaClient.IPAM.ListAllBlocks()

		// blocks are needed in both, route agent and policy agent
		// this duplicates blocks channel into the 2 new channels, one
		// used here for policies and another one passed down for routes.
		var extraBlocksChannel <-chan api.IPAMBlocksResponse
		blocksChannel, extraBlocksChannel = fanOut(ctx, blocksChannel)

		enforcer, err := enforcer.New(policyCache, policies, *blocksList, extraBlocksChannel, *hostname, new(utilexec.DefaultExecutor), 10)
		if err != nil {
			log.Errorf("Failed to create policy enforcer, %s", err)
			os.Exit(2)
		}

		enforcer.Run(ctx)

	}

	hostsChannel, err := romanaClient.WatchHosts(ctx.Done())
	if err != nil {
		log.Errorf("Failed to start watching for blocks, %s", err)
		os.Exit(2)
	}

	// wait for the first list of hosts to be received
	initialHosts := <-hostsChannel
	hosts := agent.IpamHosts(initialHosts.Hosts)

	for {
		select {
		case blocks := <-blocksChannel:
			startTime := time.Now()
			err := rtable.FlushRomanaTable()
			if err != nil {
				log.Errorf("failed to flush romana route table err=(%s)", err)
				continue
			}

			agent.CreateRouteToBlocks(blocks.Blocks, hosts, *romanaRouteTableId, *hostname, *multihop, nlHandle)
			runTime := time.Now().Sub(startTime)
			log.Tracef(4, "Time between route table flush and route table rebuild %s", runTime)

		case newHosts := <-hostsChannel:
			// TODO need mutex for this.
			hosts = agent.IpamHosts(newHosts.Hosts)
		}
	}
}

// fanOut duplicates data from one channel into 2 identical channels.
func fanOut(ctx context.Context, in <-chan api.IPAMBlocksResponse) (<-chan api.IPAMBlocksResponse, <-chan api.IPAMBlocksResponse) {
	out1 := make(chan api.IPAMBlocksResponse, 1)
	out2 := make(chan api.IPAMBlocksResponse, 1)

	go func() {
		for {
			select {
			case b := <-in:
				log.Trace(5, "Blocks fan out tick")
				out1 <- b
				out2 <- b
			case <-ctx.Done():
				close(out1)
				close(out2)
				return
			}

		}

	}()

	return out1, out2
}

// checkSysctls checks that esseantial sysctl options are set.
func checkSysctls() (ok bool, err error) {
	for _, path := range kernelParameter {
		ok, err = sysctl.Check(path)
		if !ok || err != nil {
			break
		}
	}

	return ok, err
}

// setSysctls sets essential sysctl options.
func setSysctls() (err error) {
	for _, path := range kernelParameter {
		err = sysctl.Set(path)
		if err != nil {
			break
		}
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/influxdata/telegraf/agent"
	"github.com/influxdata/telegraf/internal/config"
	_ "github.com/influxdata/telegraf/plugins/inputs/all"
	_ "github.com/influxdata/telegraf/plugins/outputs/all"
)

var fDebug = flag.Bool("debug", false,
	"show metrics as they're generated to stdout")
var fQuiet = flag.Bool("quiet", false,
	"run in quiet mode")
var fTest = flag.Bool("test", false, "gather metrics, print them out, and exit")
var fConfig = flag.String("config", "", "configuration file to load")
var fConfigDirectory = flag.String("config-directory", "",
	"directory containing additional *.conf files")
var fVersion = flag.Bool("version", false, "display the version")
var fSampleConfig = flag.Bool("sample-config", false,
	"print out full sample configuration")
var fPidfile = flag.String("pidfile", "", "file to write our pid to")
var fInputFilters = flag.String("input-filter", "",
	"filter the inputs to enable, separator is :")
var fOutputFilters = flag.String("output-filter", "",
	"filter the outputs to enable, separator is :")
var fUsage = flag.String("usage", "",
	"print usage for a plugin, ie, 'telegraf -usage mysql'")

var fInputFiltersLegacy = flag.String("filter", "",
	"filter the inputs to enable, separator is :")
var fOutputFiltersLegacy = flag.String("outputfilter", "",
	"filter the outputs to enable, separator is :")
var fConfigDirectoryLegacy = flag.String("configdirectory", "",
	"directory containing additional *.conf files")

// Telegraf version
//	-ldflags "-X main.Version=`git describe --always --tags`"
var Version string

const usage = `Telegraf, The plugin-driven server agent for collecting and reporting metrics.

Usage:

  telegraf <flags>

The flags are:

  -config <file>     configuration file to load
  -test              gather metrics once, print them to stdout, and exit
  -sample-config     print out full sample configuration to stdout
  -config-directory  directory containing additional *.conf files
  -input-filter      filter the input plugins to enable, separator is :
  -output-filter     filter the output plugins to enable, separator is :
  -usage             print usage for a plugin, ie, 'telegraf -usage mysql'
  -debug             print metrics as they're generated to stdout
  -quiet             run in quiet mode
  -version           print the version to stdout

Examples:

  # generate a telegraf config file:
  telegraf -sample-config > telegraf.conf

  # generate config with only cpu input & influxdb output plugins defined
  telegraf -sample-config -input-filter cpu -output-filter influxdb

  # run a single telegraf collection, outputing metrics to stdout
  telegraf -config telegraf.conf -test

  # run telegraf with all plugins defined in config file
  telegraf -config telegraf.conf

  # run telegraf, enabling the cpu & memory input, and influxdb output plugins
  telegraf -config telegraf.conf -input-filter cpu:mem -output-filter influxdb
`

func main() {
	reload := make(chan bool, 1)
	reload <- true
	for <-reload {
		reload <- false
		flag.Usage = usageExit
		flag.Parse()

		if flag.NFlag() == 0 {
			usageExit()
		}

		var inputFilters []string
		if *fInputFiltersLegacy != "" {
			inputFilter := strings.TrimSpace(*fInputFiltersLegacy)
			inputFilters = strings.Split(":"+inputFilter+":", ":")
		}
		if *fInputFilters != "" {
			inputFilter := strings.TrimSpace(*fInputFilters)
			inputFilters = strings.Split(":"+inputFilter+":", ":")
		}

		var outputFilters []string
		if *fOutputFiltersLegacy != "" {
			outputFilter := strings.TrimSpace(*fOutputFiltersLegacy)
			outputFilters = strings.Split(":"+outputFilter+":", ":")
		}
		if *fOutputFilters != "" {
			outputFilter := strings.TrimSpace(*fOutputFilters)
			outputFilters = strings.Split(":"+outputFilter+":", ":")
		}

		if *fVersion {
			v := fmt.Sprintf("Telegraf - Version %s", Version)
			fmt.Println(v)
			return
		}

		if *fSampleConfig {
			config.PrintSampleConfig(inputFilters, outputFilters)
			return
		}

		if *fUsage != "" {
			if err := config.PrintInputConfig(*fUsage); err != nil {
				if err2 := config.PrintOutputConfig(*fUsage); err2 != nil {
					log.Fatalf("%s and %s", err, err2)
				}
			}
			return
		}

		var (
			c   *config.Config
			err error
		)

		if *fConfig != "" {
			c = config.NewConfig()
			c.OutputFilters = outputFilters
			c.InputFilters = inputFilters
			err = c.LoadConfig(*fConfig)
			if err != nil {
				log.Fatal(err)
			}
		} else {
			fmt.Println("Usage: Telegraf")
			flag.PrintDefaults()
			return
		}

		if *fConfigDirectoryLegacy != "" {
			err = c.LoadDirectory(*fConfigDirectoryLegacy)
			if err != nil {
				log.Fatal(err)
			}
		}

		if *fConfigDirectory != "" {
			err = c.LoadDirectory(*fConfigDirectory)
			if err != nil {
				log.Fatal(err)
			}
		}
		if len(c.Outputs) == 0 {
			log.Fatalf("Error: no outputs found, did you provide a valid config file?")
		}
		if len(c.Inputs) == 0 {
			log.Fatalf("Error: no inputs found, did you provide a valid config file?")
		}

		ag, err := agent.NewAgent(c)
		if err != nil {
			log.Fatal(err)
		}

		if *fDebug {
			ag.Config.Agent.Debug = true
		}

		if *fQuiet {
			ag.Config.Agent.Quiet = true
		}

		if *fTest {
			err = ag.Test()
			if err != nil {
				log.Fatal(err)
			}
			return
		}

		err = ag.Connect()
		if err != nil {
			log.Fatal(err)
		}

		shutdown := make(chan struct{})
		signals := make(chan os.Signal)
		signal.Notify(signals, os.Interrupt, syscall.SIGHUP)
		go func() {
			sig := <-signals
			if sig == os.Interrupt {
				close(shutdown)
			}
			if sig == syscall.SIGHUP {
				log.Printf("Reloading Telegraf config\n")
				<-reload
				reload <- true
				close(shutdown)
			}
		}()

		log.Printf("Starting Telegraf (version %s)\n", Version)
		log.Printf("Loaded outputs: %s", strings.Join(c.OutputNames(), " "))
		log.Printf("Loaded inputs: %s", strings.Join(c.InputNames(), " "))
		log.Printf("Tags enabled: %s", c.ListTags())

		if *fPidfile != "" {
			f, err := os.Create(*fPidfile)
			if err != nil {
				log.Fatalf("Unable to create pidfile: %s", err)
			}

			fmt.Fprintf(f, "%d\n", os.Getpid())

			f.Close()
		}

		ag.Run(shutdown)
	}
}

func usageExit() {
	fmt.Println(usage)
	os.Exit(0)
}
<commit_msg>Minor change to forgotten config file exit<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/influxdata/telegraf/agent"
	"github.com/influxdata/telegraf/internal/config"
	_ "github.com/influxdata/telegraf/plugins/inputs/all"
	_ "github.com/influxdata/telegraf/plugins/outputs/all"
)

var fDebug = flag.Bool("debug", false,
	"show metrics as they're generated to stdout")
var fQuiet = flag.Bool("quiet", false,
	"run in quiet mode")
var fTest = flag.Bool("test", false, "gather metrics, print them out, and exit")
var fConfig = flag.String("config", "", "configuration file to load")
var fConfigDirectory = flag.String("config-directory", "",
	"directory containing additional *.conf files")
var fVersion = flag.Bool("version", false, "display the version")
var fSampleConfig = flag.Bool("sample-config", false,
	"print out full sample configuration")
var fPidfile = flag.String("pidfile", "", "file to write our pid to")
var fInputFilters = flag.String("input-filter", "",
	"filter the inputs to enable, separator is :")
var fOutputFilters = flag.String("output-filter", "",
	"filter the outputs to enable, separator is :")
var fUsage = flag.String("usage", "",
	"print usage for a plugin, ie, 'telegraf -usage mysql'")

var fInputFiltersLegacy = flag.String("filter", "",
	"filter the inputs to enable, separator is :")
var fOutputFiltersLegacy = flag.String("outputfilter", "",
	"filter the outputs to enable, separator is :")
var fConfigDirectoryLegacy = flag.String("configdirectory", "",
	"directory containing additional *.conf files")

// Telegraf version
//	-ldflags "-X main.Version=`git describe --always --tags`"
var Version string

const usage = `Telegraf, The plugin-driven server agent for collecting and reporting metrics.

Usage:

  telegraf <flags>

The flags are:

  -config <file>     configuration file to load
  -test              gather metrics once, print them to stdout, and exit
  -sample-config     print out full sample configuration to stdout
  -config-directory  directory containing additional *.conf files
  -input-filter      filter the input plugins to enable, separator is :
  -output-filter     filter the output plugins to enable, separator is :
  -usage             print usage for a plugin, ie, 'telegraf -usage mysql'
  -debug             print metrics as they're generated to stdout
  -quiet             run in quiet mode
  -version           print the version to stdout

Examples:

  # generate a telegraf config file:
  telegraf -sample-config > telegraf.conf

  # generate config with only cpu input & influxdb output plugins defined
  telegraf -sample-config -input-filter cpu -output-filter influxdb

  # run a single telegraf collection, outputing metrics to stdout
  telegraf -config telegraf.conf -test

  # run telegraf with all plugins defined in config file
  telegraf -config telegraf.conf

  # run telegraf, enabling the cpu & memory input, and influxdb output plugins
  telegraf -config telegraf.conf -input-filter cpu:mem -output-filter influxdb
`

func main() {
	reload := make(chan bool, 1)
	reload <- true
	for <-reload {
		reload <- false
		flag.Usage = func() { usageExit(0) }
		flag.Parse()

		if flag.NFlag() == 0 {
			usageExit(0)
		}

		var inputFilters []string
		if *fInputFiltersLegacy != "" {
			inputFilter := strings.TrimSpace(*fInputFiltersLegacy)
			inputFilters = strings.Split(":"+inputFilter+":", ":")
		}
		if *fInputFilters != "" {
			inputFilter := strings.TrimSpace(*fInputFilters)
			inputFilters = strings.Split(":"+inputFilter+":", ":")
		}

		var outputFilters []string
		if *fOutputFiltersLegacy != "" {
			outputFilter := strings.TrimSpace(*fOutputFiltersLegacy)
			outputFilters = strings.Split(":"+outputFilter+":", ":")
		}
		if *fOutputFilters != "" {
			outputFilter := strings.TrimSpace(*fOutputFilters)
			outputFilters = strings.Split(":"+outputFilter+":", ":")
		}

		if *fVersion {
			v := fmt.Sprintf("Telegraf - Version %s", Version)
			fmt.Println(v)
			return
		}

		if *fSampleConfig {
			config.PrintSampleConfig(inputFilters, outputFilters)
			return
		}

		if *fUsage != "" {
			if err := config.PrintInputConfig(*fUsage); err != nil {
				if err2 := config.PrintOutputConfig(*fUsage); err2 != nil {
					log.Fatalf("%s and %s", err, err2)
				}
			}
			return
		}

		var (
			c   *config.Config
			err error
		)

		if *fConfig != "" {
			c = config.NewConfig()
			c.OutputFilters = outputFilters
			c.InputFilters = inputFilters
			err = c.LoadConfig(*fConfig)
			if err != nil {
				log.Fatal(err)
			}
		} else {
			fmt.Println("You must specify a config file. See telegraf --help")
			os.Exit(1)
		}

		if *fConfigDirectoryLegacy != "" {
			err = c.LoadDirectory(*fConfigDirectoryLegacy)
			if err != nil {
				log.Fatal(err)
			}
		}

		if *fConfigDirectory != "" {
			err = c.LoadDirectory(*fConfigDirectory)
			if err != nil {
				log.Fatal(err)
			}
		}
		if len(c.Outputs) == 0 {
			log.Fatalf("Error: no outputs found, did you provide a valid config file?")
		}
		if len(c.Inputs) == 0 {
			log.Fatalf("Error: no inputs found, did you provide a valid config file?")
		}

		ag, err := agent.NewAgent(c)
		if err != nil {
			log.Fatal(err)
		}

		if *fDebug {
			ag.Config.Agent.Debug = true
		}

		if *fQuiet {
			ag.Config.Agent.Quiet = true
		}

		if *fTest {
			err = ag.Test()
			if err != nil {
				log.Fatal(err)
			}
			return
		}

		err = ag.Connect()
		if err != nil {
			log.Fatal(err)
		}

		shutdown := make(chan struct{})
		signals := make(chan os.Signal)
		signal.Notify(signals, os.Interrupt, syscall.SIGHUP)
		go func() {
			sig := <-signals
			if sig == os.Interrupt {
				close(shutdown)
			}
			if sig == syscall.SIGHUP {
				log.Printf("Reloading Telegraf config\n")
				<-reload
				reload <- true
				close(shutdown)
			}
		}()

		log.Printf("Starting Telegraf (version %s)\n", Version)
		log.Printf("Loaded outputs: %s", strings.Join(c.OutputNames(), " "))
		log.Printf("Loaded inputs: %s", strings.Join(c.InputNames(), " "))
		log.Printf("Tags enabled: %s", c.ListTags())

		if *fPidfile != "" {
			f, err := os.Create(*fPidfile)
			if err != nil {
				log.Fatalf("Unable to create pidfile: %s", err)
			}

			fmt.Fprintf(f, "%d\n", os.Getpid())

			f.Close()
		}

		ag.Run(shutdown)
	}
}

func usageExit(rc int) {
	fmt.Println(usage)
	os.Exit(rc)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/insomniacslk/dhcp/dhcpv4"
	"github.com/insomniacslk/dhcp/dhcpv4/client4"
	"github.com/insomniacslk/dhcp/netboot"
	"github.com/u-root/u-root/pkg/boot/stboot"
	"github.com/vishvananda/netlink"
)

var debug = func(string, ...interface{}) {}

var (
	dryRun  = flag.Bool("dryrun", false, "Do everything except booting the loaded kernel")
	doDebug = flag.Bool("d", false, "Print debug output")
)

const (
	eth                = "eth0"
	rootCACertPath     = "/root/LetsEncrypt_Authority_X3.pem"
	entropyAvail       = "/proc/sys/kernel/random/entropy_avail"
	interfaceUpTimeout = 10 * time.Second
)

var banner = `
  _____ _______   _____   ____   ____________
 / ____|__   __|  |  _ \ / __ \ / __ \__   __|
| (___    | |     | |_) | |  | | |  | | | |
 \___ \   | |     |  _ <| |  | | |  | | | |
 ____) |  | |     | |_) | |__| | |__| | | |
|_____/   |_|     |____/ \____/ \____/  |_|

`

var check = `
           //\\
OS is     //  \\
valid    //   //
        //   //
 //\\  //   //
//  \\//   //
\\        //
 \\      //
  \\    //
   \\__//
`

func main() {
	log.SetPrefix("stboot: ")

	flag.Parse()
	if *doDebug {
		debug = log.Printf
	}
	log.Print(banner)

	vars, err := stboot.FindHostVarsInInitramfs()
	if err != nil {
		log.Fatalf("Cant find Netvars at all: %v", err)
	}

	if *doDebug {
		str, _ := json.MarshalIndent(vars, "", "  ")
		log.Printf("Host variables: %s", str)
	}

	debug("Configuring network interfaces")

	if vars.HostIP != "" {
		err = configureStaticNetwork(vars, *doDebug)
	} else {
		err = configureDHCPNetwork()
	}

	if err != nil {
		log.Fatalf("Can not set up IO: %v", err)
	}

	ballPath := path.Join("root/", stboot.BallName)
	url, err := url.Parse(vars.BootstrapURL)
	if err != nil {
		log.Fatalf("Invalid bootstrap URL: %v", err)
	}
	url.Path = path.Join(url.Path, stboot.BallName)
	err = downloadFromHTTPS(url.String(), ballPath)
	if err != nil {
		log.Fatalf("Downloading bootball from %s failed: %v", url, err)
	}

	ball, err := stboot.BootBallFromArchie(ballPath)
	if err != nil {
		log.Fatal("Cannot open bootball")
	}

	// Just choose the first Bootconfig for now
	log.Printf("Pick the first boot configuration")
	var index = 0
	bc, err := ball.GetBootConfigByIndex(index)
	if err != nil {
		log.Fatalf("Cannot get boot configuration %d: %v", index, err)
	}

	if *doDebug {
		str, _ := json.MarshalIndent(*bc, "", "  ")
		log.Printf("Bootconfig: %s", str)
	}

	n, err := ball.VerifyBootconfigByName(bc.Name)
	if err != nil {
		log.Fatalf("Bootconfig %d seems to be not trustworthy: %v", index, err)
	}
	if n < vars.MinimalSignaturesMatch {
		log.Fatalf("Did not found enough valid signatures. %d valid, %d required", n, vars.MinimalSignaturesMatch)
	}
	log.Printf("Bootconfig '%s' passed verification", bc.Name)
	log.Print(check)

	if *dryRun {
		debug("Dryrun mode: will not boot")
		return
	}

	log.Println("Starting up new kernel.")

	if err := bc.Boot(); err != nil {
		log.Printf("Failed to boot kernel %s: %v", bc.Kernel, err)
	}
	// if we reach this point, no boot configuration succeeded
	log.Print("No boot configuration succeeded")

	return
}

func configureStaticNetwork(vars stboot.HostVars, doDebug bool) error {
	log.Printf("Setup network configuration with IP: " + vars.HostIP)
	addr, err := netlink.ParseAddr(vars.HostIP)
	if err != nil {
		return fmt.Errorf("Error parsing HostIP string to CIDR format address: %v", err)
	}

	iface, err := netlink.LinkByName(eth)
	if err != nil {
		return fmt.Errorf("Error retrieving interface %s: %v", eth, err)
	}

	if err = netlink.AddrAdd(iface, addr); err != nil {
		return fmt.Errorf("Error adding address: %v", err)
	}

	if err = netlink.LinkSetUp(iface); err != nil {
		return fmt.Errorf("Error bringing up interface %s: %v", eth, err)
	}

	gateway, err := netlink.ParseAddr(vars.DefaultGateway)
	if err != nil {
		return fmt.Errorf("Error parsing GatewayIP string to CIDR format address: %v", err)
	}

	r := &netlink.Route{LinkIndex: iface.Attrs().Index, Gw: gateway.IPNet.IP}
	if err = netlink.RouteAdd(r); err != nil {
		return fmt.Errorf("Error setting default gateway: %v", err)
	}

	return nil
}

func configureDHCPNetwork() error {

	log.Printf("Trying to configure network configuration dynamically...")
	attempts := 10
	var conversation []*dhcpv4.DHCPv4

	_, err := netboot.IfUp(eth, interfaceUpTimeout)
	if err != nil {
		return fmt.Errorf("Error bringing up interface %s: %v", eth, err)
	}
	if attempts < 1 {
		attempts = 1
	}

	client := client4.NewClient()
	for attempt := 0; attempt < attempts; attempt++ {
		log.Printf("Attempt to get DHCP lease %d of %d for interface %s", attempt+1, attempts, eth)
		conversation, err = client.Exchange(eth)

		if err != nil && attempt < attempts {
			log.Printf("Error: %v", err)
			continue
		}
		break
	}

	if conversation[3] == nil {
		return fmt.Errorf("Gateway is null")
	}

	netbootConfig, err := netboot.GetNetConfFromPacketv4(conversation[3])
	if err != nil {
		return err
	}

	if *doDebug {
		str, _ := json.MarshalIndent(netbootConfig, "", "  ")
		log.Printf("Network configuration: %s", str)
	}

	return netboot.ConfigureInterface(eth, netbootConfig)
}

func downloadFromHTTPS(url string, destination string) error {
	roots := x509.NewCertPool()
	if err := loadHTTPSCertificate(roots); err != nil {
		return fmt.Errorf("Failed to load root certificate: %v", err)
	}

	// setup https client
	client := http.Client{
		Transport: (&http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
				DualStack: true,
			}).DialContext,
			MaxIdleConns:          100,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
			TLSClientConfig: (&tls.Config{
				RootCAs: roots,
			}),
		}),
	}

	// check available kernel entropy
	e, err := ioutil.ReadFile(entropyAvail)
	es := strings.TrimSpace(string(e))
	entr, err := strconv.Atoi(es)
	if err != nil {
		return fmt.Errorf("Cannot evaluate entropy, %v", err)
	}
	log.Printf("Available kernel entropy: %d", entr)
	if entr < 128 {
		log.Print("WARNING: low entropy!")
		log.Printf("%s : %d", entropyAvail, entr)
	}
	// get remote boot bundle
	log.Print("Downloading bootball ...")
	resp, err := client.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("non-200 HTTP status: %d", resp.StatusCode)
	}
	f, err := os.Create(destination)
	if err != nil {
		return fmt.Errorf("failed create boot config file: %v", err)
	}
	defer f.Close()

	_, err = io.Copy(f, resp.Body)
	if err != nil {
		return fmt.Errorf("failed to save bootball: %v", err)
	}

	return nil
}

// loadHTTPSCertificate loads the certificate needed
// for HTTPS and verifies it.
func loadHTTPSCertificate(roots *x509.CertPool) error {
	log.Printf("Load %s as CA certificate", rootCACertPath)
	rootCertBytes, err := ioutil.ReadFile(rootCACertPath)
	if err != nil {
		return err
	}
	rootCertPem, _ := pem.Decode(rootCertBytes)
	if rootCertPem.Type != "CERTIFICATE" {
		return fmt.Errorf("Failed decoding certificate: Certificate is of the wrong type. PEM Type is: %s", rootCertPem.Type)
	}
	ok := roots.AppendCertsFromPEM([]byte(rootCertBytes))
	if !ok {
		return fmt.Errorf("Error parsing CA root certificate")
	}
	return nil
}
<commit_msg>cmds/boot/stboot: switch to pkg/dhclient<commit_after>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/u-root/u-root/pkg/boot/stboot"
	"github.com/u-root/u-root/pkg/dhclient"
	"github.com/vishvananda/netlink"
)

var debug = func(string, ...interface{}) {}

var (
	dryRun  = flag.Bool("dryrun", false, "Do everything except booting the loaded kernel")
	doDebug = flag.Bool("d", false, "Print debug output")
)

const (
	eth                = "eth0"
	rootCACertPath     = "/root/LetsEncrypt_Authority_X3.pem"
	entropyAvail       = "/proc/sys/kernel/random/entropy_avail"
	interfaceUpTimeout = 10 * time.Second
)

var banner = `
  _____ _______   _____   ____   ____________
 / ____|__   __|  |  _ \ / __ \ / __ \__   __|
| (___    | |     | |_) | |  | | |  | | | |
 \___ \   | |     |  _ <| |  | | |  | | | |
 ____) |  | |     | |_) | |__| | |__| | | |
|_____/   |_|     |____/ \____/ \____/  |_|

`

var check = `
           //\\
OS is     //  \\
valid    //   //
        //   //
 //\\  //   //
//  \\//   //
\\        //
 \\      //
  \\    //
   \\__//
`

func main() {
	log.SetPrefix("stboot: ")

	flag.Parse()
	if *doDebug {
		debug = log.Printf
	}
	log.Print(banner)

	vars, err := stboot.FindHostVarsInInitramfs()
	if err != nil {
		log.Fatalf("Cant find Netvars at all: %v", err)
	}

	if *doDebug {
		str, _ := json.MarshalIndent(vars, "", "  ")
		log.Printf("Host variables: %s", str)
	}

	debug("Configuring network interfaces")

	if vars.HostIP != "" {
		err = configureStaticNetwork(vars)
	} else {
		err = configureDHCPNetwork()
	}

	if err != nil {
		log.Fatalf("Can not set up IO: %v", err)
	}

	ballPath := path.Join("root/", stboot.BallName)
	url, err := url.Parse(vars.BootstrapURL)
	if err != nil {
		log.Fatalf("Invalid bootstrap URL: %v", err)
	}
	url.Path = path.Join(url.Path, stboot.BallName)
	err = downloadFromHTTPS(url.String(), ballPath)
	if err != nil {
		log.Fatalf("Downloading bootball failed: %v", err)
	}

	ball, err := stboot.BootBallFromArchie(ballPath)
	if err != nil {
		log.Fatal("Cannot open bootball")
	}

	// Just choose the first Bootconfig for now
	log.Printf("Pick the first boot configuration")
	var index = 0
	bc, err := ball.GetBootConfigByIndex(index)
	if err != nil {
		log.Fatalf("Cannot get boot configuration %d: %v", index, err)
	}

	if *doDebug {
		str, _ := json.MarshalIndent(*bc, "", "  ")
		log.Printf("Bootconfig: %s", str)
	}

	n, err := ball.VerifyBootconfigByName(bc.Name)
	if err != nil {
		log.Fatalf("Bootconfig %d seems to be not trustworthy: %v", index, err)
	}
	if n < vars.MinimalSignaturesMatch {
		log.Fatalf("Did not found enough valid signatures. %d valid, %d required", n, vars.MinimalSignaturesMatch)
	}
	log.Printf("Bootconfig '%s' passed verification", bc.Name)
	log.Print(check)

	if *dryRun {
		debug("Dryrun mode: will not boot")
		return
	}

	log.Println("Starting up new kernel.")

	if err := bc.Boot(); err != nil {
		log.Printf("Failed to boot kernel %s: %v", bc.Kernel, err)
	}
	// if we reach this point, no boot configuration succeeded
	log.Print("No boot configuration succeeded")

	return
}

func configureStaticNetwork(vars stboot.HostVars) error {
	log.Printf("Setup network configuration with IP: " + vars.HostIP)
	addr, err := netlink.ParseAddr(vars.HostIP)
	if err != nil {
		return fmt.Errorf("Error parsing HostIP string to CIDR format address: %v", err)
	}

	iface, err := netlink.LinkByName(eth)
	if err != nil {
		return fmt.Errorf("Error retrieving interface %s: %v", eth, err)
	}

	if err = netlink.AddrAdd(iface, addr); err != nil {
		return fmt.Errorf("Error adding address: %v", err)
	}

	if err = netlink.LinkSetUp(iface); err != nil {
		return fmt.Errorf("Error bringing up interface %s: %v", eth, err)
	}

	gateway, err := netlink.ParseAddr(vars.DefaultGateway)
	if err != nil {
		return fmt.Errorf("Error parsing GatewayIP string to CIDR format address: %v", err)
	}

	r := &netlink.Route{LinkIndex: iface.Attrs().Index, Gw: gateway.IPNet.IP}
	if err = netlink.RouteAdd(r); err != nil {
		return fmt.Errorf("Error setting default gateway: %v", err)
	}

	return nil
}

func configureDHCPNetwork() error {
	log.Printf("Trying to configure network configuration dynamically...")

	link, err := dhclient.IfUp(eth)
	if err != nil {
		return err
	}

	var links []netlink.Link
	links = append(links, link)

	var level dhclient.LogLevel
	if *doDebug {
		level = 1
	} else {
		level = 0
	}
	config := dhclient.Config{
		Timeout:  time.Second * 6,
		Retries:  4,
		LogLevel: level,
	}

	r := dhclient.SendRequests(context.TODO(), links, true, false, config)
	for result := range r {
		if result.Interface == link {
			if result.Err == nil {
				result.Lease.Configure()
				return nil
			} else if *doDebug {
				log.Printf("dhcp response error: %v", err)
			}
		}
	}
	return errors.New("no valid DHCP configuration recieved")
}

func downloadFromHTTPS(url string, destination string) error {
	roots := x509.NewCertPool()
	if err := loadHTTPSCertificate(roots); err != nil {
		return fmt.Errorf("Failed to load root certificate: %v", err)
	}

	// setup https client
	client := http.Client{
		Transport: (&http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
				DualStack: true,
			}).DialContext,
			MaxIdleConns:          100,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
			TLSClientConfig: (&tls.Config{
				RootCAs: roots,
			}),
		}),
	}

	// check available kernel entropy
	e, err := ioutil.ReadFile(entropyAvail)
	es := strings.TrimSpace(string(e))
	entr, err := strconv.Atoi(es)
	if err != nil {
		return fmt.Errorf("Cannot evaluate entropy, %v", err)
	}
	log.Printf("Available kernel entropy: %d", entr)
	if entr < 128 {
		log.Print("WARNING: low entropy!")
		log.Printf("%s : %d", entropyAvail, entr)
	}
	// get remote boot bundle
	log.Print("Downloading bootball ...")
	resp, err := client.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("non-200 HTTP status: %d", resp.StatusCode)
	}
	f, err := os.Create(destination)
	if err != nil {
		return fmt.Errorf("failed create boot config file: %v", err)
	}
	defer f.Close()

	_, err = io.Copy(f, resp.Body)
	if err != nil {
		return fmt.Errorf("failed to save bootball: %v", err)
	}

	return nil
}

// loadHTTPSCertificate loads the certificate needed
// for HTTPS and verifies it.
func loadHTTPSCertificate(roots *x509.CertPool) error {
	log.Printf("Load %s as CA certificate", rootCACertPath)
	rootCertBytes, err := ioutil.ReadFile(rootCACertPath)
	if err != nil {
		return err
	}
	rootCertPem, _ := pem.Decode(rootCertBytes)
	if rootCertPem.Type != "CERTIFICATE" {
		return fmt.Errorf("Failed decoding certificate: Certificate is of the wrong type. PEM Type is: %s", rootCertPem.Type)
	}
	ok := roots.AppendCertsFromPEM([]byte(rootCertBytes))
	if !ok {
		return fmt.Errorf("Error parsing CA root certificate")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"container/ring"
	"io"
	"log"
	"net"
	"net/textproto"
	"sort"
	"strconv"
	"strings"

	"github.com/dustin/go-nntp"
	"github.com/dustin/go-nntp/server"
)

const maxArticles = 100

type articleRef struct {
	msgid string
	num   int64
}

type groupStorage struct {
	group *nntp.Group
	// article refs
	articles *ring.Ring
}

type articleStorage struct {
	headers  textproto.MIMEHeader
	body     string
	refcount int
}

type testBackendType struct {
	// group name -> group storage
	groups map[string]*groupStorage
	// message ID -> article
	articles map[string]*articleStorage
}

var testBackend = testBackendType{
	groups:   make(map[string]*groupStorage),
	articles: make(map[string]*articleStorage),
}

func init() {

	testBackend.groups["alt.test"] = &groupStorage{
		group: &nntp.Group{"alt.test", "A test.",
			0, 0, 0, nntp.PostingNotPermitted},
		articles: ring.New(maxArticles),
	}

	testBackend.groups["misc.test"] = &groupStorage{
		group: &nntp.Group{"misc.test", "More testing.",
			0, 0, 0, nntp.PostingPermitted},
		articles: ring.New(maxArticles),
	}

}

func (tb *testBackendType) ListGroups(max int) ([]*nntp.Group, error) {
	rv := make([]*nntp.Group, 0, 100)
	for _, g := range tb.groups {
		rv = append(rv, g.group)
	}
	return rv, nil
}

func (tb *testBackendType) GetGroup(name string) (*nntp.Group, error) {
	var group *nntp.Group

	for _, g := range tb.groups {
		if g.group.Name == name {
			group = g.group
			break
		}
	}

	if group == nil {
		return nil, nntpserver.NoSuchGroup
	}

	return group, nil
}

func mkArticle(a *articleStorage) *nntp.Article {
	return &nntp.Article{
		Header: a.headers,
		Body:   strings.NewReader(a.body),
		Bytes:  len(a.body),
		Lines:  strings.Count(a.body, "\n"),
	}
}

func findInRing(in *ring.Ring, f func(r interface{}) bool) *ring.Ring {
	if f(in.Value) {
		return in
	}
	for p := in.Next(); p != in; p = p.Next() {
		if f(p.Value) {
			return p
		}
	}
	return nil
}

func (tb *testBackendType) GetArticle(group *nntp.Group, id string) (*nntp.Article, error) {

	msgId := id
	var a *articleStorage

	if intid, err := strconv.ParseInt(id, 10, 64); err == nil {
		msgId = ""
		// by int ID.  Gotta go find it.
		if groupStorage, ok := tb.groups[group.Name]; ok {
			r := findInRing(groupStorage.articles, func(v interface{}) bool {
				if v != nil {
					log.Printf("Looking at %v", v)
				}
				if aref, ok := v.(articleRef); ok && aref.num == intid {
					return true
				}
				return false
			})
			if aref, ok := r.Value.(articleRef); ok {
				msgId = aref.msgid
			}
		}
	}

	a = tb.articles[msgId]
	if a == nil {
		return nil, nntpserver.InvalidMessageId
	}

	return mkArticle(a), nil
}

// Because I suck at ring, I'm going to just post-sort these.
type nalist []nntpserver.NumberedArticle

func (n nalist) Len() int {
	return len(n)
}

func (n nalist) Less(i, j int) bool {
	return n[i].Num < n[j].Num
}

func (n nalist) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

func (tb *testBackendType) GetArticles(group *nntp.Group,
	from, to int64) ([]nntpserver.NumberedArticle, error) {

	gs, ok := tb.groups[group.Name]
	if !ok {
		return nil, nntpserver.NoSuchGroup
	}

	log.Printf("Getting articles from %d to %d", from, to)

	rv := make([]nntpserver.NumberedArticle, 0, maxArticles)
	gs.articles.Do(func(v interface{}) {
		if v != nil {
			if aref, ok := v.(articleRef); ok {
				if aref.num >= from && aref.num <= to {
					a, ok := tb.articles[aref.msgid]
					if ok {
						article := mkArticle(a)
						rv = append(rv, nntpserver.NumberedArticle{aref.num, article})
					}
				}
			}
		}
	})

	sort.Sort(nalist(rv))

	return rv, nil
}

func (tb *testBackendType) AllowPost() bool {
	return true
}

func (tb *testBackendType) decr(msgid string) {
	if a, ok := tb.articles[msgid]; ok {
		a.refcount--
		if a.refcount == 0 {
			log.Printf("Getting rid of %v", msgid)
			delete(tb.articles, msgid)
		}
	}
}

func (tb *testBackendType) Post(article *nntp.Article) error {
	log.Printf("Got headers: %#v", article.Header)
	b := []byte{}
	buf := bytes.NewBuffer(b)
	n, err := io.Copy(buf, article.Body)
	if err != nil {
		return err
	}
	log.Printf("Read %d bytes of body", n)

	a := articleStorage{
		headers:  article.Header,
		body:     buf.String(),
		refcount: 0,
	}

	msgId := a.headers.Get("Message-Id")

	if _, ok := tb.articles[msgId]; ok {
		return nntpserver.PostingFailed
	}

	for _, g := range article.Header["Newsgroups"] {
		if g, ok := tb.groups[g]; ok {
			g.articles = g.articles.Next()
			if g.articles.Value != nil {
				aref := g.articles.Value.(articleRef)
				tb.decr(aref.msgid)
			}
			if g.articles.Value != nil || g.group.Low == 0 {
				g.group.Low++
			}
			g.group.High++
			g.articles.Value = articleRef{
				msgId,
				g.group.High,
			}
			log.Printf("Placed %v", g.articles.Value)
			a.refcount++
			g.group.Count = int64(g.articles.Len())

			log.Printf("Stored %v in %v", msgId, g.group.Name)
		}
	}

	if a.refcount > 0 {
		tb.articles[msgId] = &a
	} else {
		return nntpserver.PostingFailed
	}

	return nil
}

func (tb *testBackendType) Authorized() bool {
	return true
}

func (tb *testBackendType) Authenticate(user, pass string) (nntpserver.Backend, error) {
	return nil, nntpserver.AuthRejected
}

func maybefatal(err error, f string, a ...interface{}) {
	if err != nil {
		log.Fatalf(f, a...)
	}
}

func main() {
	a, err := net.ResolveTCPAddr("tcp", ":1119")
	maybefatal(err, "Error resolving listener: %v", err)
	l, err := net.ListenTCP("tcp", a)
	maybefatal(err, "Error setting up listener: %v", err)
	defer l.Close()

	s := nntpserver.NewServer(&testBackend)

	for {
		c, err := l.AcceptTCP()
		maybefatal(err, "Error accepting connection: %v", err)
		go s.Process(c)
	}
}
<commit_msg>Build fix<commit_after>package main

import (
	"bytes"
	"container/ring"
	"io"
	"log"
	"net"
	"net/textproto"
	"sort"
	"strconv"
	"strings"

	"github.com/dustin/go-nntp"
	"github.com/dustin/go-nntp/server"
)

const maxArticles = 100

type articleRef struct {
	msgid string
	num   int64
}

type groupStorage struct {
	group *nntp.Group
	// article refs
	articles *ring.Ring
}

type articleStorage struct {
	headers  textproto.MIMEHeader
	body     string
	refcount int
}

type testBackendType struct {
	// group name -> group storage
	groups map[string]*groupStorage
	// message ID -> article
	articles map[string]*articleStorage
}

var testBackend = testBackendType{
	groups:   make(map[string]*groupStorage),
	articles: make(map[string]*articleStorage),
}

func init() {

	testBackend.groups["alt.test"] = &groupStorage{
		group: &nntp.Group{
			Name:        "alt.test",
			Description: "A test.",
			Posting:     nntp.PostingNotPermitted},
		articles: ring.New(maxArticles),
	}

	testBackend.groups["misc.test"] = &groupStorage{
		group: &nntp.Group{
			Name:        "misc.test",
			Description: "More testing.",
			Posting:     nntp.PostingPermitted},
		articles: ring.New(maxArticles),
	}

}

func (tb *testBackendType) ListGroups(max int) ([]*nntp.Group, error) {
	rv := make([]*nntp.Group, 0, 100)
	for _, g := range tb.groups {
		rv = append(rv, g.group)
	}
	return rv, nil
}

func (tb *testBackendType) GetGroup(name string) (*nntp.Group, error) {
	var group *nntp.Group

	for _, g := range tb.groups {
		if g.group.Name == name {
			group = g.group
			break
		}
	}

	if group == nil {
		return nil, nntpserver.NoSuchGroup
	}

	return group, nil
}

func mkArticle(a *articleStorage) *nntp.Article {
	return &nntp.Article{
		Header: a.headers,
		Body:   strings.NewReader(a.body),
		Bytes:  len(a.body),
		Lines:  strings.Count(a.body, "\n"),
	}
}

func findInRing(in *ring.Ring, f func(r interface{}) bool) *ring.Ring {
	if f(in.Value) {
		return in
	}
	for p := in.Next(); p != in; p = p.Next() {
		if f(p.Value) {
			return p
		}
	}
	return nil
}

func (tb *testBackendType) GetArticle(group *nntp.Group, id string) (*nntp.Article, error) {

	msgId := id
	var a *articleStorage

	if intid, err := strconv.ParseInt(id, 10, 64); err == nil {
		msgId = ""
		// by int ID.  Gotta go find it.
		if groupStorage, ok := tb.groups[group.Name]; ok {
			r := findInRing(groupStorage.articles, func(v interface{}) bool {
				if v != nil {
					log.Printf("Looking at %v", v)
				}
				if aref, ok := v.(articleRef); ok && aref.num == intid {
					return true
				}
				return false
			})
			if aref, ok := r.Value.(articleRef); ok {
				msgId = aref.msgid
			}
		}
	}

	a = tb.articles[msgId]
	if a == nil {
		return nil, nntpserver.InvalidMessageId
	}

	return mkArticle(a), nil
}

// Because I suck at ring, I'm going to just post-sort these.
type nalist []nntpserver.NumberedArticle

func (n nalist) Len() int {
	return len(n)
}

func (n nalist) Less(i, j int) bool {
	return n[i].Num < n[j].Num
}

func (n nalist) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

func (tb *testBackendType) GetArticles(group *nntp.Group,
	from, to int64) ([]nntpserver.NumberedArticle, error) {

	gs, ok := tb.groups[group.Name]
	if !ok {
		return nil, nntpserver.NoSuchGroup
	}

	log.Printf("Getting articles from %d to %d", from, to)

	rv := make([]nntpserver.NumberedArticle, 0, maxArticles)
	gs.articles.Do(func(v interface{}) {
		if v != nil {
			if aref, ok := v.(articleRef); ok {
				if aref.num >= from && aref.num <= to {
					a, ok := tb.articles[aref.msgid]
					if ok {
						article := mkArticle(a)
						rv = append(rv,
							nntpserver.NumberedArticle{
								Num:     aref.num,
								Article: article})
					}
				}
			}
		}
	})

	sort.Sort(nalist(rv))

	return rv, nil
}

func (tb *testBackendType) AllowPost() bool {
	return true
}

func (tb *testBackendType) decr(msgid string) {
	if a, ok := tb.articles[msgid]; ok {
		a.refcount--
		if a.refcount == 0 {
			log.Printf("Getting rid of %v", msgid)
			delete(tb.articles, msgid)
		}
	}
}

func (tb *testBackendType) Post(article *nntp.Article) error {
	log.Printf("Got headers: %#v", article.Header)
	b := []byte{}
	buf := bytes.NewBuffer(b)
	n, err := io.Copy(buf, article.Body)
	if err != nil {
		return err
	}
	log.Printf("Read %d bytes of body", n)

	a := articleStorage{
		headers:  article.Header,
		body:     buf.String(),
		refcount: 0,
	}

	msgId := a.headers.Get("Message-Id")

	if _, ok := tb.articles[msgId]; ok {
		return nntpserver.PostingFailed
	}

	for _, g := range article.Header["Newsgroups"] {
		if g, ok := tb.groups[g]; ok {
			g.articles = g.articles.Next()
			if g.articles.Value != nil {
				aref := g.articles.Value.(articleRef)
				tb.decr(aref.msgid)
			}
			if g.articles.Value != nil || g.group.Low == 0 {
				g.group.Low++
			}
			g.group.High++
			g.articles.Value = articleRef{
				msgId,
				g.group.High,
			}
			log.Printf("Placed %v", g.articles.Value)
			a.refcount++
			g.group.Count = int64(g.articles.Len())

			log.Printf("Stored %v in %v", msgId, g.group.Name)
		}
	}

	if a.refcount > 0 {
		tb.articles[msgId] = &a
	} else {
		return nntpserver.PostingFailed
	}

	return nil
}

func (tb *testBackendType) Authorized() bool {
	return true
}

func (tb *testBackendType) Authenticate(user, pass string) (nntpserver.Backend, error) {
	return nil, nntpserver.AuthRejected
}

func maybefatal(err error, f string, a ...interface{}) {
	if err != nil {
		log.Fatalf(f, a...)
	}
}

func main() {
	a, err := net.ResolveTCPAddr("tcp", ":1119")
	maybefatal(err, "Error resolving listener: %v", err)
	l, err := net.ListenTCP("tcp", a)
	maybefatal(err, "Error setting up listener: %v", err)
	defer l.Close()

	s := nntpserver.NewServer(&testBackend)

	for {
		c, err := l.AcceptTCP()
		maybefatal(err, "Error accepting connection: %v", err)
		go s.Process(c)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"time"

	"./terminal"
)

const VERSION = "0.0.1"

const (
	CTRL_Q      = 0x11
	ARROW_LEFT  = 1000
	ARROW_RIGHT = 1001
	ARROW_UP    = 1002
	ARROW_DOWN  = 1003
	PAGE_UP     = 1004
	PAGE_DOWN   = 1005
	HOME        = 1006
	END         = 1007
	DELETE      = 1008
)

type Editor struct {
	Reader *bufio.Reader

	ScreenRows int
	ScreenCols int

	CursorRow int
	CursorCol int

	Message string
}

func NewEditor() *Editor {
	editor := &Editor{}
	editor.Reader = bufio.NewReader(os.Stdin)
	w, h, err := terminal.GetSize(0)
	if err != nil {
		fmt.Print(err)
	}
	editor.ScreenRows = h
	editor.ScreenCols = w
	return editor
}

func (e *Editor) ReadKey() int {
	var err error
	b := make([]byte, 10)
	n := 0
	for n == 0 {
		n, _ = os.Stdin.Read(b)
		if n == 0 {
			time.Sleep(time.Microsecond)
		}
	}
	e.Message = fmt.Sprintf(" code=%02x", b[0:n])
	if err != nil {
		fmt.Print(err)
	}
	switch b[0] {
	case 0x1b:
		switch b[1] {
		case 0x5b:
			switch b[2] {
			case 'A':
				return ARROW_UP
			case 'B':
				return ARROW_DOWN
			case 'C':
				return ARROW_RIGHT
			case 'D':
				return ARROW_LEFT
			case 0x31:
				switch b[3] {
				case 0x7e:
					return HOME
				}
			case 0x33:
				switch b[3] {
				case 0x7e:
					return DELETE
				}
			case 0x34:
				switch b[3] {
				case 0x7e:
					return END
				}
			case 0x35:
				switch b[3] {
				case 0x7e:
					return PAGE_UP
				}
			case 0x36:
				switch b[3] {
				case 0x7e:
					return PAGE_DOWN
				}
			}
		}
	}
	return int(b[0])
}

func (e *Editor) ProcessKeyPress() error {
	key := e.ReadKey()
	e.Message += fmt.Sprintf(" key=%d", key)

	switch key {
	case CTRL_Q:
		e.Exit()
		return errors.New("quit")
	case ARROW_UP, ARROW_DOWN, ARROW_LEFT, ARROW_RIGHT:
		e.MoveCursor(key)
	}
	return nil
}

func (e *Editor) RefreshScreen() {
	buffer := make([]byte, 0)
	buffer = append(buffer, []byte("\x1b[?25l")...) // hide cursor
	buffer = append(buffer, []byte("\x1b[1;1H")...) // move cursor to row 1, col 1
	buffer = e.DrawRows(buffer)

	buffer = append(buffer, []byte(fmt.Sprintf("\x1b[%d;%dH", e.CursorRow+1, e.CursorCol+1))...)

	buffer = append(buffer, []byte("\x1b[?25h")...) // show cursor
	os.Stdout.Write(buffer)
}

func (e *Editor) Exit() {
	buffer := make([]byte, 0)
	buffer = append(buffer, []byte("\x1b[2J")...)   // clear screen
	buffer = append(buffer, []byte("\x1b[1;1H")...) // move cursor to row 1, col 1
	os.Stdout.Write(buffer)
}

func (e *Editor) DrawRows(buffer []byte) []byte {
	for y := 1; y <= e.ScreenRows; y++ {
		if y == e.ScreenRows/3 {
			welcome := fmt.Sprintf("goed editor -- version %s", VERSION)
			padding := (e.ScreenCols - len(welcome)) / 2
			buffer = append(buffer, []byte("~")...)
			for i := 1; i <= padding; i++ {
				buffer = append(buffer, []byte(" ")...)
			}
			buffer = append(buffer, []byte(welcome)...)
		} else {
			buffer = append(buffer, []byte("~")...)
		}
		buffer = append(buffer, []byte("\x1b[K")...)
		if y < e.ScreenRows {
			buffer = append(buffer, []byte("\r\n")...)
		} else {
			buffer = append(buffer, []byte(e.Message)...)
		}
	}
	return buffer
}

func (e *Editor) MoveCursor(key int) {
	switch key {
	case ARROW_LEFT:
		if e.CursorCol > 0 {
			e.CursorCol--
		}
	case ARROW_RIGHT:
		if e.CursorCol < e.ScreenCols-1 {
			e.CursorCol++
		}
	case ARROW_UP:
		if e.CursorRow > 0 {
			e.CursorRow--
		}
	case ARROW_DOWN:
		if e.CursorRow < e.ScreenRows-1 {
			e.CursorRow++
		}
	}
}

func main() {

	// put the terminal into raw mode
	oldState, err := terminal.MakeRaw(0)
	if err != nil {
		panic(err)
	}
	// restore terminal however we exit
	defer terminal.Restore(0, oldState)

	e := NewEditor()
	// input loop
	for {
		e.RefreshScreen()
		err = e.ProcessKeyPress()
		if err != nil {
			break
		}
	}
}
<commit_msg>actions for page up, page down, home, and end<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"time"

	"./terminal"
)

const VERSION = "0.0.1"

const (
	CTRL_Q      = 0x11
	ARROW_LEFT  = 1000
	ARROW_RIGHT = 1001
	ARROW_UP    = 1002
	ARROW_DOWN  = 1003
	PAGE_UP     = 1004
	PAGE_DOWN   = 1005
	HOME        = 1006
	END         = 1007
	DELETE      = 1008
)

type Editor struct {
	Reader *bufio.Reader

	ScreenRows int
	ScreenCols int

	CursorRow int
	CursorCol int

	Message string
}

func NewEditor() *Editor {
	editor := &Editor{}
	editor.Reader = bufio.NewReader(os.Stdin)
	w, h, err := terminal.GetSize(0)
	if err != nil {
		fmt.Print(err)
	}
	editor.ScreenRows = h
	editor.ScreenCols = w
	return editor
}

func (e *Editor) ReadKey() int {
	var err error
	b := make([]byte, 10)
	n := 0
	for n == 0 {
		n, _ = os.Stdin.Read(b)
		if n == 0 {
			time.Sleep(time.Microsecond)
		}
	}
	e.Message = fmt.Sprintf(" code=%02x", b[0:n])
	if err != nil {
		fmt.Print(err)
	}
	switch b[0] {
	case 0x1b:
		switch b[1] {
		case 0x5b:
			switch b[2] {
			case 'A':
				return ARROW_UP
			case 'B':
				return ARROW_DOWN
			case 'C':
				return ARROW_RIGHT
			case 'D':
				return ARROW_LEFT
			case 0x31:
				switch b[3] {
				case 0x7e:
					return HOME
				}
			case 0x33:
				switch b[3] {
				case 0x7e:
					return DELETE
				}
			case 0x34:
				switch b[3] {
				case 0x7e:
					return END
				}
			case 0x35:
				switch b[3] {
				case 0x7e:
					return PAGE_UP
				}
			case 0x36:
				switch b[3] {
				case 0x7e:
					return PAGE_DOWN
				}
			}
		}
	}
	return int(b[0])
}

func (e *Editor) ProcessKeyPress() error {
	key := e.ReadKey()
	e.Message += fmt.Sprintf(" key=%d", key)

	switch key {
	case CTRL_Q:
		e.Exit()
		return errors.New("quit")
	case PAGE_UP:
		for times := e.ScreenRows; times > 0; times-- {
			e.MoveCursor(ARROW_UP)
		}
	case PAGE_DOWN:
		for times := e.ScreenRows; times > 0; times-- {
			e.MoveCursor(ARROW_DOWN)
		}
	case HOME:
		e.CursorCol = 0
	case END:
		e.CursorCol = e.ScreenCols - 1
	case ARROW_UP, ARROW_DOWN, ARROW_LEFT, ARROW_RIGHT:
		e.MoveCursor(key)
	}
	return nil
}

func (e *Editor) RefreshScreen() {
	buffer := make([]byte, 0)
	buffer = append(buffer, []byte("\x1b[?25l")...) // hide cursor
	buffer = append(buffer, []byte("\x1b[1;1H")...) // move cursor to row 1, col 1
	buffer = e.DrawRows(buffer)

	buffer = append(buffer, []byte(fmt.Sprintf("\x1b[%d;%dH", e.CursorRow+1, e.CursorCol+1))...)

	buffer = append(buffer, []byte("\x1b[?25h")...) // show cursor
	os.Stdout.Write(buffer)
}

func (e *Editor) Exit() {
	buffer := make([]byte, 0)
	buffer = append(buffer, []byte("\x1b[2J")...)   // clear screen
	buffer = append(buffer, []byte("\x1b[1;1H")...) // move cursor to row 1, col 1
	os.Stdout.Write(buffer)
}

func (e *Editor) DrawRows(buffer []byte) []byte {
	for y := 1; y <= e.ScreenRows; y++ {
		if y == e.ScreenRows/3 {
			welcome := fmt.Sprintf("goed editor -- version %s", VERSION)
			padding := (e.ScreenCols - len(welcome)) / 2
			buffer = append(buffer, []byte("~")...)
			for i := 1; i <= padding; i++ {
				buffer = append(buffer, []byte(" ")...)
			}
			buffer = append(buffer, []byte(welcome)...)
		} else {
			buffer = append(buffer, []byte("~")...)
		}
		buffer = append(buffer, []byte("\x1b[K")...)
		if y < e.ScreenRows {
			buffer = append(buffer, []byte("\r\n")...)
		} else {
			buffer = append(buffer, []byte(e.Message)...)
		}
	}
	return buffer
}

func (e *Editor) MoveCursor(key int) {
	switch key {
	case ARROW_LEFT:
		if e.CursorCol > 0 {
			e.CursorCol--
		}
	case ARROW_RIGHT:
		if e.CursorCol < e.ScreenCols-1 {
			e.CursorCol++
		}
	case ARROW_UP:
		if e.CursorRow > 0 {
			e.CursorRow--
		}
	case ARROW_DOWN:
		if e.CursorRow < e.ScreenRows-1 {
			e.CursorRow++
		}
	}
}

func main() {

	// put the terminal into raw mode
	oldState, err := terminal.MakeRaw(0)
	if err != nil {
		panic(err)
	}
	// restore terminal however we exit
	defer terminal.Restore(0, oldState)

	e := NewEditor()
	// input loop
	for {
		e.RefreshScreen()
		err = e.ProcessKeyPress()
		if err != nil {
			break
		}
	}
}
<|endoftext|>


<commit_before>package cpp

import (
	"fmt"
)

type FilePos struct {
	File string
	Line int
	Col  int
}

func (pos FilePos) String() string {
	return fmt.Sprintf("Position line %d col %d of %s.", pos.Line, pos.Col, pos.File)
}

type TokenKind int

func (tk TokenKind) String() string {
	switch tk {
	case TOK_IDENTIFIER:
		return "TOK_IDENTIFIER"
	case TOK_FOR:
		return "TOK_FOR"
	case TOK_INT:
		return "TOK_INT"
	default:
		return "Unknown token"
	}
}

//Token represents a grouping of characters
//that provide semantic meaning in a C program.
type Token struct {
	Kind TokenKind
	Val  string
	Pos  FilePos
}

func (t Token) String() string {
	return fmt.Sprintf("%s %s at %s", t.Kind, t.Val, t.Pos)
}

const (
	TOK_FOR = iota
	TOK_WHILE
	TOK_DO
	TOK_IF
	TOK_GOTO
	TOK_STRUCT
	TOK_SIGNED
	TOK_UNSIGNED
	TOK_TYPEDEF
	TOK_RETURN
	TOK_INT
	TOK_VOID
	TOK_SIZEOF
	TOK_IDENTIFIER
	TOK_CONSTANT_INT
	TOK_INC_OP
	TOK_PTR_OP
	TOK_OR_OP
	TOK_AND_OP
	TOK_EQ_OP
)

var keywordLUT = map[string]TokenKind{
	"for":      TOK_FOR,
	"while":    TOK_WHILE,
	"do":       TOK_DO,
	"if":       TOK_IF,
	"goto":     TOK_GOTO,
	"struct":   TOK_STRUCT,
	"signed":   TOK_SIGNED,
	"unsigned": TOK_UNSIGNED,
	"typedef":  TOK_TYPEDEF,
	"return":   TOK_RETURN,
	"int":      TOK_INT,
	"void":     TOK_VOID,
	"sizeof":   TOK_SIZEOF,
}
<commit_msg>fixes to tokenizer<commit_after>package cpp

import (
	"fmt"
)

const (
	TOK_FOR = 1000
	TOK_WHILE
	TOK_DO
	TOK_IF
	TOK_GOTO
	TOK_STRUCT
	TOK_SIGNED
	TOK_UNSIGNED
	TOK_TYPEDEF
	TOK_RETURN
	TOK_INT
	TOK_VOID
	TOK_SIZEOF
	TOK_IDENTIFIER
	TOK_CONSTANT_INT
	TOK_INC_OP
	TOK_PTR_OP
	TOK_OR_OP
	TOK_AND_OP
	TOK_EQ_OP
)

type FilePos struct {
	File string
	Line int
	Col  int
}

func (pos FilePos) String() string {
	return fmt.Sprintf("Position line %d col %d of %s.", pos.Line, pos.Col, pos.File)
}

type TokenKind int

func (tk TokenKind) String() string {
	switch tk {
	case TOK_IDENTIFIER:
		return "TOK_IDENTIFIER"
	case TOK_FOR:
		return "TOK_FOR"
	case TOK_INT:
		return "TOK_INT"
	default:
		return fmt.Sprintf("TOK %c", (int)tk)
	}
}

//Token represents a grouping of characters
//that provide semantic meaning in a C program.
type Token struct {
	Kind TokenKind
	Val  string
	Pos  FilePos
}

func (t Token) String() string {
	return fmt.Sprintf("%s %s at %s", t.Kind, t.Val, t.Pos)
}

var keywordLUT = map[string]TokenKind{
	"for":      TOK_FOR,
	"while":    TOK_WHILE,
	"do":       TOK_DO,
	"if":       TOK_IF,
	"goto":     TOK_GOTO,
	"struct":   TOK_STRUCT,
	"signed":   TOK_SIGNED,
	"unsigned": TOK_UNSIGNED,
	"typedef":  TOK_TYPEDEF,
	"return":   TOK_RETURN,
	"int":      TOK_INT,
	"void":     TOK_VOID,
	"sizeof":   TOK_SIZEOF,
}
<|endoftext|>


<commit_before>package gomq

import (
	"net"
	"strings"
	"time"

	"github.com/zeromq/gomq/zmtp"
)

var (
	defaultRetry = 250 * time.Millisecond
)

// Connection is a gomq connection. It holds
// both the net.Conn transport as well as the
// zmtp connection information.
type Connection struct {
	net  net.Conn
	zmtp *zmtp.Connection
}

// ZeroMQSocket is the base gomq interface.
type ZeroMQSocket interface {
	Recv() ([]byte, error)
	Send([]byte) error
	RetryInterval() time.Duration
	SocketType() zmtp.SocketType
	SecurityMechanism() zmtp.SecurityMechanism
	AddConnection(*Connection)
	RemoveConnection(string)
	RecvChannel() chan *zmtp.Message
	Close()
}

// Client is a gomq interface used for client sockets.
// It implements the Socket interface along with a
// Connect method for connecting to endpoints.
type Client interface {
	ZeroMQSocket
	Connect(endpoint string) error
}

// ConnectClient accepts a Client interface and an endpoint
// in the format <proto>://<address>:<port>. It then attempts
// to connect to the endpoint and perform a ZMTP handshake.
func ConnectClient(c Client, endpoint string) error {
	parts := strings.Split(endpoint, "://")

Connect:
	netConn, err := net.Dial(parts[0], parts[1])
	if err != nil {
		time.Sleep(c.RetryInterval())
		goto Connect
	}

	zmtpConn := zmtp.NewConnection(netConn)
	_, err = zmtpConn.Prepare(c.SecurityMechanism(), c.SocketType(), false, nil)
	if err != nil {
		return err
	}

	conn := &Connection{
		net:  netConn,
		zmtp: zmtpConn,
	}

	c.AddConnection(conn)
	zmtpConn.Recv(c.RecvChannel())
	return nil
}

// Server is a gomq interface used for server sockets.
// It implements the Socket interface along with a
// Bind method for binding to endpoints.
type Server interface {
	ZeroMQSocket
	Bind(endpoint string) (net.Addr, error)
}

// BindServer accepts a Server interface and an endpoint
// in the format <proto>://<address>:<port>. It then attempts
// to bind to the endpoint. TODO: change this to starting
// a listener on the endpoint that performs handshakes
// with any client that connects
func BindServer(s Server, endpoint string) (net.Addr, error) {
	var addr net.Addr
	parts := strings.Split(endpoint, "://")

	ln, err := net.Listen(parts[0], parts[1])
	if err != nil {
		return addr, err
	}

	netConn, err := ln.Accept()
	if err != nil {
		return addr, err
	}

	zmtpConn := zmtp.NewConnection(netConn)
	_, err = zmtpConn.Prepare(s.SecurityMechanism(), s.SocketType(), true, nil)
	if err != nil {
		return netConn.LocalAddr(), err
	}

	conn := &Connection{
		net:  netConn,
		zmtp: zmtpConn,
	}

	s.AddConnection(conn)
	zmtpConn.Recv(s.RecvChannel())
	return netConn.LocalAddr(), nil
}
<commit_msg>Add gomq.NewConnection()<commit_after>package gomq

import (
	"net"
	"strings"
	"time"

	"github.com/zeromq/gomq/zmtp"
)

var (
	defaultRetry = 250 * time.Millisecond
)

// Connection is a gomq connection. It holds
// both the net.Conn transport as well as the
// zmtp connection information.
type Connection struct {
	net  net.Conn
	zmtp *zmtp.Connection
}

// NewConnection accepts a net.Conn, a *zmtp.Connection
// and returns a *gomq.Connection.
func NewConnection(netConn net.Conn, zmtpConn *zmtp.Connection) *Connection {
	conn := &Connection{
		net:  netConn,
		zmtp: zmtpConn,
	}
	return conn
}

// ZeroMQSocket is the base gomq interface.
type ZeroMQSocket interface {
	Recv() ([]byte, error)
	Send([]byte) error
	RetryInterval() time.Duration
	SocketType() zmtp.SocketType
	SecurityMechanism() zmtp.SecurityMechanism
	AddConnection(*Connection)
	RemoveConnection(string)
	RecvChannel() chan *zmtp.Message
	Close()
}

// Client is a gomq interface used for client sockets.
// It implements the Socket interface along with a
// Connect method for connecting to endpoints.
type Client interface {
	ZeroMQSocket
	Connect(endpoint string) error
}

// ConnectClient accepts a Client interface and an endpoint
// in the format <proto>://<address>:<port>. It then attempts
// to connect to the endpoint and perform a ZMTP handshake.
func ConnectClient(c Client, endpoint string) error {
	parts := strings.Split(endpoint, "://")

Connect:
	netConn, err := net.Dial(parts[0], parts[1])
	if err != nil {
		time.Sleep(c.RetryInterval())
		goto Connect
	}

	zmtpConn := zmtp.NewConnection(netConn)
	_, err = zmtpConn.Prepare(c.SecurityMechanism(), c.SocketType(), false, nil)
	if err != nil {
		return err
	}

	conn := &Connection{
		net:  netConn,
		zmtp: zmtpConn,
	}

	c.AddConnection(conn)
	zmtpConn.Recv(c.RecvChannel())
	return nil
}

// Server is a gomq interface used for server sockets.
// It implements the Socket interface along with a
// Bind method for binding to endpoints.
type Server interface {
	ZeroMQSocket
	Bind(endpoint string) (net.Addr, error)
}

// BindServer accepts a Server interface and an endpoint
// in the format <proto>://<address>:<port>. It then attempts
// to bind to the endpoint. TODO: change this to starting
// a listener on the endpoint that performs handshakes
// with any client that connects
func BindServer(s Server, endpoint string) (net.Addr, error) {
	var addr net.Addr
	parts := strings.Split(endpoint, "://")

	ln, err := net.Listen(parts[0], parts[1])
	if err != nil {
		return addr, err
	}

	netConn, err := ln.Accept()
	if err != nil {
		return addr, err
	}

	zmtpConn := zmtp.NewConnection(netConn)
	_, err = zmtpConn.Prepare(s.SecurityMechanism(), s.SocketType(), true, nil)
	if err != nil {
		return netConn.LocalAddr(), err
	}

	conn := &Connection{
		net:  netConn,
		zmtp: zmtpConn,
	}

	s.AddConnection(conn)
	zmtpConn.Recv(s.RecvChannel())
	return netConn.LocalAddr(), nil
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2012 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goon

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"errors"
	"net/http"
	"reflect"
)

var (
	// LogErrors issues appengine.Context.Errorf on any error.
	LogErrors bool = true
)

// Goon holds the app engine context and request memory cache.
type Goon struct {
	context       appengine.Context
	cache         map[string]interface{}
	inTransaction bool
	toSet         map[string]interface{}
	toDelete      []string
}

func memkey(k *datastore.Key) string {
	return k.Encode()
}

// NewGoon creates a new Goon object from the given request.
func NewGoon(r *http.Request) *Goon {
	return FromContext(appengine.NewContext(r))
}

// FromContext creates a new Goon object from the given appengine Context.
func FromContext(c appengine.Context) *Goon {
	return &Goon{
		context: c,
		cache:   make(map[string]interface{}),
	}
}

func (g *Goon) error(err error) {
	if LogErrors {
		g.context.Errorf("goon: %v", err.Error())
	}
}

func (g *Goon) extractKeys(src interface{}) ([]*datastore.Key, error) {
	v := reflect.Indirect(reflect.ValueOf(src))
	if v.Kind() != reflect.Slice {
		return nil, errors.New("goon: value must be a slice or pointer-to-slice")
	}
	l := v.Len()

	keys := make([]*datastore.Key, l)
	for i := 0; i < l; i++ {
		vi := v.Index(i)
		key, err := g.getStructKey(vi.Interface())
		if err != nil {
			return nil, err
		}
		keys[i] = key
	}
	return keys, nil
}

// Key is the same as KeyError, except nil is returned on error.
func (g *Goon) Key(src interface{}) *datastore.Key {
	if k, err := g.KeyError(src); err == nil {
		return k
	}
	return nil
}

// Key returns the key of src based on its properties.
func (g *Goon) KeyError(src interface{}) (*datastore.Key, error) {
	return g.getStructKey(src)
}

// RunInTransaction runs f in a transaction. It calls f with a transaction
// context tg that f should use for all App Engine operations. Neither cache nor
// memcache are used or set during a transaction.
//
// Otherwise similar to appengine/datastore.RunInTransaction:
// https://developers.google.com/appengine/docs/go/datastore/reference#RunInTransaction
func (g *Goon) RunInTransaction(f func(tg *Goon) error, opts *datastore.TransactionOptions) error {
	var ng *Goon
	err := datastore.RunInTransaction(g.context, func(tc appengine.Context) error {
		ng = &Goon{
			context:       tc,
			inTransaction: true,
			toSet:         make(map[string]interface{}),
		}
		return f(ng)
	}, opts)

	if err == nil {
		for k, v := range ng.toSet {
			g.cache[k] = v
		}

		for _, k := range ng.toDelete {
			delete(g.cache, k)
		}
	} else {
		g.error(err)
	}

	return err
}

// Put saves the entity src into the datastore based on src's key k. If k
// is an incomplete key, the returned key will be a unique key generated by
// the datastore.
func (g *Goon) Put(src interface{}) (*datastore.Key, error) {
	ks, err := g.PutMulti([]interface{}{src})
	if len(ks) == 1 {
		return ks[0], err
	}
	return nil, err
}

// PutMany is a wrapper around PutMulti.
func (g *Goon) PutMany(srcs ...interface{}) ([]*datastore.Key, error) {
	return g.PutMulti(srcs)
}

const putMultiLimit = 500

// PutMulti is a batch version of Put.
//
// src must satisfy the same conditions as the dst argument to GetMulti.
func (g *Goon) PutMulti(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}

	var memkeys []string
	for _, key := range keys {
		if !key.Incomplete() {
			memkeys = append(memkeys, memkey(key))
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i <= len(keys)/putMultiLimit; i++ {
		lo := i * putMultiLimit
		hi := (i + 1) * putMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		rkeys, err := datastore.PutMulti(g.context, keys[lo:hi], v.Slice(lo, hi).Interface())
		if err != nil {
			g.error(err)
			return nil, err
		}

		for i, key := range keys[lo:hi] {
			vi := v.Index(lo + i).Interface()
			if key.Incomplete() {
				setStructKey(vi, rkeys[i])
				keys[i] = rkeys[i]
			}
			if g.inTransaction {
				g.toSet[memkey(rkeys[i])] = vi
			}
		}
	}

	if !g.inTransaction {
		g.putMemoryMulti(src)
	}

	return keys, nil
}

func (g *Goon) putMemoryMulti(src interface{}) {
	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i < v.Len(); i++ {
		g.putMemory(v.Index(i).Interface())
	}
}

func (g *Goon) putMemory(src interface{}) {
	key, _ := g.getStructKey(src)
	g.cache[memkey(key)] = src
}

func (g *Goon) putMemcache(srcs []interface{}) error {
	items := make([]*memcache.Item, len(srcs))

	for i, src := range srcs {
		gob, err := toGob(src)
		if err != nil {
			g.error(err)
			return err
		}
		key, err := g.getStructKey(src)

		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: gob,
		}
	}

	memcache.SetMulti(g.context, items)
	g.putMemoryMulti(srcs)
	return nil
}

// Get loads the entity based on dst's key into dst
// If there is no such entity for the key, Get returns
// datastore.ErrNoSuchEntity.
func (g *Goon) Get(dst interface{}) error {
	dsts := []interface{}{dst}
	if err := g.GetMulti(dsts); err != nil {
		// Look for an embedded error if it's multi
		if me, ok := err.(appengine.MultiError); ok {
			for i, merr := range me {
				if i == 0 {
					return merr
				}
			}
		}
		// Not multi, normal error
		return err
	}
	return nil
}

// GetMulti is a batch version of Get.
//
// dst has similar constraints as datastore.GetMulti.
func (g *Goon) GetMulti(dst interface{}) error {
	keys, err := g.extractKeys(dst)
	if err != nil {
		return err
	}

	if g.inTransaction {
		return datastore.GetMulti(g.context, keys, dst)
	}

	var dskeys []*datastore.Key
	var dsdst []interface{}
	var dixs []int

	var memkeys []string
	var mixs []int

	v := reflect.Indirect(reflect.ValueOf(dst))
	for i, key := range keys {
		m := memkey(key)
		if s, present := g.cache[m]; present {
			vi := v.Index(i)
			vi.Set(reflect.ValueOf(s))
		} else {
			memkeys = append(memkeys, m)
			mixs = append(mixs, i)
		}
	}

	memvalues, _ := memcache.GetMulti(g.context, memkeys)

	for i, m := range memkeys {
		d := v.Index(mixs[i]).Interface()
		if s, present := memvalues[m]; present {
			err := fromGob(d, s.Value)
			if err != nil {
				g.error(err)
				return err
			}

			g.putMemory(d)
		} else {
			key, err := g.getStructKey(d)
			if err != nil {
				g.error(err)
				return err
			}
			dskeys = append(dskeys, key)
			dsdst = append(dsdst, d)
			dixs = append(dixs, mixs[i])
		}
	}

	gmerr := datastore.GetMulti(g.context, dskeys, dsdst)
	var ret error
	var multiErr appengine.MultiError
	var toCache []interface{}
	if gmerr != nil {
		merr, ok := gmerr.(appengine.MultiError)
		if !ok {
			g.error(gmerr)
			return gmerr
		}
		multiErr = make(appengine.MultiError, len(keys))
		for i, idx := range dixs {
			multiErr[idx] = merr[i]
			if merr[i] == nil {
				toCache = append(toCache, dsdst[i])
			}
		}
		ret = multiErr
	} else {
		toCache = dsdst
	}

	if len(dskeys) > 0 {
		if err := g.putMemcache(toCache); err != nil {
			g.error(err)
			return err
		}
	}

	return ret
}

// Delete deletes the entity for the given key.
func (g *Goon) Delete(key *datastore.Key) error {
	keys := []*datastore.Key{key}
	return g.DeleteMulti(keys)
}

// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
	memkeys := make([]string, len(keys))
	for i, k := range keys {
		mk := memkey(k)
		memkeys[i] = mk

		if g.inTransaction {
			g.toDelete = append(g.toDelete, mk)
		} else {
			delete(g.cache, mk)
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	return datastore.DeleteMulti(g.context, keys)
}

// NotFound returns true if err is an appengine.MultiError and err[idx] is a datastore.ErrNoSuchEntity.
func NotFound(err error, idx int) bool {
	if merr, ok := err.(appengine.MultiError); ok {
		return idx < len(merr) && merr[idx] == datastore.ErrNoSuchEntity
	}
	return false
}
<commit_msg>Batch gets by 1000<commit_after>/*
 * Copyright (c) 2012 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goon

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"errors"
	"net/http"
	"reflect"
)

var (
	// LogErrors issues appengine.Context.Errorf on any error.
	LogErrors bool = true
)

// Goon holds the app engine context and request memory cache.
type Goon struct {
	context       appengine.Context
	cache         map[string]interface{}
	inTransaction bool
	toSet         map[string]interface{}
	toDelete      []string
}

func memkey(k *datastore.Key) string {
	return k.Encode()
}

// NewGoon creates a new Goon object from the given request.
func NewGoon(r *http.Request) *Goon {
	return FromContext(appengine.NewContext(r))
}

// FromContext creates a new Goon object from the given appengine Context.
func FromContext(c appengine.Context) *Goon {
	return &Goon{
		context: c,
		cache:   make(map[string]interface{}),
	}
}

func (g *Goon) error(err error) {
	if LogErrors {
		g.context.Errorf("goon: %v", err.Error())
	}
}

func (g *Goon) extractKeys(src interface{}) ([]*datastore.Key, error) {
	v := reflect.Indirect(reflect.ValueOf(src))
	if v.Kind() != reflect.Slice {
		return nil, errors.New("goon: value must be a slice or pointer-to-slice")
	}
	l := v.Len()

	keys := make([]*datastore.Key, l)
	for i := 0; i < l; i++ {
		vi := v.Index(i)
		key, err := g.getStructKey(vi.Interface())
		if err != nil {
			return nil, err
		}
		keys[i] = key
	}
	return keys, nil
}

// Key is the same as KeyError, except nil is returned on error.
func (g *Goon) Key(src interface{}) *datastore.Key {
	if k, err := g.KeyError(src); err == nil {
		return k
	}
	return nil
}

// Key returns the key of src based on its properties.
func (g *Goon) KeyError(src interface{}) (*datastore.Key, error) {
	return g.getStructKey(src)
}

// RunInTransaction runs f in a transaction. It calls f with a transaction
// context tg that f should use for all App Engine operations. Neither cache nor
// memcache are used or set during a transaction.
//
// Otherwise similar to appengine/datastore.RunInTransaction:
// https://developers.google.com/appengine/docs/go/datastore/reference#RunInTransaction
func (g *Goon) RunInTransaction(f func(tg *Goon) error, opts *datastore.TransactionOptions) error {
	var ng *Goon
	err := datastore.RunInTransaction(g.context, func(tc appengine.Context) error {
		ng = &Goon{
			context:       tc,
			inTransaction: true,
			toSet:         make(map[string]interface{}),
		}
		return f(ng)
	}, opts)

	if err == nil {
		for k, v := range ng.toSet {
			g.cache[k] = v
		}

		for _, k := range ng.toDelete {
			delete(g.cache, k)
		}
	} else {
		g.error(err)
	}

	return err
}

// Put saves the entity src into the datastore based on src's key k. If k
// is an incomplete key, the returned key will be a unique key generated by
// the datastore.
func (g *Goon) Put(src interface{}) (*datastore.Key, error) {
	ks, err := g.PutMulti([]interface{}{src})
	if len(ks) == 1 {
		return ks[0], err
	}
	return nil, err
}

// PutMany is a wrapper around PutMulti.
func (g *Goon) PutMany(srcs ...interface{}) ([]*datastore.Key, error) {
	return g.PutMulti(srcs)
}

const putMultiLimit = 500

// PutMulti is a batch version of Put.
//
// src must satisfy the same conditions as the dst argument to GetMulti.
func (g *Goon) PutMulti(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}

	var memkeys []string
	for _, key := range keys {
		if !key.Incomplete() {
			memkeys = append(memkeys, memkey(key))
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i <= len(keys)/putMultiLimit; i++ {
		lo := i * putMultiLimit
		hi := (i + 1) * putMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		rkeys, err := datastore.PutMulti(g.context, keys[lo:hi], v.Slice(lo, hi).Interface())
		if err != nil {
			g.error(err)
			return nil, err
		}

		for i, key := range keys[lo:hi] {
			vi := v.Index(lo + i).Interface()
			if key.Incomplete() {
				setStructKey(vi, rkeys[i])
				keys[i] = rkeys[i]
			}
			if g.inTransaction {
				g.toSet[memkey(rkeys[i])] = vi
			}
		}
	}

	if !g.inTransaction {
		g.putMemoryMulti(src)
	}

	return keys, nil
}

func (g *Goon) putMemoryMulti(src interface{}) {
	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i < v.Len(); i++ {
		g.putMemory(v.Index(i).Interface())
	}
}

func (g *Goon) putMemory(src interface{}) {
	key, _ := g.getStructKey(src)
	g.cache[memkey(key)] = src
}

func (g *Goon) putMemcache(srcs []interface{}) error {
	items := make([]*memcache.Item, len(srcs))

	for i, src := range srcs {
		gob, err := toGob(src)
		if err != nil {
			g.error(err)
			return err
		}
		key, err := g.getStructKey(src)

		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: gob,
		}
	}

	memcache.SetMulti(g.context, items)
	g.putMemoryMulti(srcs)
	return nil
}

// Get loads the entity based on dst's key into dst
// If there is no such entity for the key, Get returns
// datastore.ErrNoSuchEntity.
func (g *Goon) Get(dst interface{}) error {
	dsts := []interface{}{dst}
	if err := g.GetMulti(dsts); err != nil {
		// Look for an embedded error if it's multi
		if me, ok := err.(appengine.MultiError); ok {
			for i, merr := range me {
				if i == 0 {
					return merr
				}
			}
		}
		// Not multi, normal error
		return err
	}
	return nil
}

const getMultiLimit = 1000

// GetMulti is a batch version of Get.
//
// dst has similar constraints as datastore.GetMulti.
func (g *Goon) GetMulti(dst interface{}) error {
	keys, err := g.extractKeys(dst)
	if err != nil {
		return err
	}

	if g.inTransaction {
		// todo: support getMultiLimit in transactions
		return datastore.GetMulti(g.context, keys, dst)
	}

	var dskeys []*datastore.Key
	var dsdst []interface{}
	var dixs []int

	var memkeys []string
	var mixs []int

	v := reflect.Indirect(reflect.ValueOf(dst))
	for i, key := range keys {
		m := memkey(key)
		if s, present := g.cache[m]; present {
			vi := v.Index(i)
			vi.Set(reflect.ValueOf(s))
		} else {
			memkeys = append(memkeys, m)
			mixs = append(mixs, i)
		}
	}

	memvalues, _ := memcache.GetMulti(g.context, memkeys)

	for i, m := range memkeys {
		d := v.Index(mixs[i]).Interface()
		if s, present := memvalues[m]; present {
			err := fromGob(d, s.Value)
			if err != nil {
				g.error(err)
				return err
			}

			g.putMemory(d)
		} else {
			key, err := g.getStructKey(d)
			if err != nil {
				g.error(err)
				return err
			}
			dskeys = append(dskeys, key)
			dsdst = append(dsdst, d)
			dixs = append(dixs, mixs[i])
		}
	}

	multiErr := make(appengine.MultiError, len(keys))
	var toCache []interface{}
	var ret error
	for i := 0; i <= len(dskeys)/getMultiLimit; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(dskeys) {
			hi = len(dskeys)
		}
		gmerr := datastore.GetMulti(g.context, dskeys[lo:hi], dsdst[lo:hi])
		if gmerr != nil {
			merr, ok := gmerr.(appengine.MultiError)
			if !ok {
				g.error(gmerr)
				return gmerr
			}
			for i, idx := range dixs[lo:hi] {
				multiErr[idx] = merr[i]
				if merr[i] == nil {
					toCache = append(toCache, dsdst[lo+i])
				}
			}
			ret = multiErr
		} else {
			toCache = append(toCache, dsdst[lo:hi]...)
		}
	}

	if len(toCache) > 0 {
		if err := g.putMemcache(toCache); err != nil {
			g.error(err)
			return err
		}
	}

	return ret
}

// Delete deletes the entity for the given key.
func (g *Goon) Delete(key *datastore.Key) error {
	keys := []*datastore.Key{key}
	return g.DeleteMulti(keys)
}

// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
	memkeys := make([]string, len(keys))
	for i, k := range keys {
		mk := memkey(k)
		memkeys[i] = mk

		if g.inTransaction {
			g.toDelete = append(g.toDelete, mk)
		} else {
			delete(g.cache, mk)
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	return datastore.DeleteMulti(g.context, keys)
}

// NotFound returns true if err is an appengine.MultiError and err[idx] is a datastore.ErrNoSuchEntity.
func NotFound(err error, idx int) bool {
	if merr, ok := err.(appengine.MultiError); ok {
		return idx < len(merr) && merr[idx] == datastore.ErrNoSuchEntity
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goyo

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"net/mail"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var ROOT_DIRECTORY = flag.String("rootdir", "", "root directory for mail")

var TIME_REGEX = regexp.MustCompile(`\+([0-9]+)\.([A-Za-z]+)@`)

var UNIQ_FILENAME_REGEX = regexp.MustCompile(`(.+):`)

func init() {
	flag.Parse()
}

//processMessage processes each new message that appears in /new
func processMessage(filename string) error {
	//Parse message and determine when the message should be yo-yoed

	bts, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	message, err := mail.ReadMessage(bytes.NewBuffer(bts))

	if err != nil {
		return err
	}

	//Assume that there is only one recipient - the one we care about
	addresses, err := message.Header.AddressList("To")
	if err != nil {
		return err
	}

	to_address := addresses[0].Address
	log.Print("Found address %s", to_address)

	t, err := extractTimeFromAddress(to_address)
	if err != nil {
		return err
	}

	//Schedule future message for that yo-yoed time

	log.Print("Scheduling message for %v", t)
	if err := scheduleFutureMessage(filename, t); err != nil {
		return err
	}

	//Move message from /new to /cur, setting Maildir info flag to S (seen)
	destination := filepath.Join(ROOT_DIRECTORY, "cur", uniqueFromFilename(filename)+":2,S")
	log.Print("Moving message from %s to %s", filename, destination)
	err = os.Rename(filename, destination)

	return err
}

//Parse an email address and return the future time at which to bounce the email
func extractTimeFromAddress(to_address string) (time.Time, error) {

	matches := TIME_REGEX.FindStringSubmatch(to_address)

	number_s := matches[1]
	time_unit_s := matches[2]

	number, err := strconv.Atoi(number_s)
	if err != nil {
		panic(err)
	}

	//For now, we'll support minutes, hours, days, weeks, and months

	var time_unit time.Duration

	switch strings.ToLower(time_unit_s) {
	case "minute", "minutes":
		{
			time_unit = time.Minute
		}

	case "hour", "hours":
		{
			time_unit = time.Hour
		}

	case "day", "days":
		{
			time_unit = 24 * time.Hour
		}

	case "week", "weeks":
		{
			time_unit = 7 * 24 * time.Hour
		}

	case "month", "months":
		{
			time_unit = 30 * 7 * 24 * time.Hour
		}
	}

	delay := time.Duration(number) * time_unit
	future_time := time.Now().Add(delay)
	return future_time, nil

}

//scheduleFutureMessage schedules a future email delivery
func scheduleFutureMessage(filename string, t time.Time) (err error) {
	//TODO actually implement this
	uniq := uniqueFromFilename(filename)
	log.Print(uniq)

	return nil
}

//uniqueFromFilename extracts the unique part of a Maildir filename
func uniqueFromFilename(filename string) (uniq string) {
	//The real input set may actually be larger/more complicated than this
	//But this works for now
	matches := UNIQ_FILENAME_REGEX.FindStringSubmatch(filename)
	uniq = matches[1]
	return
}
<commit_msg>Change log.Print to log.Printf for readability<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goyo

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"net/mail"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var ROOT_DIRECTORY = flag.String("rootdir", "", "root directory for mail")

var TIME_REGEX = regexp.MustCompile(`\+([0-9]+)\.([A-Za-z]+)@`)

var UNIQ_FILENAME_REGEX = regexp.MustCompile(`(.+):`)

func init() {
	flag.Parse()
}

//processMessage processes each new message that appears in /new
func processMessage(filename string) error {
	//Parse message and determine when the message should be yo-yoed

	bts, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}

	message, err := mail.ReadMessage(bytes.NewBuffer(bts))

	if err != nil {
		return err
	}

	//Assume that there is only one recipient - the one we care about
	addresses, err := message.Header.AddressList("To")
	if err != nil {
		return err
	}

	to_address := addresses[0].Address
	log.Printf("Found address %s", to_address)

	t, err := extractTimeFromAddress(to_address)
	if err != nil {
		return err
	}

	//Schedule future message for that yo-yoed time

	log.Printf("Scheduling message for %v", t)
	if err := scheduleFutureMessage(filename, t); err != nil {
		return err
	}

	//Move message from /new to /cur, setting Maildir info flag to S (seen)
	destination := filepath.Join(ROOT_DIRECTORY, "cur", uniqueFromFilename(filename)+":2,S")
	log.Printf("Moving message from %s to %s", filename, destination)
	err = os.Rename(filename, destination)

	return err
}

//Parse an email address and return the future time at which to bounce the email
func extractTimeFromAddress(to_address string) (time.Time, error) {

	matches := TIME_REGEX.FindStringSubmatch(to_address)

	number_s := matches[1]
	time_unit_s := matches[2]

	number, err := strconv.Atoi(number_s)
	if err != nil {
		panic(err)
	}

	//For now, we'll support minutes, hours, days, weeks, and months

	var time_unit time.Duration

	switch strings.ToLower(time_unit_s) {
	case "minute", "minutes":
		{
			time_unit = time.Minute
		}

	case "hour", "hours":
		{
			time_unit = time.Hour
		}

	case "day", "days":
		{
			time_unit = 24 * time.Hour
		}

	case "week", "weeks":
		{
			time_unit = 7 * 24 * time.Hour
		}

	case "month", "months":
		{
			time_unit = 30 * 7 * 24 * time.Hour
		}
	}

	delay := time.Duration(number) * time_unit
	future_time := time.Now().Add(delay)
	return future_time, nil

}

//scheduleFutureMessage schedules a future email delivery
func scheduleFutureMessage(filename string, t time.Time) (err error) {
	//TODO actually implement this
	uniq := uniqueFromFilename(filename)
	log.Print(uniq)

	return nil
}

//uniqueFromFilename extracts the unique part of a Maildir filename
func uniqueFromFilename(filename string) (uniq string) {
	//The real input set may actually be larger/more complicated than this
	//But this works for now
	matches := UNIQ_FILENAME_REGEX.FindStringSubmatch(filename)
	uniq = matches[1]
	return
}
<|endoftext|>