<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"path"
	"time"

	"go.pedge.io/lion"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/gogo/protobuf/proto"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/worker"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps/server"
	"google.golang.org/grpc"
)

// appEnv stores the environment variables that this worker needs
type appEnv struct {
	// Address of etcd, so that worker can write its own IP there for discoverh
	EtcdAddress string `env:"ETCD_PORT_2379_TCP_ADDR,required"`

	// Address for connecting to pachd (so this can download input data)
	PachdAddress string `env:"PACHD_PORT_650_TCP_ADDR,required"`

	// Prefix in etcd for all pachd-related records
	PPSPrefix string `env:"PPS_ETCD_PREFIX,required"`

	// worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`

	// Either pipeline name or job name must be set
	PPSPipelineName string `env:"PPS_PIPELINE_NAME"`
	PPSJobID        string `env:"PPS_JOB_ID"`
	PodName         string `env:"PPS_POD_NAME,required"`
}

func main() {
	cmdutil.Main(do, &appEnv{})
}

func validateEnv(appEnv *appEnv) error {
	if appEnv.PPSPipelineName == "" && appEnv.PPSJobID == "" {
		return fmt.Errorf("worker must recieve either pipeline name or job ID, but got neither")
	} else if appEnv.PPSPipelineName != "" && appEnv.PPSJobID != "" {
		return fmt.Errorf("worker must recieve either pipeline name or job ID, but got both")
	}
	return nil
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of
func getPipelineInfo(etcdClient *etcd.Client, appEnv *appEnv) (*pps.PipelineInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "pipelines", appEnv.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline, got %d: %v", len(resp.Kvs), resp)
	}
	pipelineInfo := new(pps.PipelineInfo)
	if err := proto.UnmarshalText(string(resp.Kvs[0].Value), pipelineInfo); err != nil {
		return nil, err
	}
	return pipelineInfo, nil
}

func getJobInfo(etcdClient *etcd.Client, appEnv *appEnv) (*pps.JobInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "jobs", appEnv.PPSJobID))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 job, got %d: %v", len(resp.Kvs), resp)
	}
	jobInfo := new(pps.JobInfo)
	if err := proto.UnmarshalText(string(resp.Kvs[0].Value), jobInfo); err != nil {
		return nil, err
	}
	return jobInfo, nil
}

func do(appEnvObj interface{}) error {
	go func() {
		lion.Println(http.ListenAndServe(":652", nil))
	}()

	log.SetFlags(log.LstdFlags | log.Lshortfile)
	appEnv := appEnvObj.(*appEnv)
	if err := validateEnv(appEnv); err != nil {
		return fmt.Errorf("error validating env: %v", err)
	}

	// get pachd client, so we can upload output data from the user binary
	pachClient, err := client.NewFromAddress("localhost:650")
	if err != nil {
		return fmt.Errorf("error constructing pachClient: %v", err)
	}
	go pachClient.KeepConnected(make(chan bool)) // we never cancel the connection

	// Get etcd client, so we can register our IP (so pachd can discover us)
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{fmt.Sprintf("%s:2379", appEnv.EtcdAddress)},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return fmt.Errorf("error constructing etcdClient: %v", err)
	}

	// Construct worker API server. Get relevant pipeline or job info, and then
	// use that to create a worker.APIServer.
	var workerRcName string
	var apiServer *worker.APIServer
	if appEnv.PPSPipelineName != "" {
		pipelineInfo, err := getPipelineInfo(etcdClient, appEnv)
		if err != nil {
			return fmt.Errorf("error getting pipelineInfo: %v", err)
		}
		workerRcName = ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
		apiServer = worker.NewPipelineAPIServer(pachClient, pipelineInfo, appEnv.PodName)
	} else if appEnv.PPSJobID != "" {
		jobInfo, err := getJobInfo(etcdClient, appEnv)
		if err != nil {
			return fmt.Errorf("error getting jobInfo: %v", err)
		}
		workerRcName = ppsserver.JobRcName(jobInfo.Job.ID)
		apiServer = worker.NewJobAPIServer(pachClient, jobInfo, appEnv.PodName)
	}

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			func(s *grpc.Server) {
				worker.RegisterWorkerServer(s, apiServer)
				close(ready)
			},
			grpcutil.ServeOptions{
				Version:    version.Version,
				MaxMsgSize: grpcutil.MaxMsgSize,
			},
			grpcutil.ServeEnv{
				GRPCPort: client.PPSWorkerPort,
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(appEnv.PPSPrefix, "workers", workerRcName, appEnv.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	resp, err := etcdClient.Grant(ctx, 10 /* seconds */)
	if err != nil {
		return fmt.Errorf("error granting lease: %v", err)
	}

	// keepalive forever
	if _, err := etcdClient.KeepAlive(context.Background(), resp.ID); err != nil {
		return fmt.Errorf("error with KeepAlive: %v", err)
	}

	// Actually write "key" into etcd
	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	defer cancel()
	if _, err := etcdClient.Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return fmt.Errorf("error putting IP address: %v", err)
	}

	// If server ever exits, return error
	if err := eg.Wait(); err != nil {
		return err
	}
	return nil
}
<commit_msg>Make worker compile<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"path"
	"time"

	"go.pedge.io/lion"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/gogo/protobuf/proto"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/worker"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps/server"
	"google.golang.org/grpc"
)

// appEnv stores the environment variables that this worker needs
type appEnv struct {
	// Address of etcd, so that worker can write its own IP there for discoverh
	EtcdAddress string `env:"ETCD_PORT_2379_TCP_ADDR,required"`

	// Address for connecting to pachd (so this can download input data)
	PachdAddress string `env:"PACHD_PORT_650_TCP_ADDR,required"`

	// Prefix in etcd for all pachd-related records
	PPSPrefix string `env:"PPS_ETCD_PREFIX,required"`

	// worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`

	// Either pipeline name or job name must be set
	PPSPipelineName string `env:"PPS_PIPELINE_NAME"`
	PodName         string `env:"PPS_POD_NAME,required"`
}

func main() {
	cmdutil.Main(do, &appEnv{})
}

func validateEnv(appEnv *appEnv) error {
	if appEnv.PPSPipelineName == "" {
		return fmt.Errorf("worker must recieve pipeline name")
	}
	return nil
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of
func getPipelineInfo(etcdClient *etcd.Client, appEnv *appEnv) (*pps.PipelineInfo, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "pipelines", appEnv.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline, got %d: %v", len(resp.Kvs), resp)
	}
	pipelineInfo := new(pps.PipelineInfo)
	if err := proto.UnmarshalText(string(resp.Kvs[0].Value), pipelineInfo); err != nil {
		return nil, err
	}
	return pipelineInfo, nil
}

func do(appEnvObj interface{}) error {
	go func() {
		lion.Println(http.ListenAndServe(":652", nil))
	}()

	log.SetFlags(log.LstdFlags | log.Lshortfile)
	appEnv := appEnvObj.(*appEnv)
	if err := validateEnv(appEnv); err != nil {
		return fmt.Errorf("error validating env: %v", err)
	}

	// get pachd client, so we can upload output data from the user binary
	pachClient, err := client.NewFromAddress("localhost:650")
	if err != nil {
		return fmt.Errorf("error constructing pachClient: %v", err)
	}
	go pachClient.KeepConnected(make(chan bool)) // we never cancel the connection

	// Get etcd client, so we can register our IP (so pachd can discover us)
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{fmt.Sprintf("%s:2379", appEnv.EtcdAddress)},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return fmt.Errorf("error constructing etcdClient: %v", err)
	}

	// Construct worker API server.
	pipelineInfo, err := getPipelineInfo(etcdClient, appEnv)
	if err != nil {
		return fmt.Errorf("error getting pipelineInfo: %v", err)
	}
	workerRcName := ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
	apiServer := worker.NewPipelineAPIServer(pachClient, pipelineInfo, appEnv.PodName)

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			func(s *grpc.Server) {
				worker.RegisterWorkerServer(s, apiServer)
				close(ready)
			},
			grpcutil.ServeOptions{
				Version:    version.Version,
				MaxMsgSize: grpcutil.MaxMsgSize,
			},
			grpcutil.ServeEnv{
				GRPCPort: client.PPSWorkerPort,
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(appEnv.PPSPrefix, "workers", workerRcName, appEnv.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	resp, err := etcdClient.Grant(ctx, 10 /* seconds */)
	if err != nil {
		return fmt.Errorf("error granting lease: %v", err)
	}

	// keepalive forever
	if _, err := etcdClient.KeepAlive(context.Background(), resp.ID); err != nil {
		return fmt.Errorf("error with KeepAlive: %v", err)
	}

	// Actually write "key" into etcd
	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	defer cancel()
	if _, err := etcdClient.Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return fmt.Errorf("error putting IP address: %v", err)
	}

	// If server ever exits, return error
	if err := eg.Wait(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package persisttest

import "github.com/docker/machine/libmachine/host"

type FakeStore struct {
	Hosts                                           []*host.Host
	ExistsErr, ListErr, LoadErr, RemoveErr, SaveErr error
}

func (fs *FakeStore) Exists(name string) (bool, error) {
	if fs.ExistsErr != nil {
		return false, fs.ExistsErr
	}
	for _, h := range fs.Hosts {
		if h.Name == name {
			return true, nil
		}
	}

	return false, nil
}

func (fs *FakeStore) List() ([]string, error) {
	names := []string{}
	for _, h := range fs.Hosts {
		names = append(names, h.Name)
	}
	return names, fs.ListErr
}

func (fs *FakeStore) Load(name string) (*host.Host, error) {
	if fs.LoadErr != nil {
		return nil, fs.LoadErr
	}
	for _, h := range fs.Hosts {
		if h.Name == name {
			return h, nil
		}
	}

	return nil, nil
}

func (fs *FakeStore) Remove(name string) error {
	return fs.RemoveErr
}

func (fs *FakeStore) Save(host *host.Host) error {
	return fs.SaveErr
}
<commit_msg>libmachine: implements save and remove on FakeStore<commit_after>package persisttest

import "github.com/docker/machine/libmachine/host"

type FakeStore struct {
	Hosts                                           []*host.Host
	ExistsErr, ListErr, LoadErr, RemoveErr, SaveErr error
}

func (fs *FakeStore) Exists(name string) (bool, error) {
	if fs.ExistsErr != nil {
		return false, fs.ExistsErr
	}
	for _, h := range fs.Hosts {
		if h.Name == name {
			return true, nil
		}
	}

	return false, nil
}

func (fs *FakeStore) List() ([]string, error) {
	names := []string{}
	for _, h := range fs.Hosts {
		names = append(names, h.Name)
	}
	return names, fs.ListErr
}

func (fs *FakeStore) Load(name string) (*host.Host, error) {
	if fs.LoadErr != nil {
		return nil, fs.LoadErr
	}
	for _, h := range fs.Hosts {
		if h.Name == name {
			return h, nil
		}
	}

	return nil, nil
}

func (fs *FakeStore) Remove(name string) error {
	if fs.RemoveErr != nil {
		return fs.RemoveErr
	}
	for i, h := range fs.Hosts {
		if h.Name == name {
			fs.Hosts = append(fs.Hosts[:i], fs.Hosts[i+1:]...)
			return nil
		}
	}
	return nil
}

func (fs *FakeStore) Save(host *host.Host) error {
	if fs.SaveErr == nil {
		fs.Hosts = append(fs.Hosts, host)
		return nil
	}
	return fs.SaveErr
}
<|endoftext|>


<commit_before>package overlay

import (
	"io/ioutil"
	"path"
	"strings"

	"github.com/sirupsen/logrus"
)

var sysctlConf = map[string]string{
	"net.ipv4.neigh.default.gc_thresh1": "8192",
	"net.ipv4.neigh.default.gc_thresh2": "49152",
	"net.ipv4.neigh.default.gc_thresh3": "65536",
}

// writeSystemProperty writes the value to a path under /proc/sys as determined from the key.
// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward.
func writeSystemProperty(key, value string) error {
	keyPath := strings.Replace(key, ".", "/", -1)
	return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644)
}

func applyOStweaks() {
	for k, v := range sysctlConf {
		if err := writeSystemProperty(k, v); err != nil {
			logrus.Errorf("error setting the kernel parameter %s = %s, err: %s", k, v, err)
		}
	}
}
<commit_msg>Allow for larger preset property values, do not override<commit_after>package overlay

import (
	"io/ioutil"
	"path"
	"strconv"
	"strings"

	"github.com/sirupsen/logrus"
)

type conditionalCheck func(val1, val2 string) bool

type osValue struct {
	value   string
	checkFn conditionalCheck
}

var osConfig = map[string]osValue{
	"net.ipv4.neigh.default.gc_thresh1": {"8192", checkHigher},
	"net.ipv4.neigh.default.gc_thresh2": {"49152", checkHigher},
	"net.ipv4.neigh.default.gc_thresh3": {"65536", checkHigher},
}

func propertyIsValid(val1, val2 string, check conditionalCheck) bool {
	if check == nil || check(val1, val2) {
		return true
	}
	return false
}

func checkHigher(val1, val2 string) bool {
	val1Int, _ := strconv.ParseInt(val1, 10, 32)
	val2Int, _ := strconv.ParseInt(val2, 10, 32)
	return val1Int < val2Int
}

// writeSystemProperty writes the value to a path under /proc/sys as determined from the key.
// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward.
func writeSystemProperty(key, value string) error {
	keyPath := strings.Replace(key, ".", "/", -1)
	return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644)
}

func readSystemProperty(key string) (string, error) {
	keyPath := strings.Replace(key, ".", "/", -1)
	value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath))
	if err != nil {
		return "", err
	}
	return string(value), nil
}

func applyOStweaks() {
	for k, v := range osConfig {
		// read the existing property from disk
		oldv, err := readSystemProperty(k)
		if err != nil {
			logrus.Errorf("error reading the kernel parameter %s, error: %s", k, err)
			continue
		}

		if propertyIsValid(oldv, v.value, v.checkFn) {
			// write new prop value to disk
			if err := writeSystemProperty(k, v.value); err != nil {
				logrus.Errorf("error setting the kernel parameter %s = %s, (leaving as %s) error: %s", k, v.value, oldv, err)
				continue
			}
			logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.value, oldv)
		}
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"fmt"
	"github.com/fiorix/go-redis/redis"
	hpr_utils "github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	cfg = hpr_utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	last    time.Time
	proxy   map[string][]Proxy
	backend map[string]int
}

type Proxy struct {
	Connections int64
	Backend     string
	handler     http.Handler
}

func Listen(fd int, addr string) net.Listener {
	var l net.Listener
	var err error
	if fd >= 3 {
		l, err = net.FileListener(os.NewFile(uintptr(fd), "http"))
	} else {
		l, err = net.Listen("tcp", addr)
	}
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	s.backend = make(map[string]int)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		client := xff(r)
		hpr_utils.Log(fmt.Sprintf("Request from: %s Url: %s", client, r.Host))
		r.Header.Add("X-Forwarded-For", client)
		r.Header.Add("X-Real-IP", client)
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	h := req.Host
	if i := strings.Index(h, ":"); i >= 0 {
		h = h[:i]
	}

	_, ok := s.proxy[h]
	if !ok {
		f, _ := rc.ZRange(fmt.Sprintf("hpr-backends::%s", h), 0, -1, true)
		if len(f) == 0 {
			return nil
		}

		var url string
		for _, be := range f {
			count, err := strconv.Atoi(be)
			if err != nil {
				url = be
				continue
			}

			for r := 0; r <= count; r++ {
				s.mu.Lock()
				s.proxy[h] = append(s.proxy[h], Proxy{0, url, makeHandler(url)})
				s.mu.Unlock()
			}
		}

	}
	return s.Next(h)
}

/* TODO: Implement more balance algorithms */
func (s *Server) Next(h string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.backend[h]++
	total := len(s.proxy[h])
	if s.backend[h] == total {
		s.backend[h] = 0
	}
	hpr_utils.Log(fmt.Sprintf("Using backend: %s Url: %s", s.proxy[h][s.backend[h]].Backend, h))
	return s.proxy[h][s.backend[h]].handler
}

func dialTimeout(network, addr string) (net.Conn, error) {
	timeout := time.Duration(2 * time.Second)
	return net.DialTimeout(network, addr, timeout)
}

func (s *Server) probe_backends(probe time.Duration) {
	/*transport := http.Transport{Dial: dialTimeout}
	client := &http.Client{
		Transport: &transport,
	}*/

	for {
		time.Sleep(probe)
		// s.mu.Lock()
		for vhost, backend := range s.proxy {
			hpr_utils.Log(fmt.Sprintf("vhost: %s backends: %s", vhost, s.proxy[vhost][s.backend[vhost]].Backend))
			//client.Get(s.proxy[k][s.backend[h]])
		}
		// s.mu.Unlock()
	}
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<commit_msg>playing with probe<commit_after>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"fmt"
	"github.com/fiorix/go-redis/redis"
	hpr_utils "github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	cfg = hpr_utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	last    time.Time
	proxy   map[string][]Proxy
	backend map[string]int
}

type Proxy struct {
	Connections int64
	Backend     string
	handler     http.Handler
}

func Listen(fd int, addr string) net.Listener {
	var l net.Listener
	var err error
	if fd >= 3 {
		l, err = net.FileListener(os.NewFile(uintptr(fd), "http"))
	} else {
		l, err = net.Listen("tcp", addr)
	}
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	s.backend = make(map[string]int)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		client := xff(r)
		hpr_utils.Log(fmt.Sprintf("Request from: %s Url: %s", client, r.Host))
		r.Header.Add("X-Forwarded-For", client)
		r.Header.Add("X-Real-IP", client)
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	h := req.Host
	if i := strings.Index(h, ":"); i >= 0 {
		h = h[:i]
	}

	_, ok := s.proxy[h]
	if !ok {
		f, _ := rc.ZRange(fmt.Sprintf("hpr-backends::%s", h), 0, -1, true)
		if len(f) == 0 {
			return nil
		}

		var url string
		for _, be := range f {
			count, err := strconv.Atoi(be)
			if err != nil {
				url = be
				continue
			}

			for r := 0; r <= count; r++ {
				s.mu.Lock()
				s.proxy[h] = append(s.proxy[h], Proxy{0, url, makeHandler(url)})
				s.mu.Unlock()
			}
		}

	}
	return s.Next(h)
}

/* TODO: Implement more balance algorithms */
func (s *Server) Next(h string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.backend[h]++
	total := len(s.proxy[h])
	if s.backend[h] == total {
		s.backend[h] = 0
	}
	hpr_utils.Log(fmt.Sprintf("Using backend: %s Url: %s", s.proxy[h][s.backend[h]].Backend, h))
	return s.proxy[h][s.backend[h]].handler
}

func dialTimeout(network, addr string) (net.Conn, error) {
	timeout := time.Duration(2 * time.Second)
	return net.DialTimeout(network, addr, timeout)
}

func (s *Server) probe_backends(probe time.Duration) {
	/*transport := http.Transport{Dial: dialTimeout}
	client := &http.Client{
		Transport: &transport,
	}*/

	for {
		time.Sleep(probe)
		// s.mu.Lock()
		for vhost, _ := range s.proxy {
			hpr_utils.Log(fmt.Sprintf("vhost: %s backends: %s", vhost, s.proxy[vhost][s.backend[vhost]].Backend))
			//client.Get(s.proxy[k][s.backend[h]])
		}
		// s.mu.Unlock()
	}
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main // import "kkn.fi/cmd/vanity"

import (
	"fmt"
	"net/http"
	"strings"
)

type (
	// packageConfig defines Go package that has vanity import defined by Path,
	// VCS system type and VCS URL.
	packageConfig struct {
		// Name is the name of the Go package.
		Name string
		// VCS is version control system used by the project.
		VCS string
		// URL of the git repository
		URL string
	}
	// vanityServer is the actual HTTP server for Go vanity domains.
	vanityServer struct {
		// Domain is the vanity domain.
		Domain string
		// Packages contains settings for vanity Packages.
		Packages map[string]*packageConfig
	}
)

func (p packageConfig) path() string {
	i := strings.Index(p.Name, "/")
	return p.Name[i+1:]
}

// newPackage returns a new Package given a path and VCS.
func newPackage(name, vcs, url string) *packageConfig {
	return &packageConfig{
		Name: name,
		VCS:  vcs,
		URL:  url,
	}
}

// newServer returns a new Vanity Server given domain name and
// vanity package configuration.
func newServer(domain string, config map[string]*packageConfig) *vanityServer {
	return &vanityServer{
		Domain:   domain,
		Packages: config,
	}
}

// goDocURL returns the HTTP URL to godoc.org.
func (p packageConfig) goDocURL(domain string) string {
	return fmt.Sprintf("https://godoc.org/%v%v", domain, p.Name)
}

// goImportMeta creates the <meta/> HTML tag containing name and content attributes.
func (p packageConfig) goImportMeta(domain string) string {
	s := `<meta name="go-import" content="%v/%v %v %v">`
	return fmt.Sprintf(s, domain, p.path(), p.VCS, p.URL)
}

func (s vanityServer) find(path string) *packageConfig {
	p, ok := s.Packages[path]
	if !ok {
		return nil
	}
	return p
}

// ServeHTTP is an HTTP Handler for Go vanity domain.
func (s vanityServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	if r.Method != http.MethodGet {
		status := http.StatusMethodNotAllowed
		http.Error(w, http.StatusText(status), status)
		return
	}

	pack := s.find(r.URL.Path)
	if pack == nil {
		http.NotFound(w, r)
		return
	}
	if r.FormValue("go-get") != "1" {
		url := pack.goDocURL(s.Domain)
		http.Redirect(w, r, url, http.StatusTemporaryRedirect)
		return
	}
	fmt.Fprint(w, pack.goImportMeta(s.Domain))
}
<commit_msg>Add godoc to path() and move func<commit_after>package main // import "kkn.fi/cmd/vanity"

import (
	"fmt"
	"net/http"
	"strings"
)

type (
	// packageConfig defines Go package that has vanity import defined by Path,
	// VCS system type and VCS URL.
	packageConfig struct {
		// Name is the name of the Go package.
		Name string
		// VCS is version control system used by the project.
		VCS string
		// URL of the git repository
		URL string
	}
	// vanityServer is the actual HTTP server for Go vanity domains.
	vanityServer struct {
		// Domain is the vanity domain.
		Domain string
		// Packages contains settings for vanity Packages.
		Packages map[string]*packageConfig
	}
)

// newPackage returns a new Package given a path and VCS.
func newPackage(name, vcs, url string) *packageConfig {
	return &packageConfig{
		Name: name,
		VCS:  vcs,
		URL:  url,
	}
}

// newServer returns a new Vanity Server given domain name and
// vanity package configuration.
func newServer(domain string, config map[string]*packageConfig) *vanityServer {
	return &vanityServer{
		Domain:   domain,
		Packages: config,
	}
}

// path returns the path portition of the package name
func (p packageConfig) path() string {
	i := strings.Index(p.Name, "/")
	return p.Name[i+1:]
}

// goDocURL returns the HTTP URL to godoc.org.
func (p packageConfig) goDocURL(domain string) string {
	return fmt.Sprintf("https://godoc.org/%v%v", domain, p.Name)
}

// goImportMeta creates the <meta/> HTML tag containing name and content attributes.
func (p packageConfig) goImportMeta(domain string) string {
	s := `<meta name="go-import" content="%v/%v %v %v">`
	return fmt.Sprintf(s, domain, p.path(), p.VCS, p.URL)
}

func (s vanityServer) find(path string) *packageConfig {
	p, ok := s.Packages[path]
	if !ok {
		return nil
	}
	return p
}

// ServeHTTP is an HTTP Handler for Go vanity domain.
func (s vanityServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	if r.Method != http.MethodGet {
		status := http.StatusMethodNotAllowed
		http.Error(w, http.StatusText(status), status)
		return
	}

	pack := s.find(r.URL.Path)
	if pack == nil {
		http.NotFound(w, r)
		return
	}
	if r.FormValue("go-get") != "1" {
		url := pack.goDocURL(s.Domain)
		http.Redirect(w, r, url, http.StatusTemporaryRedirect)
		return
	}
	fmt.Fprint(w, pack.goImportMeta(s.Domain))
}
<|endoftext|>


<commit_before>package boss

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/pajlada/pajbot2/common"
)

type parse struct {
	m *common.Msg
}

/*
Parse parses an IRC message into a more readable bot.Msg
*/
func Parse(line string) common.Msg {
	p := &parse{}
	p.m = &common.Msg{
		User: common.User{},
	}
	parseTags := true
	if strings.Contains(line, "twitchnotify") {
		fmt.Println(line)
	}
	var splitline []string
	if strings.HasPrefix(line, ":") {
		parseTags = false
		splitline = strings.SplitN(line, ":", 2)
	} else {
		splitline = strings.SplitN(line, " :", 2)
	}
	tagsRaw := splitline[0]
	msg := splitline[1]
	tags := make(map[string]string)

	p.GetMessage(msg)
	if p.m.User.Name == "twitchnotify" {
		p.m.Type = "sub"
		p.Sub()
	} else {
		if strings.Contains(msg, "PRIVMSG") {
			p.m.Type = "privmsg"
		} else {
			p.m.Type = "whisper"
		}

		if parseTags {
			for _, tagValue := range strings.Split(tagsRaw, ";") {
				spl := strings.Split(tagValue, "=")
				k := spl[0]
				v := spl[1]
				tags[k] = v
			}
			p.GetTwitchEmotes(tags["emotes"])
			p.GetTags(tags)
		}
	}

	return *p.m
}

func (p *parse) GetTwitchEmotes(emotetag string) {
	p.m.Emotes = make([]common.Emote, 0)
	if emotetag == "" {
		return
	}
	emoteSlice := strings.Split(emotetag, "/")
	for i := range emoteSlice {
		id := strings.Split(emoteSlice[i], ":")[0]
		e := &common.Emote{}
		e.Type = "twitch"
		e.Name = ""
		e.ID = id
		e.Count = strings.Count(emoteSlice[i], "-")
		p.m.Emotes = append(p.m.Emotes, *e)
	}
}

func (p *parse) GetTags(tags map[string]string) {
	if tags["display-name"] == "" {
		p.m.User.DisplayName = p.m.User.Name
	} else {
		p.m.User.DisplayName = tags["display-name"]
	}
	p.m.User.Type = tags["user-type"]
	if tags["turbo"] == "1" {
		p.m.User.Turbo = true
	}
	if tags["mod"] == "1" {
		p.m.User.Mod = true
	}
	if tags["subscriber"] == "1" {
		p.m.User.Mod = true
	}

}

func (p *parse) GetMessage(msg string) {
	if strings.HasPrefix(msg, ":") {
		msg = strings.Replace(msg, ":", "", 1)
	}
	//fmt.Println(msg)
	p.m.Message = strings.SplitN(msg, " :", 2)[1]
	p.m.User.Name = strings.SplitN(msg, "!", 2)[0]
	c := strings.SplitN(msg, "#", 3)[1]
	p.m.Channel = strings.SplitN(c, " ", 2)[0]
}

func (p *parse) Sub() {
	m := p.m.Message
	if strings.Contains(m, "just ") {
		p.m.Length = 1
	} else {
		temp := strings.Split(m, " for ")[1]
		l := strings.Split(temp, " ")[0]
		length, err := strconv.Atoi(l)
		if err == nil {
			p.m.Length = length
		} else {
			panic(err)
		}
	}
	p.m.User.DisplayName = strings.Split(m, " ")[0]
	p.m.User.Name = strings.ToLower(p.m.User.DisplayName)
}
<commit_msg>fixed crash on host subs<commit_after>package boss

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/pajlada/pajbot2/common"
)

type parse struct {
	m *common.Msg
}

/*
Parse parses an IRC message into a more readable bot.Msg
*/
func Parse(line string) common.Msg {
	p := &parse{}
	p.m = &common.Msg{
		User: common.User{},
	}
	parseTags := true
	if strings.Contains(line, "twitchnotify") {
		fmt.Println(line)
	}
	var splitline []string
	if strings.HasPrefix(line, ":") {
		parseTags = false
		splitline = strings.SplitN(line, ":", 2)
	} else {
		splitline = strings.SplitN(line, " :", 2)
	}
	tagsRaw := splitline[0]
	msg := splitline[1]
	tags := make(map[string]string)

	p.GetMessage(msg)
	if p.m.User.Name == "twitchnotify" {
		if !strings.Contains(p.m.Message, " to ") && !strings.Contains(p.m.Message, " while ") {
			p.m.Type = "sub"
			p.Sub()
		} else {
			p.m.Type = "hostSub" // useless xD
		}

	} else {
		if strings.Contains(msg, "PRIVMSG") {
			p.m.Type = "privmsg"
		} else {
			p.m.Type = "whisper"
		}

		if parseTags {
			for _, tagValue := range strings.Split(tagsRaw, ";") {
				spl := strings.Split(tagValue, "=")
				k := spl[0]
				v := spl[1]
				tags[k] = v
			}
			p.GetTwitchEmotes(tags["emotes"])
			p.GetTags(tags)
		}
	}

	return *p.m
}

func (p *parse) GetTwitchEmotes(emotetag string) {
	p.m.Emotes = make([]common.Emote, 0)
	if emotetag == "" {
		return
	}
	emoteSlice := strings.Split(emotetag, "/")
	for i := range emoteSlice {
		id := strings.Split(emoteSlice[i], ":")[0]
		e := &common.Emote{}
		e.Type = "twitch"
		e.Name = ""
		e.ID = id
		e.Count = strings.Count(emoteSlice[i], "-")
		p.m.Emotes = append(p.m.Emotes, *e)
	}
}

func (p *parse) GetTags(tags map[string]string) {
	if tags["display-name"] == "" {
		p.m.User.DisplayName = p.m.User.Name
	} else {
		p.m.User.DisplayName = tags["display-name"]
	}
	p.m.User.Type = tags["user-type"]
	if tags["turbo"] == "1" {
		p.m.User.Turbo = true
	}
	if tags["mod"] == "1" {
		p.m.User.Mod = true
	}
	if tags["subscriber"] == "1" {
		p.m.User.Mod = true
	}

}

func (p *parse) GetMessage(msg string) {
	if strings.HasPrefix(msg, ":") {
		msg = strings.Replace(msg, ":", "", 1)
	}
	//fmt.Println(msg)
	p.m.Message = strings.SplitN(msg, " :", 2)[1]
	p.m.User.Name = strings.SplitN(msg, "!", 2)[0]
	c := strings.SplitN(msg, "#", 3)[1]
	p.m.Channel = strings.SplitN(c, " ", 2)[0]
}

func (p *parse) Sub() {
	m := p.m.Message
	if strings.Contains(m, "just ") {
		p.m.Length = 1
	} else {
		temp := strings.Split(m, " for ")[1]
		l := strings.Split(temp, " ")[0]
		length, err := strconv.Atoi(l)
		if err == nil {
			p.m.Length = length
		} else {
			panic(err)
		}
	}
	p.m.User.DisplayName = strings.Split(m, " ")[0]
	p.m.User.Name = strings.ToLower(p.m.User.DisplayName)
}
<|endoftext|>


<commit_before>package main

// BaseTemplateContent is HTML for base.tmpl
const BaseTemplateContent = `

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <title></title>

{{ template "content" . }}

`

// IndexTemplateContent is HTML for index page
const IndexTemplateContent = `{{ define "content" }}

Index

{{ end }}
`

// PostTemplateContent is HTML for post page
const PostTemplateContent = `{{ define "content"}}

{{.Content}}

{{ end }}
`
<commit_msg>update default index template<commit_after>package main

// BaseTemplateContent is HTML for base.tmpl
const BaseTemplateContent = `

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <title></title>

{{ template "content" . }}

`

// IndexTemplateContent is HTML for index page
const IndexTemplateContent = `{{ define "content" }}

Index

  {{ range .Posts }}

{{ .Title }}

  {{ end }}

{{ end }}
`

// PostTemplateContent is HTML for post page
const PostTemplateContent = `{{ define "content"}}

{{.Content}}

{{ end }}
`
<|endoftext|>


<commit_before>package dialer

import (
	"errors"
	"net"

	"github.com/v2ray/v2ray-core/common/dice"
	v2net "github.com/v2ray/v2ray-core/common/net"
)

var (
	ErrorInvalidHost = errors.New("Invalid Host.")
)

func Dial(dest v2net.Destination) (net.Conn, error) {
	var ip net.IP
	if dest.Address().IsIPv4() || dest.Address().IsIPv6() {
		ip = dest.Address().IP()
	} else {
		ips, err := net.LookupIP(dest.Address().Domain())
		if err != nil {
			return nil, err
		}
		if len(ips) == 0 {
			return nil, ErrorInvalidHost
		}
		ip = ips[dice.Roll(len(ips))]
	}
	if dest.IsTCP() {
		return net.DialTCP("tcp", nil, &net.TCPAddr{
			IP:   ip,
			Port: int(dest.Port()),
		})
	} else {
		return net.DialUDP("udp", nil, &net.UDPAddr{
			IP:   ip,
			Port: int(dest.Port()),
		})
	}
}
<commit_msg>use golang's default dual stack algo when dialling to a domain<commit_after>package dialer

import (
	"errors"
	"net"
	"time"

	v2net "github.com/v2ray/v2ray-core/common/net"
)

var (
	ErrorInvalidHost = errors.New("Invalid Host.")
)

func Dial(dest v2net.Destination) (net.Conn, error) {
	if dest.Address().IsDomain() {
		dialer := &net.Dialer{
			Timeout:   time.Second * 60,
			DualStack: true,
		}
		network := "tcp"
		if dest.IsUDP() {
			network = "udp"
		}
		return dialer.Dial(network, dest.NetAddr())
	}

	ip := dest.Address().IP()
	if dest.IsTCP() {
		return net.DialTCP("tcp", nil, &net.TCPAddr{
			IP:   ip,
			Port: int(dest.Port()),
		})
	} else {
		return net.DialUDP("udp", nil, &net.UDPAddr{
			IP:   ip,
			Port: int(dest.Port()),
		})
	}
}
<|endoftext|>


<commit_before>package gocouchbase

import "encoding/json"
import "fmt"
import "time"
import "github.com/couchbaselabs/gocouchbaseio"
import "net/http"
import "net/url"
import "math/rand"

// An interface representing a single bucket within a cluster.
type Bucket struct {
	httpCli *http.Client
	client  *gocouchbaseio.Agent
}

// Sets the timeout period for any CRUD operations
func (b *Bucket) SetOperationTimeout(val time.Duration) {
	b.client.SetOperationTimeout(val)
}

// Retrieves the timeout period for any CRUD operations.
func (b *Bucket) GetOperationTimeout() time.Duration {
	return b.client.GetOperationTimeout()
}

type getResult struct {
	bytes []byte
	flags uint32
	cas   uint64
	err   error
}

func (b *Bucket) afterOpTimeout() <-chan time.Time {
	return time.After(10 * time.Second)
}

type GetCallback func(interface{}, uint64, error)

type PendingOp interface {
	Cancel() bool
}

type ioGetCallback func([]byte, uint32, uint64, error)
type ioCasCallback func(uint64, error)
type ioCtrCallback func(uint64, uint64, error)

type hlpGetHandler func(ioGetCallback) (PendingOp, error)

func (b *Bucket) hlpGetExec(valuePtr interface{}, execFn hlpGetHandler) (valOut interface{}, casOut uint64, errOut error) {
	signal := make(chan bool)
	op, err := execFn(func(bytes []byte, flags uint32, cas uint64, err error) {
		go func() {
			if err != nil {
				errOut = err
				return
			}

			value, err := b.decodeValue(bytes, flags, valuePtr)
			if err != nil {
				errOut = err
				return
			}

			valOut = value
			casOut = cas
			signal <- true
		}()
	})
	if err != nil {
		return nil, 0, err
	}

	select {
	case <-signal:
		return
	case <-b.afterOpTimeout():
		op.Cancel()
		return nil, 0, timeoutError{}
	}
}

type hlpCasHandler func(ioCasCallback) (PendingOp, error)

func (b *Bucket) hlpCasExec(execFn hlpCasHandler) (casOut uint64, errOut error) {
	signal := make(chan bool)
	op, err := execFn(func(cas uint64, err error) {
		go func() {
			if err != nil {
				errOut = err
				return
			}

			casOut = cas
			signal <- true
		}()
	})
	if err != nil {
		return 0, err
	}

	select {
	case <-signal:
		return
	case <-b.afterOpTimeout():
		op.Cancel()
		return 0, timeoutError{}
	}
}

type hlpCtrHandler func(ioCtrCallback) (PendingOp, error)

func (b *Bucket) hlpCtrExec(execFn hlpCtrHandler) (valOut uint64, casOut uint64, errOut error) {
	signal := make(chan bool)
	op, err := execFn(func(value uint64, cas uint64, err error) {
		go func() {
			if err != nil {
				errOut = err
				return
			}

			valOut = value
			casOut = cas
			signal <- true
		}()
	})
	if err != nil {
		return 0, 0, err
	}

	select {
	case <-signal:
		return
	case <-b.afterOpTimeout():
		op.Cancel()
		return 0, 0, timeoutError{}
	}
}

// Retrieves a document from the bucket
func (b *Bucket) Get(key string, valuePtr interface{}) (interface{}, uint64, error) {
	return b.hlpGetExec(valuePtr, func(cb ioGetCallback) (PendingOp, error) {
		op, err := b.client.Get([]byte(key), gocouchbaseio.GetCallback(cb))
		return op, err
	})
}

// Retrieves a document and simultaneously updates its expiry time.
func (b *Bucket) GetAndTouch(key string, expiry uint32, valuePtr interface{}) (interface{}, uint64, error) {
	return b.hlpGetExec(valuePtr, func(cb ioGetCallback) (PendingOp, error) {
		op, err := b.client.GetAndTouch([]byte(key), expiry, gocouchbaseio.GetCallback(cb))
		return op, err
	})
}

// Locks a document for a period of time, providing exclusive RW access to it.
func (b *Bucket) GetAndLock(key string, lockTime uint32, valuePtr interface{}) (interface{}, uint64, error) {
	return b.hlpGetExec(valuePtr, func(cb ioGetCallback) (PendingOp, error) {
		op, err := b.client.GetAndLock([]byte(key), lockTime, gocouchbaseio.GetCallback(cb))
		return op, err
	})
}

// Unlocks a document which was locked with GetAndLock.
func (b *Bucket) Unlock(key string, cas uint64) (casOut uint64, errOut error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Unlock([]byte(key), cas, gocouchbaseio.UnlockCallback(cb))
		return op, err
	})
}

// Returns the value of a particular document from a replica server.
func (b *Bucket) GetReplica(key string, valuePtr interface{}, replicaIdx int) (interface{}, uint64, error) {
	panic("GetReplica not yet supported")
}

// Touches a document, specifying a new expiry time for it.
func (b *Bucket) Touch(key string, expiry uint32) (uint64, error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Touch([]byte(key), expiry, gocouchbaseio.TouchCallback(cb))
		return op, err
	})
}

// Removes a document from the bucket.
func (b *Bucket) Remove(key string, cas uint64) (casOut uint64, errOut error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Remove([]byte(key), cas, gocouchbaseio.RemoveCallback(cb))
		return op, err
	})
}

// Inserts or replaces a document in the bucket.
func (b *Bucket) Upsert(key string, value interface{}, expiry uint32) (casOut uint64, errOut error) {
	bytes, flags, err := b.encodeValue(value)
	if err != nil {
		return 0, err
	}

	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Set([]byte(key), bytes, flags, expiry, gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Inserts a new document to the bucket.
func (b *Bucket) Insert(key string, value interface{}, expiry uint32) (uint64, error) {
	bytes, flags, err := b.encodeValue(value)
	if err != nil {
		return 0, err
	}

	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Add([]byte(key), bytes, flags, expiry, gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Replaces a document in the bucket.
func (b *Bucket) Replace(key string, value interface{}, cas uint64, expiry uint32) (uint64, error) {
	bytes, flags, err := b.encodeValue(value)
	if err != nil {
		return 0, err
	}

	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Replace([]byte(key), bytes, flags, cas, expiry, gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Appends a string value to a document.
func (b *Bucket) Append(key, value string) (uint64, error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Append([]byte(key), []byte(value), gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Prepends a string value to a document.
func (b *Bucket) Prepend(key, value string) (uint64, error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Prepend([]byte(key), []byte(value), gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Performs an atomic addition or subtraction for an integer document.
func (b *Bucket) Counter(key string, delta, initial int64, expiry uint32) (uint64, uint64, error) {
	realInitial := uint64(0xFFFFFFFFFFFFFFFF)
	if initial > 0 {
		realInitial = uint64(initial)
	}

	if delta > 0 {
		return b.hlpCtrExec(func(cb ioCtrCallback) (PendingOp, error) {
			op, err := b.client.Increment([]byte(key), uint64(delta), realInitial, expiry, gocouchbaseio.CounterCallback(cb))
			return op, err
		})
	} else if delta < 0 {
		return b.hlpCtrExec(func(cb ioCtrCallback) (PendingOp, error) {
			op, err := b.client.Decrement([]byte(key), uint64(-delta), realInitial, expiry, gocouchbaseio.CounterCallback(cb))
			return op, err
		})
	} else {
		return 0, 0, clientError{"Delta must be a non-zero value."}
	}
}

// Returns a CAPI endpoint.  Guarenteed to return something for now...
func (b *Bucket) getViewEp() string {
	capiEps := b.client.GetCapiEps()
	return capiEps[rand.Intn(len(capiEps))]
}

type viewRowDecoder struct {
	Target interface{}
}

func (vrd *viewRowDecoder) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, vrd.Target)
}

type viewResponse struct {
	TotalRows int            `json:"total_rows,omitempty"`
	Rows      viewRowDecoder `json:"rows,omitempty"`
	Error     string         `json:"error,omitempty"`
	Reason    string         `json:"reason,omitempty"`
}

type viewError struct {
	message string
	reason  string
}

func (e *viewError) Error() string {
	return e.message + " - " + e.reason
}

// Performs a view query and returns a list of rows or an error.
func (b *Bucket) ExecuteViewQuery(q *ViewQuery, valsOut interface{}) (interface{}, error) {
	capiEp := b.getViewEp()

	urlParams := url.Values{}
	for k, v := range q.options {
		urlParams.Add(k, v)
	}

	reqUri := fmt.Sprintf("%s/_design/%s/_view/%s?%s", capiEp, q.ddoc, q.name, urlParams.Encode())

	resp, err := b.httpCli.Get(reqUri)
	if err != nil {
		return nil, err
	}

	if valsOut == nil {
		var vals []interface{}
		valsOut = &vals
	}
	viewResp := viewResponse{
		Rows: viewRowDecoder{
			Target: valsOut,
		},
	}

	jsonDec := json.NewDecoder(resp.Body)
	jsonDec.Decode(&viewResp)

	if resp.StatusCode != 200 {
		if viewResp.Error != "" {
			return nil, &viewError{
				message: viewResp.Error,
				reason:  viewResp.Reason,
			}
		}
		return nil, &viewError{
			message: "HTTP Error",
			reason:  fmt.Sprintf("Status code was %d.", resp.StatusCode),
		}
	}

	return valsOut, nil
}

func (b *Bucket) GetIoRouter() *gocouchbaseio.Agent {
	return b.client
}
<commit_msg>Removed some unused stuff.<commit_after>package gocouchbase

import "encoding/json"
import "fmt"
import "time"
import "github.com/couchbaselabs/gocouchbaseio"
import "net/http"
import "net/url"
import "math/rand"

// An interface representing a single bucket within a cluster.
type Bucket struct {
	httpCli *http.Client
	client  *gocouchbaseio.Agent
}

type getResult struct {
	bytes []byte
	flags uint32
	cas   uint64
	err   error
}

func (b *Bucket) afterOpTimeout() <-chan time.Time {
	return time.After(10 * time.Second)
}

type GetCallback func(interface{}, uint64, error)

type PendingOp interface {
	Cancel() bool
}

type ioGetCallback func([]byte, uint32, uint64, error)
type ioCasCallback func(uint64, error)
type ioCtrCallback func(uint64, uint64, error)

type hlpGetHandler func(ioGetCallback) (PendingOp, error)

func (b *Bucket) hlpGetExec(valuePtr interface{}, execFn hlpGetHandler) (valOut interface{}, casOut uint64, errOut error) {
	signal := make(chan bool)
	op, err := execFn(func(bytes []byte, flags uint32, cas uint64, err error) {
		go func() {
			if err != nil {
				errOut = err
				return
			}

			value, err := b.decodeValue(bytes, flags, valuePtr)
			if err != nil {
				errOut = err
				return
			}

			valOut = value
			casOut = cas
			signal <- true
		}()
	})
	if err != nil {
		return nil, 0, err
	}

	select {
	case <-signal:
		return
	case <-b.afterOpTimeout():
		op.Cancel()
		return nil, 0, timeoutError{}
	}
}

type hlpCasHandler func(ioCasCallback) (PendingOp, error)

func (b *Bucket) hlpCasExec(execFn hlpCasHandler) (casOut uint64, errOut error) {
	signal := make(chan bool)
	op, err := execFn(func(cas uint64, err error) {
		go func() {
			if err != nil {
				errOut = err
				return
			}

			casOut = cas
			signal <- true
		}()
	})
	if err != nil {
		return 0, err
	}

	select {
	case <-signal:
		return
	case <-b.afterOpTimeout():
		op.Cancel()
		return 0, timeoutError{}
	}
}

type hlpCtrHandler func(ioCtrCallback) (PendingOp, error)

func (b *Bucket) hlpCtrExec(execFn hlpCtrHandler) (valOut uint64, casOut uint64, errOut error) {
	signal := make(chan bool)
	op, err := execFn(func(value uint64, cas uint64, err error) {
		go func() {
			if err != nil {
				errOut = err
				return
			}

			valOut = value
			casOut = cas
			signal <- true
		}()
	})
	if err != nil {
		return 0, 0, err
	}

	select {
	case <-signal:
		return
	case <-b.afterOpTimeout():
		op.Cancel()
		return 0, 0, timeoutError{}
	}
}

// Retrieves a document from the bucket
func (b *Bucket) Get(key string, valuePtr interface{}) (interface{}, uint64, error) {
	return b.hlpGetExec(valuePtr, func(cb ioGetCallback) (PendingOp, error) {
		op, err := b.client.Get([]byte(key), gocouchbaseio.GetCallback(cb))
		return op, err
	})
}

// Retrieves a document and simultaneously updates its expiry time.
func (b *Bucket) GetAndTouch(key string, expiry uint32, valuePtr interface{}) (interface{}, uint64, error) {
	return b.hlpGetExec(valuePtr, func(cb ioGetCallback) (PendingOp, error) {
		op, err := b.client.GetAndTouch([]byte(key), expiry, gocouchbaseio.GetCallback(cb))
		return op, err
	})
}

// Locks a document for a period of time, providing exclusive RW access to it.
func (b *Bucket) GetAndLock(key string, lockTime uint32, valuePtr interface{}) (interface{}, uint64, error) {
	return b.hlpGetExec(valuePtr, func(cb ioGetCallback) (PendingOp, error) {
		op, err := b.client.GetAndLock([]byte(key), lockTime, gocouchbaseio.GetCallback(cb))
		return op, err
	})
}

// Unlocks a document which was locked with GetAndLock.
func (b *Bucket) Unlock(key string, cas uint64) (casOut uint64, errOut error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Unlock([]byte(key), cas, gocouchbaseio.UnlockCallback(cb))
		return op, err
	})
}

// Returns the value of a particular document from a replica server.
func (b *Bucket) GetReplica(key string, valuePtr interface{}, replicaIdx int) (interface{}, uint64, error) {
	panic("GetReplica not yet supported")
}

// Touches a document, specifying a new expiry time for it.
func (b *Bucket) Touch(key string, expiry uint32) (uint64, error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Touch([]byte(key), expiry, gocouchbaseio.TouchCallback(cb))
		return op, err
	})
}

// Removes a document from the bucket.
func (b *Bucket) Remove(key string, cas uint64) (casOut uint64, errOut error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Remove([]byte(key), cas, gocouchbaseio.RemoveCallback(cb))
		return op, err
	})
}

// Inserts or replaces a document in the bucket.
func (b *Bucket) Upsert(key string, value interface{}, expiry uint32) (casOut uint64, errOut error) {
	bytes, flags, err := b.encodeValue(value)
	if err != nil {
		return 0, err
	}

	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Set([]byte(key), bytes, flags, expiry, gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Inserts a new document to the bucket.
func (b *Bucket) Insert(key string, value interface{}, expiry uint32) (uint64, error) {
	bytes, flags, err := b.encodeValue(value)
	if err != nil {
		return 0, err
	}

	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Add([]byte(key), bytes, flags, expiry, gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Replaces a document in the bucket.
func (b *Bucket) Replace(key string, value interface{}, cas uint64, expiry uint32) (uint64, error) {
	bytes, flags, err := b.encodeValue(value)
	if err != nil {
		return 0, err
	}

	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Replace([]byte(key), bytes, flags, cas, expiry, gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Appends a string value to a document.
func (b *Bucket) Append(key, value string) (uint64, error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Append([]byte(key), []byte(value), gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Prepends a string value to a document.
func (b *Bucket) Prepend(key, value string) (uint64, error) {
	return b.hlpCasExec(func(cb ioCasCallback) (PendingOp, error) {
		op, err := b.client.Prepend([]byte(key), []byte(value), gocouchbaseio.StoreCallback(cb))
		return op, err
	})
}

// Performs an atomic addition or subtraction for an integer document.
func (b *Bucket) Counter(key string, delta, initial int64, expiry uint32) (uint64, uint64, error) {
	realInitial := uint64(0xFFFFFFFFFFFFFFFF)
	if initial > 0 {
		realInitial = uint64(initial)
	}

	if delta > 0 {
		return b.hlpCtrExec(func(cb ioCtrCallback) (PendingOp, error) {
			op, err := b.client.Increment([]byte(key), uint64(delta), realInitial, expiry, gocouchbaseio.CounterCallback(cb))
			return op, err
		})
	} else if delta < 0 {
		return b.hlpCtrExec(func(cb ioCtrCallback) (PendingOp, error) {
			op, err := b.client.Decrement([]byte(key), uint64(-delta), realInitial, expiry, gocouchbaseio.CounterCallback(cb))
			return op, err
		})
	} else {
		return 0, 0, clientError{"Delta must be a non-zero value."}
	}
}

// Returns a CAPI endpoint.  Guarenteed to return something for now...
func (b *Bucket) getViewEp() string {
	capiEps := b.client.GetCapiEps()
	return capiEps[rand.Intn(len(capiEps))]
}

type viewRowDecoder struct {
	Target interface{}
}

func (vrd *viewRowDecoder) UnmarshalJSON(data []byte) error {
	return json.Unmarshal(data, vrd.Target)
}

type viewResponse struct {
	TotalRows int            `json:"total_rows,omitempty"`
	Rows      viewRowDecoder `json:"rows,omitempty"`
	Error     string         `json:"error,omitempty"`
	Reason    string         `json:"reason,omitempty"`
}

type viewError struct {
	message string
	reason  string
}

func (e *viewError) Error() string {
	return e.message + " - " + e.reason
}

// Performs a view query and returns a list of rows or an error.
func (b *Bucket) ExecuteViewQuery(q *ViewQuery, valsOut interface{}) (interface{}, error) {
	capiEp := b.getViewEp()

	urlParams := url.Values{}
	for k, v := range q.options {
		urlParams.Add(k, v)
	}

	reqUri := fmt.Sprintf("%s/_design/%s/_view/%s?%s", capiEp, q.ddoc, q.name, urlParams.Encode())

	resp, err := b.httpCli.Get(reqUri)
	if err != nil {
		return nil, err
	}

	if valsOut == nil {
		var vals []interface{}
		valsOut = &vals
	}
	viewResp := viewResponse{
		Rows: viewRowDecoder{
			Target: valsOut,
		},
	}

	jsonDec := json.NewDecoder(resp.Body)
	jsonDec.Decode(&viewResp)

	if resp.StatusCode != 200 {
		if viewResp.Error != "" {
			return nil, &viewError{
				message: viewResp.Error,
				reason:  viewResp.Reason,
			}
		}
		return nil, &viewError{
			message: "HTTP Error",
			reason:  fmt.Sprintf("Status code was %d.", resp.StatusCode),
		}
	}

	return valsOut, nil
}

func (b *Bucket) GetIoRouter() *gocouchbaseio.Agent {
	return b.client
}
<|endoftext|>


<commit_before>package views

import (
	"github.com/flosch/pongo2"
	"lambda.sx/marcus/lambdago/models"
	"lambda.sx/marcus/lambdago/session"
	"lambda.sx/marcus/lambdago/settings"
	"lambda.sx/marcus/lambdago/sql"
	"net/http"
	"os"
	"strings"
	"upper.io/db"
)

//Compile the templates on startup for a speed boost
var usercpTpl = pongo2.Must(pongo2.FromFile("templates/usercp.html"))
var manageUploadsTpl = pongo2.Must(pongo2.FromFile("templates/manageuploads.html"))

func HandleUserCP(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	if user.ID == 0 { // Not logged in
		http.Redirect(w, r, "/login", 302)
		return nil, ""
	}
	rendered_user_cp, err := usercpTpl.Execute(pongo2.Context{
		"user":   user,
		"themes": settings.Themes,
	})
	if err != nil {
		return err, ""
	}
	return nil, rendered_user_cp
}

func HandleManageUploads(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	if user.ID == 0 { // Not logged in
		http.Redirect(w, r, "/login", 302)
		return nil, ""
	}

	pasteCol, _ := sql.Connection().Collection("pastes")
	var pastes []models.Paste
	pasteCol.Find(db.Cond{"owner": user.ID}).Limit(10).All(&pastes)

	fileCol, _ := sql.Connection().Collection("files")
	var files []models.File
	fileCol.Find(db.Cond{"owner": user.ID}).Limit(10).All(&files)

	rendered_manage_uploads, err := manageUploadsTpl.Execute(pongo2.Context{
		"user":           user,
		"pastes":         pastes,
		"images":         files,
		"img_extensions": []string{".png", ".jpg"},
	})
	if err != nil {
		return err, ""
	}
	return nil, rendered_manage_uploads
}

func HandleToggleEncryption(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	if user.ID == 0 { // Not logged in
		http.Redirect(w, r, "/login", 302)
		return nil, ""
	}
	user.EncryptionEnabled = !user.EncryptionEnabled
	col, _ := sql.Connection().Collection("users")
	col.Find(db.Cond{"ID": user.ID}).Update(user)

	// Bring back to user cp
	http.Redirect(w, r, "/usercp", 302)
	return nil, ""
}

func HandleDelete(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	if user.ID == 0 { // Not logged in
		http.Redirect(w, r, "/login", 302)
		return nil, ""
	}

	filename := r.FormValue("file")
	if filename == "" {
		return nil, "No filename sent!"
	}
	col, _ := sql.Connection().Collection("files")
	var file models.File
	result := col.Find(db.Cond{"owner": user.ID, "name": filename})
	result.One(&file)
	if file.ID == 0 {
		// Try to find a paste
		col, _ := sql.Connection().Collection("pastes")
		result = col.Find(db.Cond{"owner": user.ID, "name": filename})
		result.One(&file)
		if file.ID == 0 {
			return nil, "File not found!"
		}
	}
	os.Remove("uploads/" + file.Name + file.Extension)
	result.Remove()

	// Bring back to manage uploads
	http.Redirect(w, r, "/manageuploads", 302)
	return nil, ""
}

func HandleSetTheme(r *http.Request, w http.ResponseWriter) (error, string) {
	if r.Method == "GET" {
		user := session.GetUser(r, w)
		if user.ID == 0 { // Not logged in
			http.Redirect(w, r, "/login", 302)
			return nil, ""
		}

		themeName := strings.ToLower(r.FormValue("name"))

		themeExists := false
		for _, name := range settings.Themes {
			if themeName == strings.ToLower(name) {
				themeExists = true
			}
		}

		if themeExists {
			user.ThemeName = themeName
			col, _ := sql.Connection().Collection("users")
			col.Find(db.Cond{"ID": user.ID}).Update(user)
		}

		// Bring back to user cp
		http.Redirect(w, r, "/usercp", 302)
		return nil, ""
	} else {
		return nil, "POST NOT SUPPORTED"
	}
}
<commit_msg>Sort uploads in the manager in the correct order<commit_after>package views

import (
	"github.com/flosch/pongo2"
	"lambda.sx/marcus/lambdago/models"
	"lambda.sx/marcus/lambdago/session"
	"lambda.sx/marcus/lambdago/settings"
	"lambda.sx/marcus/lambdago/sql"
	"net/http"
	"os"
	"strings"
	"upper.io/db"
)

//Compile the templates on startup for a speed boost
var usercpTpl = pongo2.Must(pongo2.FromFile("templates/usercp.html"))
var manageUploadsTpl = pongo2.Must(pongo2.FromFile("templates/manageuploads.html"))

func HandleUserCP(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	if user.ID == 0 { // Not logged in
		http.Redirect(w, r, "/login", 302)
		return nil, ""
	}
	rendered_user_cp, err := usercpTpl.Execute(pongo2.Context{
		"user":   user,
		"themes": settings.Themes,
	})
	if err != nil {
		return err, ""
	}
	return nil, rendered_user_cp
}

func HandleManageUploads(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	if user.ID == 0 { // Not logged in
		http.Redirect(w, r, "/login", 302)
		return nil, ""
	}

	pasteCol, _ := sql.Connection().Collection("pastes")
	var pastes []models.Paste
	pasteCol.Find(db.Cond{"owner": user.ID}).Limit(10).All(&pastes)

	fileCol, _ := sql.Connection().Collection("files")
	var files []models.File
	fileCol.Find(db.Cond{"owner": user.ID}).Sort("-upload_date").Limit(10).All(&files)

	rendered_manage_uploads, err := manageUploadsTpl.Execute(pongo2.Context{
		"user":           user,
		"pastes":         pastes,
		"images":         files,
		"img_extensions": []string{".png", ".jpg"},
	})
	if err != nil {
		return err, ""
	}
	return nil, rendered_manage_uploads
}

func HandleToggleEncryption(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	if user.ID == 0 { // Not logged in
		http.Redirect(w, r, "/login", 302)
		return nil, ""
	}
	user.EncryptionEnabled = !user.EncryptionEnabled
	col, _ := sql.Connection().Collection("users")
	col.Find(db.Cond{"ID": user.ID}).Update(user)

	// Bring back to user cp
	http.Redirect(w, r, "/usercp", 302)
	return nil, ""
}

func HandleDelete(r *http.Request, w http.ResponseWriter) (error, string) {
	user := session.GetUser(r, w)
	if user.ID == 0 { // Not logged in
		http.Redirect(w, r, "/login", 302)
		return nil, ""
	}

	filename := r.FormValue("file")
	if filename == "" {
		return nil, "No filename sent!"
	}
	col, _ := sql.Connection().Collection("files")
	var file models.File
	result := col.Find(db.Cond{"owner": user.ID, "name": filename})
	result.One(&file)
	if file.ID == 0 {
		// Try to find a paste
		col, _ := sql.Connection().Collection("pastes")
		result = col.Find(db.Cond{"owner": user.ID, "name": filename})
		result.One(&file)
		if file.ID == 0 {
			return nil, "File not found!"
		}
	}
	os.Remove("uploads/" + file.Name + file.Extension)
	result.Remove()

	// Bring back to manage uploads
	http.Redirect(w, r, "/manageuploads", 302)
	return nil, ""
}

func HandleSetTheme(r *http.Request, w http.ResponseWriter) (error, string) {
	if r.Method == "GET" {
		user := session.GetUser(r, w)
		if user.ID == 0 { // Not logged in
			http.Redirect(w, r, "/login", 302)
			return nil, ""
		}

		themeName := strings.ToLower(r.FormValue("name"))

		themeExists := false
		for _, name := range settings.Themes {
			if themeName == strings.ToLower(name) {
				themeExists = true
			}
		}

		if themeExists {
			user.ThemeName = themeName
			col, _ := sql.Connection().Collection("users")
			col.Find(db.Cond{"ID": user.ID}).Update(user)
		}

		// Bring back to user cp
		http.Redirect(w, r, "/usercp", 302)
		return nil, ""
	} else {
		return nil, "POST NOT SUPPORTED"
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gamelost/bot3server/module/fight"
	"github.com/gamelost/bot3server/module/mongo"
	// "github.com/gamelost/bot3server/module/logger"
	"github.com/gamelost/bot3server/server"
	//"github.com/gamelost/bot3server/module/helloworld"
	"github.com/gamelost/bot3server/module/help"
	"github.com/gamelost/bot3server/module/inconceivable"
	"github.com/gamelost/bot3server/module/slap"
	//"github.com/gamelost/bot3server/module/sleep"
	"github.com/gamelost/bot3server/module/cah"
	"github.com/gamelost/bot3server/module/nextwedding"
	// "github.com/gamelost/bot3server/module/panic"
	"github.com/gamelost/bot3server/module/dice"
	"github.com/gamelost/bot3server/module/remindme"
	"github.com/gamelost/bot3server/module/zed"
	//"github.com/gamelost/bot3server/module/testcommands"
	"github.com/gamelost/bot3server/module/boulderingtime"
	"github.com/gamelost/bot3server/module/catfacts"
	wuconditions "github.com/gamelost/bot3server/module/weather/conditions"
	wuforecast "github.com/gamelost/bot3server/module/weather/forecast"
	//"github.com/gamelost/bot3server/module/howlongbeforeicanquityouintel"
	iniconf "code.google.com/p/goconf/conf"
	"encoding/json"
	nsq "github.com/gamelost/go-nsq"
	"github.com/twinj/uuid"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

const DEFAULT_CONFIG_FILENAME = "bot3server.config"
const CONFIG_CAT_DEFAULT = "default"
const CONFIG_CAT_PLUGINS = "plugins"

// nsq specific constants
const CONFIG_CAT_NSQ = "nsq"
const CONFIG_BOT3SERVER_INPUT = "bot3server-input"
const CONFIG_BOT3SERVER_OUTPUT = "bot3server-output"
const CONFIG_OUTPUT_WRITER_ADDR = "output-writer-address"
const CONFIG_LOOKUPD_ADDR = "lookupd-address"
const TOPIC_MAIN = "main"

var conf *iniconf.ConfigFile

func main() {

	// create unique UUID on startup
	newUUID := uuid.NewV1()

	// the quit channel
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	config, err := iniconf.ReadConfigFile(DEFAULT_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Unable to read configuration file. Exiting now.")
	}
	server.ServerConfig = config

	bot3serverInput, _ := config.GetString(CONFIG_CAT_DEFAULT, CONFIG_BOT3SERVER_INPUT)
	outputWriterAddress, _ := config.GetString(CONFIG_CAT_NSQ, CONFIG_OUTPUT_WRITER_ADDR)
	lookupdAddress, _ := config.GetString(CONFIG_CAT_NSQ, CONFIG_LOOKUPD_ADDR)

	// set up listener instance
	incomingFromIRC, err := nsq.NewReader(bot3serverInput, TOPIC_MAIN)
	if err != nil {
		panic(err)
		sigChan <- syscall.SIGINT
	}

	// set up channels
	outgoingToNSQChan := make(chan *server.BotResponse)

	outputWriter := nsq.NewWriter(outputWriterAddress)

	// set up heartbeat ticker
	heartbeatTicker := time.NewTicker(1 * time.Second)

	// initialize the handlers
	botApp := &BotApp{Config: config, OutgoingChan: outgoingToNSQChan, UniqueID: newUUID}
	botApp.initServices()

	incomingFromIRC.AddHandler(botApp)
	incomingFromIRC.ConnectToLookupd(lookupdAddress)

	go HandleOutgoingToNSQ(outgoingToNSQChan, heartbeatTicker.C, outputWriter, newUUID)

	log.Printf("Done starting up. UUID:[%s]. Waiting on quit signal.", newUUID.String())
	<-sigChan
}

func HandleOutgoingToNSQ(outgoingToNSQChan chan *server.BotResponse, heartbeatTicker <-chan time.Time, outputWriter *nsq.Writer, serverID uuid.UUID) {

	for {
		select {
		case msg := <-outgoingToNSQChan:
			val, _ := json.Marshal(msg)
			outputWriter.Publish("bot3server-output", val)
			break
		case t := <-heartbeatTicker:
			hb := &server.Bot3ServerHeartbeat{ServerID: serverID.String(), Timestamp: t}
			val, _ := json.Marshal(hb)
			outputWriter.Publish("bot3server-heartbeat", val)
			break
		}
	}
}

type BotApp struct {
	Config       *iniconf.ConfigFile
	Handlers     map[string]server.BotHandler
	OutgoingChan chan *server.BotResponse
	UniqueID     uuid.UUID
}

func (ba *BotApp) AddHandler(key string, h server.BotHandler) {
	plugins, err := server.ServerConfig.GetString(CONFIG_CAT_PLUGINS, "enabled")
	// If plugins string does not exist, assume that all plugins
	// are enabled.
	if err == nil {
		if !strings.Contains(" "+plugins+" ", " "+key+" ") {
			return
		}
	}
	ba.Handlers[key] = h
}

func (ba *BotApp) GetHandler(key string) server.BotHandler {
	return ba.Handlers[key]
}

func (ba *BotApp) initServices() error {

	ba.Handlers = make(map[string]server.BotHandler)

	// implement all services
	ba.AddHandler("fight", (new(fight.FightService)).NewService())
	ba.AddHandler("cah", (new(cah.CahService)).NewService())
	ba.AddHandler("mongo", (new(mongo.MongoService)).NewService())
	ba.AddHandler("slap", (new(slap.SlapService)).NewService())
	ba.AddHandler("inconceivable", (new(inconceivable.InconceivableService)).NewService())
	ba.AddHandler("help", (new(help.HelpService)).NewService())
	ba.AddHandler("remindme", (new(remindme.RemindMeService)).NewService())
	ba.AddHandler("nextwedding", (new(nextwedding.NextWeddingService)).NewService())
	ba.AddHandler("weather", (new(wuconditions.WeatherConditionsService)).NewService())
	ba.AddHandler("forecast", (new(wuforecast.WeatherForecastService)).NewService())
	ba.AddHandler("zed", (new(zed.ZedsDeadService)).NewService())
	ba.AddHandler("boulderingtime", (new(boulderingtime.BoulderingTimeService)).NewService())
	ba.AddHandler("dice", (new(dice.DiceService)).NewService())
	ba.AddHandler("catfacts", (new(catfacts.CatFactsService)).NewService())
	return nil
}

func (ba *BotApp) HandleMessage(message *nsq.Message) error {

	//	ba.IncomingChan <- message
	var req = &server.BotRequest{}
	json.Unmarshal(message.Body, req)
	go ba.HandleIncoming(req)
	return nil
}

func (ba *BotApp) HandleIncoming(botRequest *server.BotRequest) error {

	// since we dont want the entire botapi to crash if a module throws an error or panic
	// we'll trap it here and log/notify the owner.

	// mabye a good idea to automatically-disable the module if it panics too frequently

	// throwout malformed requests
	if len(botRequest.ChatText) < 1 {
		return nil
	}

	// deferred method to protect against panics
	defer func(rawline string) {
		// if module panics, log it and discard
		if x := recover(); x != nil {
			log.Printf("Trapped panic. Rawline is:[%s]: %v\n", rawline, x)
		}
	}(botRequest.Text())

	// log all lines
	// logger.Log(botRequest)

	// check if command before processing
	if botRequest.RequestIsCommand() {

		command := botRequest.Command()

		handler := ba.GetHandler(command)

		if handler != nil {
			//log.Printf("Assigning handler for: %s\n", command)
			botResponse := &server.BotResponse{Target: botRequest.Channel, Identifier: botRequest.Identifier}
			handler.Handle(botRequest, botResponse)
			ba.OutgoingChan <- botResponse
		}
	}

	// return error object (nil for now)
	return nil
}
<commit_msg>moving method under bot3 struct<commit_after>package main

import (
	"github.com/gamelost/bot3server/module/fight"
	"github.com/gamelost/bot3server/module/mongo"
	// "github.com/gamelost/bot3server/module/logger"
	"github.com/gamelost/bot3server/server"
	//"github.com/gamelost/bot3server/module/helloworld"
	"github.com/gamelost/bot3server/module/help"
	"github.com/gamelost/bot3server/module/inconceivable"
	"github.com/gamelost/bot3server/module/slap"
	//"github.com/gamelost/bot3server/module/sleep"
	"github.com/gamelost/bot3server/module/cah"
	"github.com/gamelost/bot3server/module/nextwedding"
	// "github.com/gamelost/bot3server/module/panic"
	"github.com/gamelost/bot3server/module/dice"
	"github.com/gamelost/bot3server/module/remindme"
	"github.com/gamelost/bot3server/module/zed"
	//"github.com/gamelost/bot3server/module/testcommands"
	"github.com/gamelost/bot3server/module/boulderingtime"
	"github.com/gamelost/bot3server/module/catfacts"
	wuconditions "github.com/gamelost/bot3server/module/weather/conditions"
	wuforecast "github.com/gamelost/bot3server/module/weather/forecast"
	//"github.com/gamelost/bot3server/module/howlongbeforeicanquityouintel"
	iniconf "code.google.com/p/goconf/conf"
	"encoding/json"
	nsq "github.com/gamelost/go-nsq"
	"github.com/twinj/uuid"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

const DEFAULT_CONFIG_FILENAME = "bot3server.config"
const CONFIG_CAT_DEFAULT = "default"
const CONFIG_CAT_PLUGINS = "plugins"

// nsq specific constants
const CONFIG_CAT_NSQ = "nsq"
const CONFIG_BOT3SERVER_INPUT = "bot3server-input"
const CONFIG_BOT3SERVER_OUTPUT = "bot3server-output"
const CONFIG_OUTPUT_WRITER_ADDR = "output-writer-address"
const CONFIG_LOOKUPD_ADDR = "lookupd-address"
const TOPIC_MAIN = "main"

var conf *iniconf.ConfigFile

func main() {

	// create unique UUID on startup
	newUUID := uuid.NewV1()

	// the quit channel
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	config, err := iniconf.ReadConfigFile(DEFAULT_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Unable to read configuration file. Exiting now.")
	}
	server.ServerConfig = config

	bot3serverInput, _ := config.GetString(CONFIG_CAT_DEFAULT, CONFIG_BOT3SERVER_INPUT)
	outputWriterAddress, _ := config.GetString(CONFIG_CAT_NSQ, CONFIG_OUTPUT_WRITER_ADDR)
	lookupdAddress, _ := config.GetString(CONFIG_CAT_NSQ, CONFIG_LOOKUPD_ADDR)

	// set up listener instance
	incomingFromIRC, err := nsq.NewReader(bot3serverInput, TOPIC_MAIN)
	if err != nil {
		panic(err)
		sigChan <- syscall.SIGINT
	}

	// set up channels
	outgoingToNSQChan := make(chan *server.BotResponse)

	outputWriter := nsq.NewWriter(outputWriterAddress)

	// set up heartbeat ticker
	heartbeatTicker := time.NewTicker(1 * time.Second)

	// initialize the handlers
	botApp := &BotApp{Config: config, OutgoingChan: outgoingToNSQChan, UniqueID: newUUID}
	botApp.initServices()

	incomingFromIRC.AddHandler(botApp)
	incomingFromIRC.ConnectToLookupd(lookupdAddress)

	go botApp.HandleOutgoingToNSQ(outgoingToNSQChan, heartbeatTicker.C, outputWriter, newUUID)

	log.Printf("Done starting up. UUID:[%s]. Waiting on quit signal.", newUUID.String())
	<-sigChan
}

type BotApp struct {
	Config       *iniconf.ConfigFile
	Handlers     map[string]server.BotHandler
	OutgoingChan chan *server.BotResponse
	UniqueID     uuid.UUID
}

func (ba *BotApp) AddHandler(key string, h server.BotHandler) {
	plugins, err := server.ServerConfig.GetString(CONFIG_CAT_PLUGINS, "enabled")
	// If plugins string does not exist, assume that all plugins
	// are enabled.
	if err == nil {
		if !strings.Contains(" "+plugins+" ", " "+key+" ") {
			return
		}
	}
	ba.Handlers[key] = h
}

func (ba *BotApp) GetHandler(key string) server.BotHandler {
	return ba.Handlers[key]
}

func (ba *BotApp) initServices() error {

	ba.Handlers = make(map[string]server.BotHandler)

	// implement all services
	ba.AddHandler("fight", (new(fight.FightService)).NewService())
	ba.AddHandler("cah", (new(cah.CahService)).NewService())
	ba.AddHandler("mongo", (new(mongo.MongoService)).NewService())
	ba.AddHandler("slap", (new(slap.SlapService)).NewService())
	ba.AddHandler("inconceivable", (new(inconceivable.InconceivableService)).NewService())
	ba.AddHandler("help", (new(help.HelpService)).NewService())
	ba.AddHandler("remindme", (new(remindme.RemindMeService)).NewService())
	ba.AddHandler("nextwedding", (new(nextwedding.NextWeddingService)).NewService())
	ba.AddHandler("weather", (new(wuconditions.WeatherConditionsService)).NewService())
	ba.AddHandler("forecast", (new(wuforecast.WeatherForecastService)).NewService())
	ba.AddHandler("zed", (new(zed.ZedsDeadService)).NewService())
	ba.AddHandler("boulderingtime", (new(boulderingtime.BoulderingTimeService)).NewService())
	ba.AddHandler("dice", (new(dice.DiceService)).NewService())
	ba.AddHandler("catfacts", (new(catfacts.CatFactsService)).NewService())
	return nil
}

func (ba *BotApp) HandleMessage(message *nsq.Message) error {

	//	ba.IncomingChan <- message
	var req = &server.BotRequest{}
	json.Unmarshal(message.Body, req)
	go ba.HandleIncoming(req)
	return nil
}

func (ba *BotApp) HandleOutgoingToNSQ(outgoingToNSQChan chan *server.BotResponse, heartbeatTicker <-chan time.Time, outputWriter *nsq.Writer, serverID uuid.UUID) {

	for {
		select {
		case msg := <-outgoingToNSQChan:
			val, _ := json.Marshal(msg)
			outputWriter.Publish("bot3server-output", val)
			break
		case t := <-heartbeatTicker:
			hb := &server.Bot3ServerHeartbeat{ServerID: serverID.String(), Timestamp: t}
			val, _ := json.Marshal(hb)
			outputWriter.Publish("bot3server-heartbeat", val)
			break
		}
	}
}

func (ba *BotApp) HandleIncoming(botRequest *server.BotRequest) error {

	// since we dont want the entire botapi to crash if a module throws an error or panic
	// we'll trap it here and log/notify the owner.

	// mabye a good idea to automatically-disable the module if it panics too frequently

	// throwout malformed requests
	if len(botRequest.ChatText) < 1 {
		return nil
	}

	// deferred method to protect against panics
	defer func(rawline string) {
		// if module panics, log it and discard
		if x := recover(); x != nil {
			log.Printf("Trapped panic. Rawline is:[%s]: %v\n", rawline, x)
		}
	}(botRequest.Text())

	// log all lines
	// logger.Log(botRequest)

	// check if command before processing
	if botRequest.RequestIsCommand() {

		command := botRequest.Command()

		handler := ba.GetHandler(command)

		if handler != nil {
			//log.Printf("Assigning handler for: %s\n", command)
			botResponse := &server.BotResponse{Target: botRequest.Channel, Identifier: botRequest.Identifier}
			handler.Handle(botRequest, botResponse)
			ba.OutgoingChan <- botResponse
		}
	}

	// return error object (nil for now)
	return nil
}
<|endoftext|>


<commit_before>package cl11

import clw "github.com/rdwilliamson/clw11"

// 3 cases:
// 1. Device memory
// 2. Host memory alloced by the CL
// 3. Host memory alloced by go
type Buffer struct {
	ID     MemoryID
	Device bool

	// mapped  []byte
	// Read    bool
	// Write   bool
	// Host    bool // is it on the host or device
	// alloced bool // if the implementation alloced the memory
}

type MemoryID clw.Memory

func CreateDeviceBuffer(c *Context, size int, read, write bool, host []byte, copyHost bool) (*Buffer, error) {
	var flags clw.MemoryFlags
	if read {
		if write {
			flags = clw.MemoryReadWrite
		} else {
			flags = clw.MemoryReadOnly
		}
	} else if write {
		flags = clw.MemoryWriteOnly
	}

	if copyHost {
		flags |= clw.MemoryCopyHostPointer
	}

	memory, err := clw.CreateBuffer(clw.Context(c.ID), flags, clw.Size(size), host)
	if err != nil {
		return nil, err
	}
	return &Buffer{ID: MemoryID(memory), Device: true}, nil
}

func CreateHostBuffer(c *Context, size int, read, write bool) (*Buffer, error) {
	flags := clw.MemoryAllocHostPointer

	if read {
		if write {
			flags |= clw.MemoryReadWrite
		} else {
			flags |= clw.MemoryReadOnly
		}

	} else {
		flags |= clw.MemoryWriteOnly
	}

	memory, err := clw.CreateBuffer(clw.Context(c.ID), flags, clw.Size(size), nil)
	if err != nil {
		return nil, err
	}
	return &Buffer{ID: MemoryID(memory), Device: false}, nil
}
<commit_msg>Added release memory object.<commit_after>package cl11

import clw "github.com/rdwilliamson/clw11"

// 3 cases:
// 1. Device memory
// 2. Host memory alloced by the CL
// 3. Host memory alloced by go
type Buffer struct {
	ID     MemoryID
	Device bool // Is the memory on the device.

	// mapped  []byte
	// Read    bool
	// Write   bool
	// Host    bool // is it on the host or device
	// alloced bool // if the implementation alloced the memory
}

type MemoryID clw.Memory

func CreateDeviceBuffer(c *Context, size int, read, write bool, host []byte, copyHost bool) (*Buffer, error) {
	var flags clw.MemoryFlags
	if read {
		if write {
			flags = clw.MemoryReadWrite
		} else {
			flags = clw.MemoryReadOnly
		}
	} else if write {
		flags = clw.MemoryWriteOnly
	}

	if copyHost {
		flags |= clw.MemoryCopyHostPointer
	}

	memory, err := clw.CreateBuffer(clw.Context(c.ID), flags, clw.Size(size), host)
	if err != nil {
		return nil, err
	}
	return &Buffer{ID: MemoryID(memory), Device: true}, nil
}

func CreateHostBuffer(c *Context, size int, read, write bool) (*Buffer, error) {
	flags := clw.MemoryAllocHostPointer

	if read {
		if write {
			flags |= clw.MemoryReadWrite
		} else {
			flags |= clw.MemoryReadOnly
		}

	} else {
		flags |= clw.MemoryWriteOnly
	}

	memory, err := clw.CreateBuffer(clw.Context(c.ID), flags, clw.Size(size), nil)
	if err != nil {
		return nil, err
	}
	return &Buffer{ID: MemoryID(memory), Device: false}, nil
}

func (b *Buffer) Release() error {
	return clw.ReleaseMemObject(clw.Memory(b.ID))
}
<|endoftext|>


<commit_before>package typhon

import (
	"bytes"
	"io"
	"sync"
)

type bufCloser struct {
	bytes.Buffer
}

func (b *bufCloser) Close() error {
	return nil // No-op
}

type streamer struct {
	pipeR *io.PipeReader
	pipeW *io.PipeWriter
}

// Streamer returns a reader/writer/closer that can be used to stream service responses. It does not necessarily
// perform internal buffering, so users should take care not to depend on such behaviour.
func Streamer() io.ReadWriteCloser {
	pipeR, pipeW := io.Pipe()
	return &streamer{
		pipeR: pipeR,
		pipeW: pipeW}
}

func (s *streamer) Read(p []byte) (int, error) {
	return s.pipeR.Read(p)
}

func (s *streamer) Write(p []byte) (int, error) {
	return s.pipeW.Write(p)
}

func (s *streamer) Close() error {
	return s.pipeW.Close()
}

// countingWriter is a writer which proxies writes to an underlying io.Writer, keeping track of how many bytes have
// been written in total
type countingWriter struct {
	n int
	io.Writer
}

func (w *countingWriter) Write(p []byte) (int, error) {
	n, err := w.Writer.Write(p)
	w.n += n
	return n, err
}

// doneReader is a wrapper around a ReadCloser which provides notification when the stream has been fully consumed
// (ie. when EOF is reached, or when the reader is closed.)
type doneReader struct {
	closed     chan struct{}
	closedOnce sync.Once
	length     int64 // length of the underlying reader in bytes, if known. -1 indicates unknown
	read       int64 // number of bytes read
	io.ReadCloser
}

func newDoneReader(r io.ReadCloser, length int64) *doneReader {
	return &doneReader{
		closed:     make(chan struct{}),
		ReadCloser: r}
}

func (r *doneReader) Close() error {
	err := r.ReadCloser.Close()
	r.closedOnce.Do(func() { close(r.closed) })
	return err
}

func (r *doneReader) Read(p []byte) (int, error) {
	n, err := r.ReadCloser.Read(p)
	r.read += int64(n)
	// If we got an error reading, or the reader's length is known and is now exhausted, close
	// the underlying reaer
	if err != nil || (r.length >= 0 && r.read >= r.length) {
		r.Close()
	}
	return n, err
}
<commit_msg>Typo 🔤<commit_after>package typhon

import (
	"bytes"
	"io"
	"sync"
)

type bufCloser struct {
	bytes.Buffer
}

func (b *bufCloser) Close() error {
	return nil // No-op
}

type streamer struct {
	pipeR *io.PipeReader
	pipeW *io.PipeWriter
}

// Streamer returns a reader/writer/closer that can be used to stream service responses. It does not necessarily
// perform internal buffering, so users should take care not to depend on such behaviour.
func Streamer() io.ReadWriteCloser {
	pipeR, pipeW := io.Pipe()
	return &streamer{
		pipeR: pipeR,
		pipeW: pipeW}
}

func (s *streamer) Read(p []byte) (int, error) {
	return s.pipeR.Read(p)
}

func (s *streamer) Write(p []byte) (int, error) {
	return s.pipeW.Write(p)
}

func (s *streamer) Close() error {
	return s.pipeW.Close()
}

// countingWriter is a writer which proxies writes to an underlying io.Writer, keeping track of how many bytes have
// been written in total
type countingWriter struct {
	n int
	io.Writer
}

func (w *countingWriter) Write(p []byte) (int, error) {
	n, err := w.Writer.Write(p)
	w.n += n
	return n, err
}

// doneReader is a wrapper around a ReadCloser which provides notification when the stream has been fully consumed
// (ie. when EOF is reached, or when the reader is closed.)
type doneReader struct {
	closed     chan struct{}
	closedOnce sync.Once
	length     int64 // length of the underlying reader in bytes, if known. -1 indicates unknown
	read       int64 // number of bytes read
	io.ReadCloser
}

func newDoneReader(r io.ReadCloser, length int64) *doneReader {
	return &doneReader{
		closed:     make(chan struct{}),
		ReadCloser: r}
}

func (r *doneReader) Close() error {
	err := r.ReadCloser.Close()
	r.closedOnce.Do(func() { close(r.closed) })
	return err
}

func (r *doneReader) Read(p []byte) (int, error) {
	n, err := r.ReadCloser.Read(p)
	r.read += int64(n)
	// If we got an error reading, or the reader's length is known and is now exhausted, close
	// the underlying reader
	if err != nil || (r.length >= 0 && r.read >= r.length) {
		r.Close()
	}
	return n, err
}
<|endoftext|>


<commit_before>package states

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"sort"
	"strings"

	ctyjson "github.com/zclconf/go-cty/cty/json"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/config/hcl2shim"
)

// String returns a rather-odd string representation of the entire state.
//
// This is intended to match the behavior of the older terraform.State.String
// method that is used in lots of existing tests. It should not be used in
// new tests: instead, use "cmp" to directly compare the state data structures
// and print out a diff if they do not match.
//
// This method should never be used in non-test code, whether directly by call
// or indirectly via a %s or %q verb in package fmt.
func (s *State) String() string {
	if s == nil {
		return "<nil>"
	}

	// sort the modules by name for consistent output
	modules := make([]string, 0, len(s.Modules))
	for m := range s.Modules {
		modules = append(modules, m)
	}
	sort.Strings(modules)

	var buf bytes.Buffer
	for _, name := range modules {
		m := s.Modules[name]
		mStr := m.testString()

		// If we're the root module, we just write the output directly.
		if m.Addr.IsRoot() {
			buf.WriteString(mStr + "\n")
			continue
		}

		// We need to build out a string that resembles the not-quite-standard
		// format that terraform.State.String used to use, where there's a
		// "module." prefix but then just a chain of all of the module names
		// without any further "module." portions.
		buf.WriteString("module")
		for _, step := range m.Addr {
			buf.WriteByte('.')
			buf.WriteString(step.Name)
			if step.InstanceKey != addrs.NoKey {
				buf.WriteByte('[')
				buf.WriteString(step.InstanceKey.String())
				buf.WriteByte(']')
			}
		}
		buf.WriteString(":\n")

		s := bufio.NewScanner(strings.NewReader(mStr))
		for s.Scan() {
			text := s.Text()
			if text != "" {
				text = "  " + text
			}

			buf.WriteString(fmt.Sprintf("%s\n", text))
		}
	}

	return strings.TrimSpace(buf.String())
}

// testString is used to produce part of the output of State.String. It should
// never be used directly.
func (m *Module) testString() string {
	var buf bytes.Buffer

	if len(m.Resources) == 0 {
		buf.WriteString("<no state>")
	}

	// We use AbsResourceInstance here, even though everything belongs to
	// the same module, just because we have a sorting behavior defined
	// for those but not for just ResourceInstance.
	addrsOrder := make([]addrs.AbsResourceInstance, 0, len(m.Resources))
	for _, rs := range m.Resources {
		for ik := range rs.Instances {
			addrsOrder = append(addrsOrder, rs.Addr.Instance(ik).Absolute(addrs.RootModuleInstance))
		}
	}

	sort.Slice(addrsOrder, func(i, j int) bool {
		return addrsOrder[i].Less(addrsOrder[j])
	})

	for _, fakeAbsAddr := range addrsOrder {
		addr := fakeAbsAddr.Resource
		rs := m.Resource(addr.ContainingResource())
		is := m.ResourceInstance(addr)

		// Here we need to fake up a legacy-style address as the old state
		// types would've used, since that's what our tests against those
		// old types expect. The significant difference is that instancekey
		// is dot-separated rather than using index brackets.
		k := addr.ContainingResource().String()
		if addr.Key != addrs.NoKey {
			switch tk := addr.Key.(type) {
			case addrs.IntKey:
				k = fmt.Sprintf("%s.%d", k, tk)
			default:
				// No other key types existed for the legacy types, so we
				// can do whatever we want here. We'll just use our standard
				// syntax for these.
				k = k + tk.String()
			}
		}

		id := legacyInstanceObjectID(is.Current)

		taintStr := ""
		if is.Current != nil && is.Current.Status == ObjectTainted {
			taintStr = " (tainted)"
		}

		deposedStr := ""
		if len(is.Deposed) > 0 {
			deposedStr = fmt.Sprintf(" (%d deposed)", len(is.Deposed))
		}

		buf.WriteString(fmt.Sprintf("%s:%s%s\n", k, taintStr, deposedStr))
		buf.WriteString(fmt.Sprintf("  ID = %s\n", id))
		buf.WriteString(fmt.Sprintf("  provider = %s\n", rs.ProviderConfig.String()))

		// Attributes were a flatmap before, but are not anymore. To preserve
		// our old output as closely as possible we need to do a conversion
		// to flatmap. Normally we'd want to do this with schema for
		// accuracy, but for our purposes here it only needs to be approximate.
		// This should produce an identical result for most cases, though
		// in particular will differ in a few cases:
		//  - The keys used for elements in a set will be different
		//  - Values for attributes of type cty.DynamicPseudoType will be
		//    misinterpreted (but these weren't possible in old world anyway)
		var attributes map[string]string
		if obj := is.Current; obj != nil {
			switch {
			case obj.AttrsFlat != nil:
				// Easy (but increasingly unlikely) case: the state hasn't
				// actually been upgraded to the new form yet.
				attributes = obj.AttrsFlat
			case obj.AttrsJSON != nil:
				ty, err := ctyjson.ImpliedType(obj.AttrsJSON)
				if err == nil {
					val, err := ctyjson.Unmarshal(obj.AttrsJSON, ty)
					if err == nil {
						attributes = hcl2shim.FlatmapValueFromHCL2(val)
					}
				}
			}
		}
		attrKeys := make([]string, 0, len(attributes))
		for ak, _ := range attributes {
			if ak == "id" {
				continue
			}

			attrKeys = append(attrKeys, ak)
		}

		sort.Strings(attrKeys)

		for _, ak := range attrKeys {
			av := attributes[ak]
			buf.WriteString(fmt.Sprintf("  %s = %s\n", ak, av))
		}

		// CAUTION: Since deposed keys are now random strings instead of
		// incrementing integers, this result will not be deterministic
		// if there is more than one deposed object.
		for k, t := range is.Deposed {
			id := legacyInstanceObjectID(t)
			taintStr := ""
			if t.Status == ObjectTainted {
				taintStr = " (tainted)"
			}
			buf.WriteString(fmt.Sprintf("  Deposed ID %s = %s%s\n", k, id, taintStr))
		}

		if obj := is.Current; obj != nil && len(obj.Dependencies) > 0 {
			buf.WriteString(fmt.Sprintf("\n  Dependencies:\n"))
			for _, dep := range obj.Dependencies {
				buf.WriteString(fmt.Sprintf("    %s\n", dep.String()))
			}
		}
	}

	if len(m.OutputValues) > 0 {
		buf.WriteString("\nOutputs:\n\n")

		ks := make([]string, 0, len(m.OutputValues))
		for k := range m.OutputValues {
			ks = append(ks, k)
		}
		sort.Strings(ks)

		for _, k := range ks {
			v := m.OutputValues[k]
			lv := hcl2shim.ConfigValueFromHCL2(v.Value)
			switch vTyped := lv.(type) {
			case string:
				buf.WriteString(fmt.Sprintf("%s = %s\n", k, vTyped))
			case []interface{}:
				buf.WriteString(fmt.Sprintf("%s = %s\n", k, vTyped))
			case map[string]interface{}:
				var mapKeys []string
				for key := range vTyped {
					mapKeys = append(mapKeys, key)
				}
				sort.Strings(mapKeys)

				var mapBuf bytes.Buffer
				mapBuf.WriteString("{")
				for _, key := range mapKeys {
					mapBuf.WriteString(fmt.Sprintf("%s:%s ", key, vTyped[key]))
				}
				mapBuf.WriteString("}")

				buf.WriteString(fmt.Sprintf("%s = %s\n", k, mapBuf.String()))
			}
		}
	}

	return buf.String()
}

// legacyInstanceObjectID is a helper for extracting an object id value from
// an instance object in a way that approximates how we used to do this
// for the old state types. ID is no longer first-class, so this is preserved
// only for compatibility with old tests that include the id as part of their
// expected value.
func legacyInstanceObjectID(obj *ResourceInstanceObjectSrc) string {
	if obj == nil {
		return "<not created>"
	}

	if obj.AttrsJSON != nil {
		type WithID struct {
			ID string `json:"id"`
		}
		var withID WithID
		err := json.Unmarshal(obj.AttrsJSON, &withID)
		if err == nil {
			return withID.ID
		}
	} else if obj.AttrsFlat != nil {
		if flatID, exists := obj.AttrsFlat["id"]; exists {
			return flatID
		}
	}

	// For resource types created after we removed id as special there may
	// not actually be one at all. This is okay because older tests won't
	// encounter this, and new tests shouldn't be using ids.
	return "<none>"
}
<commit_msg>states: Support non-string primitives in state string representation<commit_after>package states

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"sort"
	"strings"

	ctyjson "github.com/zclconf/go-cty/cty/json"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/config/hcl2shim"
)

// String returns a rather-odd string representation of the entire state.
//
// This is intended to match the behavior of the older terraform.State.String
// method that is used in lots of existing tests. It should not be used in
// new tests: instead, use "cmp" to directly compare the state data structures
// and print out a diff if they do not match.
//
// This method should never be used in non-test code, whether directly by call
// or indirectly via a %s or %q verb in package fmt.
func (s *State) String() string {
	if s == nil {
		return "<nil>"
	}

	// sort the modules by name for consistent output
	modules := make([]string, 0, len(s.Modules))
	for m := range s.Modules {
		modules = append(modules, m)
	}
	sort.Strings(modules)

	var buf bytes.Buffer
	for _, name := range modules {
		m := s.Modules[name]
		mStr := m.testString()

		// If we're the root module, we just write the output directly.
		if m.Addr.IsRoot() {
			buf.WriteString(mStr + "\n")
			continue
		}

		// We need to build out a string that resembles the not-quite-standard
		// format that terraform.State.String used to use, where there's a
		// "module." prefix but then just a chain of all of the module names
		// without any further "module." portions.
		buf.WriteString("module")
		for _, step := range m.Addr {
			buf.WriteByte('.')
			buf.WriteString(step.Name)
			if step.InstanceKey != addrs.NoKey {
				buf.WriteByte('[')
				buf.WriteString(step.InstanceKey.String())
				buf.WriteByte(']')
			}
		}
		buf.WriteString(":\n")

		s := bufio.NewScanner(strings.NewReader(mStr))
		for s.Scan() {
			text := s.Text()
			if text != "" {
				text = "  " + text
			}

			buf.WriteString(fmt.Sprintf("%s\n", text))
		}
	}

	return strings.TrimSpace(buf.String())
}

// testString is used to produce part of the output of State.String. It should
// never be used directly.
func (m *Module) testString() string {
	var buf bytes.Buffer

	if len(m.Resources) == 0 {
		buf.WriteString("<no state>")
	}

	// We use AbsResourceInstance here, even though everything belongs to
	// the same module, just because we have a sorting behavior defined
	// for those but not for just ResourceInstance.
	addrsOrder := make([]addrs.AbsResourceInstance, 0, len(m.Resources))
	for _, rs := range m.Resources {
		for ik := range rs.Instances {
			addrsOrder = append(addrsOrder, rs.Addr.Instance(ik).Absolute(addrs.RootModuleInstance))
		}
	}

	sort.Slice(addrsOrder, func(i, j int) bool {
		return addrsOrder[i].Less(addrsOrder[j])
	})

	for _, fakeAbsAddr := range addrsOrder {
		addr := fakeAbsAddr.Resource
		rs := m.Resource(addr.ContainingResource())
		is := m.ResourceInstance(addr)

		// Here we need to fake up a legacy-style address as the old state
		// types would've used, since that's what our tests against those
		// old types expect. The significant difference is that instancekey
		// is dot-separated rather than using index brackets.
		k := addr.ContainingResource().String()
		if addr.Key != addrs.NoKey {
			switch tk := addr.Key.(type) {
			case addrs.IntKey:
				k = fmt.Sprintf("%s.%d", k, tk)
			default:
				// No other key types existed for the legacy types, so we
				// can do whatever we want here. We'll just use our standard
				// syntax for these.
				k = k + tk.String()
			}
		}

		id := legacyInstanceObjectID(is.Current)

		taintStr := ""
		if is.Current != nil && is.Current.Status == ObjectTainted {
			taintStr = " (tainted)"
		}

		deposedStr := ""
		if len(is.Deposed) > 0 {
			deposedStr = fmt.Sprintf(" (%d deposed)", len(is.Deposed))
		}

		buf.WriteString(fmt.Sprintf("%s:%s%s\n", k, taintStr, deposedStr))
		buf.WriteString(fmt.Sprintf("  ID = %s\n", id))
		buf.WriteString(fmt.Sprintf("  provider = %s\n", rs.ProviderConfig.String()))

		// Attributes were a flatmap before, but are not anymore. To preserve
		// our old output as closely as possible we need to do a conversion
		// to flatmap. Normally we'd want to do this with schema for
		// accuracy, but for our purposes here it only needs to be approximate.
		// This should produce an identical result for most cases, though
		// in particular will differ in a few cases:
		//  - The keys used for elements in a set will be different
		//  - Values for attributes of type cty.DynamicPseudoType will be
		//    misinterpreted (but these weren't possible in old world anyway)
		var attributes map[string]string
		if obj := is.Current; obj != nil {
			switch {
			case obj.AttrsFlat != nil:
				// Easy (but increasingly unlikely) case: the state hasn't
				// actually been upgraded to the new form yet.
				attributes = obj.AttrsFlat
			case obj.AttrsJSON != nil:
				ty, err := ctyjson.ImpliedType(obj.AttrsJSON)
				if err == nil {
					val, err := ctyjson.Unmarshal(obj.AttrsJSON, ty)
					if err == nil {
						attributes = hcl2shim.FlatmapValueFromHCL2(val)
					}
				}
			}
		}
		attrKeys := make([]string, 0, len(attributes))
		for ak, _ := range attributes {
			if ak == "id" {
				continue
			}

			attrKeys = append(attrKeys, ak)
		}

		sort.Strings(attrKeys)

		for _, ak := range attrKeys {
			av := attributes[ak]
			buf.WriteString(fmt.Sprintf("  %s = %s\n", ak, av))
		}

		// CAUTION: Since deposed keys are now random strings instead of
		// incrementing integers, this result will not be deterministic
		// if there is more than one deposed object.
		for k, t := range is.Deposed {
			id := legacyInstanceObjectID(t)
			taintStr := ""
			if t.Status == ObjectTainted {
				taintStr = " (tainted)"
			}
			buf.WriteString(fmt.Sprintf("  Deposed ID %s = %s%s\n", k, id, taintStr))
		}

		if obj := is.Current; obj != nil && len(obj.Dependencies) > 0 {
			buf.WriteString(fmt.Sprintf("\n  Dependencies:\n"))
			for _, dep := range obj.Dependencies {
				buf.WriteString(fmt.Sprintf("    %s\n", dep.String()))
			}
		}
	}

	if len(m.OutputValues) > 0 {
		buf.WriteString("\nOutputs:\n\n")

		ks := make([]string, 0, len(m.OutputValues))
		for k := range m.OutputValues {
			ks = append(ks, k)
		}
		sort.Strings(ks)

		for _, k := range ks {
			v := m.OutputValues[k]
			lv := hcl2shim.ConfigValueFromHCL2(v.Value)
			switch vTyped := lv.(type) {
			case string:
				buf.WriteString(fmt.Sprintf("%s = %s\n", k, vTyped))
			case []interface{}:
				buf.WriteString(fmt.Sprintf("%s = %s\n", k, vTyped))
			case map[string]interface{}:
				var mapKeys []string
				for key := range vTyped {
					mapKeys = append(mapKeys, key)
				}
				sort.Strings(mapKeys)

				var mapBuf bytes.Buffer
				mapBuf.WriteString("{")
				for _, key := range mapKeys {
					mapBuf.WriteString(fmt.Sprintf("%s:%s ", key, vTyped[key]))
				}
				mapBuf.WriteString("}")

				buf.WriteString(fmt.Sprintf("%s = %s\n", k, mapBuf.String()))
			default:
				buf.WriteString(fmt.Sprintf("%s = %#v\n", k, lv))
			}
		}
	}

	return buf.String()
}

// legacyInstanceObjectID is a helper for extracting an object id value from
// an instance object in a way that approximates how we used to do this
// for the old state types. ID is no longer first-class, so this is preserved
// only for compatibility with old tests that include the id as part of their
// expected value.
func legacyInstanceObjectID(obj *ResourceInstanceObjectSrc) string {
	if obj == nil {
		return "<not created>"
	}

	if obj.AttrsJSON != nil {
		type WithID struct {
			ID string `json:"id"`
		}
		var withID WithID
		err := json.Unmarshal(obj.AttrsJSON, &withID)
		if err == nil {
			return withID.ID
		}
	} else if obj.AttrsFlat != nil {
		if flatID, exists := obj.AttrsFlat["id"]; exists {
			return flatID
		}
	}

	// For resource types created after we removed id as special there may
	// not actually be one at all. This is okay because older tests won't
	// encounter this, and new tests shouldn't be using ids.
	return "<none>"
}
<|endoftext|>