<commit_before>package cgroups

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"

	"golang.org/x/sys/unix"

	specs "github.com/opencontainers/runtime-spec/specs-go"
)

func NewMemory(root string) *memoryController {
	return &memoryController{
		root: filepath.Join(root, string(Memory)),
	}
}

type memoryController struct {
	root string
}

func (m *memoryController) Name() Name {
	return Memory
}

func (m *memoryController) Path(path string) string {
	return filepath.Join(m.root, path)
}

func (m *memoryController) Create(path string, resources *specs.LinuxResources) error {
	if err := os.MkdirAll(m.Path(path), defaultDirPerm); err != nil {
		return err
	}
	if resources.Memory == nil {
		return nil
	}
	if resources.Memory.Kernel != nil {
		// Check if kernel memory is enabled
		// We have to limit the kernel memory here as it won't be accounted at all
		// until a limit is set on the cgroup and limit cannot be set once the
		// cgroup has children, or if there are already tasks in the cgroup.
		for _, i := range []int64{1, -1} {
			if err := ioutil.WriteFile(
				filepath.Join(m.Path(path), "memory.kmem.limit_in_bytes"),
				[]byte(strconv.FormatInt(i, 10)),
				defaultFilePerm,
			); err != nil {
				return checkEBUSY(err)
			}
		}
	}
	return m.set(path, getMemorySettings(resources))
}

func (m *memoryController) Update(path string, resources *specs.LinuxResources) error {
	if resources.Memory == nil {
		return nil
	}
	g := func(v *int64) bool {
		return v != nil && *v > 0
	}
	settings := getMemorySettings(resources)
	if g(resources.Memory.Limit) && g(resources.Memory.Swap) {
		// if the updated swap value is larger than the current memory limit set the swap changes first
		// then set the memory limit as swap must always be larger than the current limit
		current, err := readUint(filepath.Join(m.Path(path), "memory.limit_in_bytes"))
		if err != nil {
			return err
		}
		if current < uint64(*resources.Memory.Swap) {
			settings[0], settings[1] = settings[1], settings[0]
		}
	}
	return m.set(path, settings)
}

func (m *memoryController) Stat(path string, stats *Metrics) error {
	f, err := os.Open(filepath.Join(m.Path(path), "memory.stat"))
	if err != nil {
		return err
	}
	defer f.Close()
	stats.Memory = &MemoryStat{
		Usage:     &MemoryEntry{},
		Swap:      &MemoryEntry{},
		Kernel:    &MemoryEntry{},
		KernelTCP: &MemoryEntry{},
	}
	if err := m.parseStats(f, stats.Memory); err != nil {
		return err
	}
	for _, t := range []struct {
		module string
		entry  *MemoryEntry
	}{
		{
			module: "",
			entry:  stats.Memory.Usage,
		},
		{
			module: "memsw",
			entry:  stats.Memory.Swap,
		},
		{
			module: "kmem",
			entry:  stats.Memory.Kernel,
		},
		{
			module: "kmem.tcp",
			entry:  stats.Memory.KernelTCP,
		},
	} {
		for _, tt := range []struct {
			name  string
			value *uint64
		}{
			{
				name:  "usage_in_bytes",
				value: &t.entry.Usage,
			},
			{
				name:  "max_usage_in_bytes",
				value: &t.entry.Max,
			},
			{
				name:  "failcnt",
				value: &t.entry.Failcnt,
			},
			{
				name:  "limit_in_bytes",
				value: &t.entry.Limit,
			},
		} {
			parts := []string{"memory"}
			if t.module != "" {
				parts = append(parts, t.module)
			}
			parts = append(parts, tt.name)
			v, err := readUint(filepath.Join(m.Path(path), strings.Join(parts, ".")))
			if err != nil {
				return err
			}
			*tt.value = v
		}
	}
	return nil
}

func (m *memoryController) OOMEventFD(path string) (uintptr, error) {
	root := m.Path(path)
	f, err := os.Open(filepath.Join(root, "memory.oom_control"))
	if err != nil {
		return 0, err
	}
	defer f.Close()
	fd, _, serr := unix.RawSyscall(unix.SYS_EVENTFD2, 0, unix.FD_CLOEXEC, 0)
	if serr != 0 {
		return 0, serr
	}
	if err := writeEventFD(root, f.Fd(), fd); err != nil {
		unix.Close(int(fd))
		return 0, err
	}
	return fd, nil
}

func writeEventFD(root string, cfd, efd uintptr) error {
	f, err := os.OpenFile(filepath.Join(root, "cgroup.event_control"), os.O_WRONLY, 0)
	if err != nil {
		return err
	}
	_, err = f.WriteString(fmt.Sprintf("%d %d", efd, cfd))
	f.Close()
	return err
}

func (m *memoryController) parseStats(r io.Reader, stat *MemoryStat) error {
	var (
		raw  = make(map[string]uint64)
		sc   = bufio.NewScanner(r)
		line int
	)
	for sc.Scan() {
		if err := sc.Err(); err != nil {
			return err
		}
		key, v, err := parseKV(sc.Text())
		if err != nil {
			return fmt.Errorf("%d: %v", line, err)
		}
		raw[key] = v
		line++
	}
	stat.Cache = raw["cache"]
	stat.RSS = raw["rss"]
	stat.RSSHuge = raw["rss_huge"]
	stat.MappedFile = raw["mapped_file"]
	stat.Dirty = raw["dirty"]
	stat.Writeback = raw["writeback"]
	stat.PgPgIn = raw["pgpgin"]
	stat.PgPgOut = raw["pgpgout"]
	stat.PgFault = raw["pgfault"]
	stat.PgMajFault = raw["pgmajfault"]
	stat.InactiveAnon = raw["inactive_anon"]
	stat.ActiveAnon = raw["active_anon"]
	stat.InactiveFile = raw["inactive_file"]
	stat.ActiveFile = raw["active_file"]
	stat.Unevictable = raw["unevictable"]
	stat.HierarchicalMemoryLimit = raw["hierarchical_memory_limit"]
	stat.HierarchicalSwapLimit = raw["hierarchical_memsw_limit"]
	stat.TotalCache = raw["total_cache"]
	stat.TotalRSS = raw["total_rss"]
	stat.TotalRSSHuge = raw["total_rss_huge"]
	stat.TotalMappedFile = raw["total_mapped_file"]
	stat.TotalDirty = raw["total_dirty"]
	stat.TotalWriteback = raw["total_writeback"]
	stat.TotalPgPgIn = raw["total_pgpgin"]
	stat.TotalPgPgOut = raw["total_pgpgout"]
	stat.TotalPgFault = raw["total_pgfault"]
	stat.TotalPgMajFault = raw["total_pgmajfault"]
	stat.TotalInactiveAnon = raw["total_inactive_anon"]
	stat.TotalActiveAnon = raw["total_active_anon"]
	stat.TotalInactiveFile = raw["total_inactive_file"]
	stat.TotalActiveFile = raw["total_active_file"]
	stat.TotalUnevictable = raw["total_unevictable"]
	return nil
}

func (m *memoryController) set(path string, settings []memorySettings) error {
	for _, t := range settings {
		if t.value != nil {
			if err := ioutil.WriteFile(
				filepath.Join(m.Path(path), fmt.Sprintf("memory.%s", t.name)),
				[]byte(strconv.FormatInt(*t.value, 10)),
				defaultFilePerm,
			); err != nil {
				return err
			}
		}
	}
	return nil
}

type memorySettings struct {
	name  string
	value *int64
}

func getMemorySettings(resources *specs.LinuxResources) []memorySettings {
	mem := resources.Memory
	var swappiness *int64
	if mem.Swappiness != nil {
		v := int64(*mem.Swappiness)
		swappiness = &v
	}
	return []memorySettings{
		{
			name:  "limit_in_bytes",
			value: mem.Limit,
		},
		{
			name:  "memsw.limit_in_bytes",
			value: mem.Swap,
		},
		{
			name:  "kmem.limit_in_bytes",
			value: mem.Kernel,
		},
		{
			name:  "kmem.tcp.limit_in_bytes",
			value: mem.KernelTCP,
		},
		{
			name:  "oom_control",
			value: getOomControlValue(mem),
		},
		{
			name:  "swappiness",
			value: swappiness,
		},
	}
}

func checkEBUSY(err error) error {
	if pathErr, ok := err.(*os.PathError); ok {
		if errNo, ok := pathErr.Err.(syscall.Errno); ok {
			if errNo == unix.EBUSY {
				return fmt.Errorf(
					"failed to set memory.kmem.limit_in_bytes, because either tasks have already joined this cgroup or it has children")
			}
		}
	}
	return err
}

func getOomControlValue(mem *specs.LinuxMemory) *int64 {
	if mem.DisableOOMKiller != nil && *mem.DisableOOMKiller {
		i := int64(1)
		return &i
	}
	return nil
}
<commit_msg>Pass proper close-on-exec flags with eventfd()<commit_after>package cgroups

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"

	"golang.org/x/sys/unix"

	specs "github.com/opencontainers/runtime-spec/specs-go"
)

func NewMemory(root string) *memoryController {
	return &memoryController{
		root: filepath.Join(root, string(Memory)),
	}
}

type memoryController struct {
	root string
}

func (m *memoryController) Name() Name {
	return Memory
}

func (m *memoryController) Path(path string) string {
	return filepath.Join(m.root, path)
}

func (m *memoryController) Create(path string, resources *specs.LinuxResources) error {
	if err := os.MkdirAll(m.Path(path), defaultDirPerm); err != nil {
		return err
	}
	if resources.Memory == nil {
		return nil
	}
	if resources.Memory.Kernel != nil {
		// Check if kernel memory is enabled
		// We have to limit the kernel memory here as it won't be accounted at all
		// until a limit is set on the cgroup and limit cannot be set once the
		// cgroup has children, or if there are already tasks in the cgroup.
		for _, i := range []int64{1, -1} {
			if err := ioutil.WriteFile(
				filepath.Join(m.Path(path), "memory.kmem.limit_in_bytes"),
				[]byte(strconv.FormatInt(i, 10)),
				defaultFilePerm,
			); err != nil {
				return checkEBUSY(err)
			}
		}
	}
	return m.set(path, getMemorySettings(resources))
}

func (m *memoryController) Update(path string, resources *specs.LinuxResources) error {
	if resources.Memory == nil {
		return nil
	}
	g := func(v *int64) bool {
		return v != nil && *v > 0
	}
	settings := getMemorySettings(resources)
	if g(resources.Memory.Limit) && g(resources.Memory.Swap) {
		// if the updated swap value is larger than the current memory limit set the swap changes first
		// then set the memory limit as swap must always be larger than the current limit
		current, err := readUint(filepath.Join(m.Path(path), "memory.limit_in_bytes"))
		if err != nil {
			return err
		}
		if current < uint64(*resources.Memory.Swap) {
			settings[0], settings[1] = settings[1], settings[0]
		}
	}
	return m.set(path, settings)
}

func (m *memoryController) Stat(path string, stats *Metrics) error {
	f, err := os.Open(filepath.Join(m.Path(path), "memory.stat"))
	if err != nil {
		return err
	}
	defer f.Close()
	stats.Memory = &MemoryStat{
		Usage:     &MemoryEntry{},
		Swap:      &MemoryEntry{},
		Kernel:    &MemoryEntry{},
		KernelTCP: &MemoryEntry{},
	}
	if err := m.parseStats(f, stats.Memory); err != nil {
		return err
	}
	for _, t := range []struct {
		module string
		entry  *MemoryEntry
	}{
		{
			module: "",
			entry:  stats.Memory.Usage,
		},
		{
			module: "memsw",
			entry:  stats.Memory.Swap,
		},
		{
			module: "kmem",
			entry:  stats.Memory.Kernel,
		},
		{
			module: "kmem.tcp",
			entry:  stats.Memory.KernelTCP,
		},
	} {
		for _, tt := range []struct {
			name  string
			value *uint64
		}{
			{
				name:  "usage_in_bytes",
				value: &t.entry.Usage,
			},
			{
				name:  "max_usage_in_bytes",
				value: &t.entry.Max,
			},
			{
				name:  "failcnt",
				value: &t.entry.Failcnt,
			},
			{
				name:  "limit_in_bytes",
				value: &t.entry.Limit,
			},
		} {
			parts := []string{"memory"}
			if t.module != "" {
				parts = append(parts, t.module)
			}
			parts = append(parts, tt.name)
			v, err := readUint(filepath.Join(m.Path(path), strings.Join(parts, ".")))
			if err != nil {
				return err
			}
			*tt.value = v
		}
	}
	return nil
}

func (m *memoryController) OOMEventFD(path string) (uintptr, error) {
	root := m.Path(path)
	f, err := os.Open(filepath.Join(root, "memory.oom_control"))
	if err != nil {
		return 0, err
	}
	defer f.Close()
	fd, _, serr := unix.RawSyscall(unix.SYS_EVENTFD2, 0, unix.EFD_CLOEXEC, 0)
	if serr != 0 {
		return 0, serr
	}
	if err := writeEventFD(root, f.Fd(), fd); err != nil {
		unix.Close(int(fd))
		return 0, err
	}
	return fd, nil
}

func writeEventFD(root string, cfd, efd uintptr) error {
	f, err := os.OpenFile(filepath.Join(root, "cgroup.event_control"), os.O_WRONLY, 0)
	if err != nil {
		return err
	}
	_, err = f.WriteString(fmt.Sprintf("%d %d", efd, cfd))
	f.Close()
	return err
}

func (m *memoryController) parseStats(r io.Reader, stat *MemoryStat) error {
	var (
		raw  = make(map[string]uint64)
		sc   = bufio.NewScanner(r)
		line int
	)
	for sc.Scan() {
		if err := sc.Err(); err != nil {
			return err
		}
		key, v, err := parseKV(sc.Text())
		if err != nil {
			return fmt.Errorf("%d: %v", line, err)
		}
		raw[key] = v
		line++
	}
	stat.Cache = raw["cache"]
	stat.RSS = raw["rss"]
	stat.RSSHuge = raw["rss_huge"]
	stat.MappedFile = raw["mapped_file"]
	stat.Dirty = raw["dirty"]
	stat.Writeback = raw["writeback"]
	stat.PgPgIn = raw["pgpgin"]
	stat.PgPgOut = raw["pgpgout"]
	stat.PgFault = raw["pgfault"]
	stat.PgMajFault = raw["pgmajfault"]
	stat.InactiveAnon = raw["inactive_anon"]
	stat.ActiveAnon = raw["active_anon"]
	stat.InactiveFile = raw["inactive_file"]
	stat.ActiveFile = raw["active_file"]
	stat.Unevictable = raw["unevictable"]
	stat.HierarchicalMemoryLimit = raw["hierarchical_memory_limit"]
	stat.HierarchicalSwapLimit = raw["hierarchical_memsw_limit"]
	stat.TotalCache = raw["total_cache"]
	stat.TotalRSS = raw["total_rss"]
	stat.TotalRSSHuge = raw["total_rss_huge"]
	stat.TotalMappedFile = raw["total_mapped_file"]
	stat.TotalDirty = raw["total_dirty"]
	stat.TotalWriteback = raw["total_writeback"]
	stat.TotalPgPgIn = raw["total_pgpgin"]
	stat.TotalPgPgOut = raw["total_pgpgout"]
	stat.TotalPgFault = raw["total_pgfault"]
	stat.TotalPgMajFault = raw["total_pgmajfault"]
	stat.TotalInactiveAnon = raw["total_inactive_anon"]
	stat.TotalActiveAnon = raw["total_active_anon"]
	stat.TotalInactiveFile = raw["total_inactive_file"]
	stat.TotalActiveFile = raw["total_active_file"]
	stat.TotalUnevictable = raw["total_unevictable"]
	return nil
}

func (m *memoryController) set(path string, settings []memorySettings) error {
	for _, t := range settings {
		if t.value != nil {
			if err := ioutil.WriteFile(
				filepath.Join(m.Path(path), fmt.Sprintf("memory.%s", t.name)),
				[]byte(strconv.FormatInt(*t.value, 10)),
				defaultFilePerm,
			); err != nil {
				return err
			}
		}
	}
	return nil
}

type memorySettings struct {
	name  string
	value *int64
}

func getMemorySettings(resources *specs.LinuxResources) []memorySettings {
	mem := resources.Memory
	var swappiness *int64
	if mem.Swappiness != nil {
		v := int64(*mem.Swappiness)
		swappiness = &v
	}
	return []memorySettings{
		{
			name:  "limit_in_bytes",
			value: mem.Limit,
		},
		{
			name:  "memsw.limit_in_bytes",
			value: mem.Swap,
		},
		{
			name:  "kmem.limit_in_bytes",
			value: mem.Kernel,
		},
		{
			name:  "kmem.tcp.limit_in_bytes",
			value: mem.KernelTCP,
		},
		{
			name:  "oom_control",
			value: getOomControlValue(mem),
		},
		{
			name:  "swappiness",
			value: swappiness,
		},
	}
}

func checkEBUSY(err error) error {
	if pathErr, ok := err.(*os.PathError); ok {
		if errNo, ok := pathErr.Err.(syscall.Errno); ok {
			if errNo == unix.EBUSY {
				return fmt.Errorf(
					"failed to set memory.kmem.limit_in_bytes, because either tasks have already joined this cgroup or it has children")
			}
		}
	}
	return err
}

func getOomControlValue(mem *specs.LinuxMemory) *int64 {
	if mem.DisableOOMKiller != nil && *mem.DisableOOMKiller {
		i := int64(1)
		return &i
	}
	return nil
}
<|endoftext|>


<commit_before>package controller

/*
Copyright 2019 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"context"

	"github.com/crunchydata/postgres-operator/config"
	"github.com/crunchydata/postgres-operator/operator"
	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

// NamespaceController holds the connections for the controller
type NamespaceController struct {
	NamespaceClient        *rest.RESTClient
	NamespaceClientset     *kubernetes.Clientset
	Ctx                    context.Context
	ThePodController       *PodController
	TheJobController       *JobController
	ThePgpolicyController  *PgpolicyController
	ThePgbackupController  *PgbackupController
	ThePgreplicaController *PgreplicaController
	ThePgclusterController *PgclusterController
	ThePgtaskController    *PgtaskController
}

// Run starts a namespace resource controller
func (c *NamespaceController) Run() error {

	err := c.watchNamespaces(c.Ctx)
	if err != nil {
		log.Errorf("Failed to register watch for namespace resource: %v", err)
		return err
	}

	<-c.Ctx.Done()
	return c.Ctx.Err()
}

// watchNamespaces is the event loop for namespace resources
func (c *NamespaceController) watchNamespaces(ctx context.Context) error {
	log.Info("starting namespace controller")

	//watch all namespaces
	ns := ""

	source := cache.NewListWatchFromClient(
		c.NamespaceClientset.CoreV1().RESTClient(),
		"namespaces",
		ns,
		fields.Everything())

	_, controller := cache.NewInformer(
		source,

		// The object type.
		&v1.Namespace{},

		// resyncPeriod
		// Every resyncPeriod, all resources in the cache will retrigger events.
		// Set to 0 to disable the resync.
		0,

		// Your custom resource event handlers.
		cache.ResourceEventHandlerFuncs{
			AddFunc:    c.onAdd,
			UpdateFunc: c.onUpdate,
			DeleteFunc: c.onDelete,
		})

	go controller.Run(ctx.Done())

	return nil
}

func (c *NamespaceController) onAdd(obj interface{}) {
	newNs := obj.(*v1.Namespace)

	log.Debugf("[NamespaceController] OnAdd ns=%s", newNs.ObjectMeta.SelfLink)
	labels := newNs.GetObjectMeta().GetLabels()
	if labels[config.LABEL_VENDOR] != config.LABEL_CRUNCHY || labels[config.LABEL_PGO_INSTALLATION_NAME] != operator.InstallationName {
		log.Debugf("NamespaceController: onAdd skipping namespace that is not crunchydata or not belonging to this Operator installation %s", newNs.ObjectMeta.SelfLink)
		return
	} else {
		log.Debugf("NamespaceController: onAdd crunchy namespace %s created", newNs.ObjectMeta.SelfLink)
		c.ThePodController.SetupWatch(newNs.Name)
		c.TheJobController.SetupWatch(newNs.Name)
		c.ThePgpolicyController.SetupWatch(newNs.Name)
		c.ThePgbackupController.SetupWatch(newNs.Name)
		c.ThePgreplicaController.SetupWatch(newNs.Name)
		c.ThePgclusterController.SetupWatch(newNs.Name)
		c.ThePgtaskController.SetupWatch(newNs.Name)
	}

}

// onUpdate is called when a pgcluster is updated
func (c *NamespaceController) onUpdate(oldObj, newObj interface{}) {
	//oldNs := oldObj.(*v1.Namespace)
	newNs := newObj.(*v1.Namespace)
	log.Debugf("[NamespaceController] onUpdate ns=%s", newNs.ObjectMeta.SelfLink)

	labels := newNs.GetObjectMeta().GetLabels()
	if labels[config.LABEL_VENDOR] != config.LABEL_CRUNCHY {
		log.Debugf("NamespaceController: onUpdate skipping namespace that is not crunchydata %s", newNs.ObjectMeta.SelfLink)
		return
	} else {
		log.Debugf("NamespaceController: onUpdate crunchy namespace updated %s", newNs.ObjectMeta.SelfLink)
		c.ThePodController.SetupWatch(newNs.Name)
		c.TheJobController.SetupWatch(newNs.Name)
		c.ThePgpolicyController.SetupWatch(newNs.Name)
		c.ThePgbackupController.SetupWatch(newNs.Name)
		c.ThePgreplicaController.SetupWatch(newNs.Name)
		c.ThePgclusterController.SetupWatch(newNs.Name)
		c.ThePgtaskController.SetupWatch(newNs.Name)
	}

}

func (c *NamespaceController) onDelete(obj interface{}) {
	ns := obj.(*v1.Namespace)

	log.Debugf("[NamespaceController] onDelete ns=%s", ns.ObjectMeta.SelfLink)
	labels := ns.GetObjectMeta().GetLabels()
	if labels[config.LABEL_VENDOR] != config.LABEL_CRUNCHY {
		log.Debugf("NamespaceController: onDelete skipping namespace that is not crunchydata %s", ns.ObjectMeta.SelfLink)
		return
	} else {
		log.Debugf("NamespaceController: onDelete crunchy operator namespace %s is deleted", ns.ObjectMeta.SelfLink)
	}

}
<commit_msg>Added additional label check to support multiple operator deployments on a single kube/openshift cluster.<commit_after>package controller

/*
Copyright 2019 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"context"

	"github.com/crunchydata/postgres-operator/config"
	"github.com/crunchydata/postgres-operator/operator"
	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
)

// NamespaceController holds the connections for the controller
type NamespaceController struct {
	NamespaceClient        *rest.RESTClient
	NamespaceClientset     *kubernetes.Clientset
	Ctx                    context.Context
	ThePodController       *PodController
	TheJobController       *JobController
	ThePgpolicyController  *PgpolicyController
	ThePgbackupController  *PgbackupController
	ThePgreplicaController *PgreplicaController
	ThePgclusterController *PgclusterController
	ThePgtaskController    *PgtaskController
}

// Run starts a namespace resource controller
func (c *NamespaceController) Run() error {

	err := c.watchNamespaces(c.Ctx)
	if err != nil {
		log.Errorf("Failed to register watch for namespace resource: %v", err)
		return err
	}

	<-c.Ctx.Done()
	return c.Ctx.Err()
}

// watchNamespaces is the event loop for namespace resources
func (c *NamespaceController) watchNamespaces(ctx context.Context) error {
	log.Info("starting namespace controller")

	//watch all namespaces
	ns := ""

	source := cache.NewListWatchFromClient(
		c.NamespaceClientset.CoreV1().RESTClient(),
		"namespaces",
		ns,
		fields.Everything())

	_, controller := cache.NewInformer(
		source,

		// The object type.
		&v1.Namespace{},

		// resyncPeriod
		// Every resyncPeriod, all resources in the cache will retrigger events.
		// Set to 0 to disable the resync.
		0,

		// Your custom resource event handlers.
		cache.ResourceEventHandlerFuncs{
			AddFunc:    c.onAdd,
			UpdateFunc: c.onUpdate,
			DeleteFunc: c.onDelete,
		})

	go controller.Run(ctx.Done())

	return nil
}

func (c *NamespaceController) onAdd(obj interface{}) {
	newNs := obj.(*v1.Namespace)

	log.Debugf("[NamespaceController] OnAdd ns=%s", newNs.ObjectMeta.SelfLink)
	labels := newNs.GetObjectMeta().GetLabels()
	if labels[config.LABEL_VENDOR] != config.LABEL_CRUNCHY || labels[config.LABEL_PGO_INSTALLATION_NAME] != operator.InstallationName {
		log.Debugf("NamespaceController: onAdd skipping namespace that is not crunchydata or not belonging to this Operator installation %s", newNs.ObjectMeta.SelfLink)
		return
	} else {
		log.Debugf("NamespaceController: onAdd crunchy namespace %s created", newNs.ObjectMeta.SelfLink)
		c.ThePodController.SetupWatch(newNs.Name)
		c.TheJobController.SetupWatch(newNs.Name)
		c.ThePgpolicyController.SetupWatch(newNs.Name)
		c.ThePgbackupController.SetupWatch(newNs.Name)
		c.ThePgreplicaController.SetupWatch(newNs.Name)
		c.ThePgclusterController.SetupWatch(newNs.Name)
		c.ThePgtaskController.SetupWatch(newNs.Name)
	}

}

// onUpdate is called when a pgcluster is updated
func (c *NamespaceController) onUpdate(oldObj, newObj interface{}) {
	//oldNs := oldObj.(*v1.Namespace)
	newNs := newObj.(*v1.Namespace)
	log.Debugf("[NamespaceController] onUpdate ns=%s", newNs.ObjectMeta.SelfLink)

	labels := newNs.GetObjectMeta().GetLabels()
	if labels[config.LABEL_VENDOR] != config.LABEL_CRUNCHY || labels[config.LABEL_PGO_INSTALLATION_NAME] != operator.InstallationName {
		log.Debugf("NamespaceController: onUpdate skipping namespace that is not crunchydata %s", newNs.ObjectMeta.SelfLink)
		return
	} else {
		log.Debugf("NamespaceController: onUpdate crunchy namespace updated %s", newNs.ObjectMeta.SelfLink)
		c.ThePodController.SetupWatch(newNs.Name)
		c.TheJobController.SetupWatch(newNs.Name)
		c.ThePgpolicyController.SetupWatch(newNs.Name)
		c.ThePgbackupController.SetupWatch(newNs.Name)
		c.ThePgreplicaController.SetupWatch(newNs.Name)
		c.ThePgclusterController.SetupWatch(newNs.Name)
		c.ThePgtaskController.SetupWatch(newNs.Name)
	}

}

func (c *NamespaceController) onDelete(obj interface{}) {
	ns := obj.(*v1.Namespace)

	log.Debugf("[NamespaceController] onDelete ns=%s", ns.ObjectMeta.SelfLink)
	labels := ns.GetObjectMeta().GetLabels()
	if labels[config.LABEL_VENDOR] != config.LABEL_CRUNCHY {
		log.Debugf("NamespaceController: onDelete skipping namespace that is not crunchydata %s", ns.ObjectMeta.SelfLink)
		return
	} else {
		log.Debugf("NamespaceController: onDelete crunchy operator namespace %s is deleted", ns.ObjectMeta.SelfLink)
	}

}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"bytes"
	"fmt"
	"github.com/op/go-logging"
	stdlog "log"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	log     = logging.MustGetLogger("main")
	rlogger RuntimeLogger
	dlogger DownloadsLogger
)

type RuntimeLogger struct {
	f *os.File
}

type DownloadsLogger struct {
	sync.RWMutex
	l *stdlog.Logger
	f *os.File
}

// ReloadLogs will reopen the logs to allow rotations
func ReloadLogs() {
	ReloadRuntimeLogs()
	ReloadDownloadLogs()
}

func ReloadRuntimeLogs() {
	logging.SetFormatter(logging.MustStringFormatter("%{time:2006/01/02 15:04:05.000 MST} %{message}"))
	logFlags := 0

	if os.Getenv("DEBUG") != "" {
		logFlags = stdlog.Lshortfile
		logging.SetLevel(logging.DEBUG, "main")
	} else {
		logging.SetLevel(logging.INFO, "main")
	}

	logColor := false

	stat, _ := os.Stdout.Stat()
	if (stat.Mode() & os.ModeCharDevice) != 0 {
		logColor = true //TODO make it optionnal
	}

	if rlogger.f != nil {
		rlogger.f.Close()
	} else {
		rlogger.f = os.Stderr
	}

	if runLog != "" {
		var err error
		rlogger.f, err = os.OpenFile(runLog, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Cannot open log file for writing")
			rlogger.f = os.Stderr
		} else {
			logColor = false
		}
	}

	logBackend := logging.NewLogBackend(rlogger.f, "", logFlags)
	logBackend.Color = logColor

	logging.SetBackend(logBackend)
}

func ReloadDownloadLogs() {
	dlogger.Lock()
	defer dlogger.Unlock()

	if GetConfig().LogDir == "" {
		if dlogger.f != nil {
			dlogger.f.Close()
		}
		dlogger.f = nil
		dlogger.l = nil
		return
	}

	logfile := GetConfig().LogDir + "/downloads.log"
	createHeader := true

	s, err := os.Stat(logfile)
	if err == nil && s.Size() > 0 {
		createHeader = false
	}

	if dlogger.f != nil {
		dlogger.f.Close()
	}
	dlogger.f, err = os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)

	if err != nil {
		log.Critical("Warning: cannot open log file %s", logfile)
		return
	}

	if createHeader {
		var buf bytes.Buffer
		hostname, _ := os.Hostname()
		fmt.Fprintf(&buf, "# Log file created at: %s\n", time.Now().Format("2006/01/02 15:04:05"))
		fmt.Fprintf(&buf, "# Running on machine: %s\n", hostname)
		fmt.Fprintf(&buf, "# Binary: Built with %s %s for %s/%s\n", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH)
		dlogger.f.Write(buf.Bytes())
	}

	dlogger.l = stdlog.New(dlogger.f, "", stdlog.Ldate|stdlog.Lmicroseconds)
}

// This function will write a download result in the logs.
func logDownload(typ string, statuscode int, p *MirrorlistPage, err error) {
	dlogger.RLock()
	defer dlogger.RUnlock()

	if dlogger.l == nil {
		// Logs are disabled
		return
	}

	if statuscode == 302 || statuscode == 200 {
		var distance, countries string
		m := p.MirrorList[0]
		distance = strconv.FormatFloat(float64(m.Distance), 'f', 2, 32)
		countries = strings.Join(m.CountryFields, ",")
		fallback := ""
		if p.Fallback == true {
			fallback = " fallback:true"
		}
		sameASNum := ""
		if m.Asnum > 0 && m.Asnum == p.ClientInfo.ASNum {
			sameASNum = "same"
		}

		dlogger.l.Printf("%s %d \"%s\" ip:%s mirror:%s%s %sasn:%d distance:%skm countries:%s",
			typ, statuscode, p.FileInfo.Path, p.IP, m.ID, fallback, sameASNum, m.Asnum, distance, countries)
	} else if statuscode == 404 {
		dlogger.l.Printf("%s 404 \"%s\" %s", typ, p.FileInfo.Path, p.IP)
	} else if statuscode == 500 {
		mirrorID := "unknown"
		if len(p.MirrorList) > 0 {
			mirrorID = p.MirrorList[0].ID
		}
		dlogger.l.Printf("%s 500 \"%s\" ip:%s mirror:%s error:%s", typ, p.FileInfo.Path, p.IP, mirrorID, err.Error())
	} else {
		dlogger.l.Printf("%s %d \"%s\" ip:%s error:%s", typ, statuscode, p.FileInfo.Path, p.IP, err.Error())
	}
}
<commit_msg>Improve the logger output in debug and non-debug<commit_after>// Copyright (c) 2014 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"bytes"
	"fmt"
	"github.com/op/go-logging"
	stdlog "log"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	log     = logging.MustGetLogger("main")
	rlogger RuntimeLogger
	dlogger DownloadsLogger
)

type RuntimeLogger struct {
	f *os.File
}

type DownloadsLogger struct {
	sync.RWMutex
	l *stdlog.Logger
	f *os.File
}

// ReloadLogs will reopen the logs to allow rotations
func ReloadLogs() {
	ReloadRuntimeLogs()
	ReloadDownloadLogs()
}

func ReloadRuntimeLogs() {
	logColor := false

	stat, _ := os.Stdout.Stat()
	if (stat.Mode() & os.ModeCharDevice) != 0 {
		logColor = true //TODO make it optionnal
	}

	if rlogger.f != nil {
		rlogger.f.Close()
	} else {
		rlogger.f = os.Stderr
	}

	if runLog != "" {
		var err error
		rlogger.f, err = os.OpenFile(runLog, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Cannot open log file for writing")
			rlogger.f = os.Stderr
		} else {
			logColor = false
		}
	}

	logBackend := logging.NewLogBackend(rlogger.f, "", 0)
	logBackend.Color = logColor

	logging.SetBackend(logBackend)

	if debug {
		logging.SetFormatter(logging.MustStringFormatter("%{shortfile:-20s}%{time:2006/01/02 15:04:05.000 MST} %{message}"))
		logging.SetLevel(logging.DEBUG, "main")
	} else {
		logging.SetFormatter(logging.MustStringFormatter("%{time:2006/01/02 15:04:05.000 MST} %{message}"))
		logging.SetLevel(logging.INFO, "main")
	}
}

func ReloadDownloadLogs() {
	dlogger.Lock()
	defer dlogger.Unlock()

	if GetConfig().LogDir == "" {
		if dlogger.f != nil {
			dlogger.f.Close()
		}
		dlogger.f = nil
		dlogger.l = nil
		return
	}

	logfile := GetConfig().LogDir + "/downloads.log"
	createHeader := true

	s, err := os.Stat(logfile)
	if err == nil && s.Size() > 0 {
		createHeader = false
	}

	if dlogger.f != nil {
		dlogger.f.Close()
	}
	dlogger.f, err = os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)

	if err != nil {
		log.Critical("Warning: cannot open log file %s", logfile)
		return
	}

	if createHeader {
		var buf bytes.Buffer
		hostname, _ := os.Hostname()
		fmt.Fprintf(&buf, "# Log file created at: %s\n", time.Now().Format("2006/01/02 15:04:05"))
		fmt.Fprintf(&buf, "# Running on machine: %s\n", hostname)
		fmt.Fprintf(&buf, "# Binary: Built with %s %s for %s/%s\n", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH)
		dlogger.f.Write(buf.Bytes())
	}

	dlogger.l = stdlog.New(dlogger.f, "", stdlog.Ldate|stdlog.Lmicroseconds)
}

// This function will write a download result in the logs.
func logDownload(typ string, statuscode int, p *MirrorlistPage, err error) {
	dlogger.RLock()
	defer dlogger.RUnlock()

	if dlogger.l == nil {
		// Logs are disabled
		return
	}

	if statuscode == 302 || statuscode == 200 {
		var distance, countries string
		m := p.MirrorList[0]
		distance = strconv.FormatFloat(float64(m.Distance), 'f', 2, 32)
		countries = strings.Join(m.CountryFields, ",")
		fallback := ""
		if p.Fallback == true {
			fallback = " fallback:true"
		}
		sameASNum := ""
		if m.Asnum > 0 && m.Asnum == p.ClientInfo.ASNum {
			sameASNum = "same"
		}

		dlogger.l.Printf("%s %d \"%s\" ip:%s mirror:%s%s %sasn:%d distance:%skm countries:%s",
			typ, statuscode, p.FileInfo.Path, p.IP, m.ID, fallback, sameASNum, m.Asnum, distance, countries)
	} else if statuscode == 404 {
		dlogger.l.Printf("%s 404 \"%s\" %s", typ, p.FileInfo.Path, p.IP)
	} else if statuscode == 500 {
		mirrorID := "unknown"
		if len(p.MirrorList) > 0 {
			mirrorID = p.MirrorList[0].ID
		}
		dlogger.l.Printf("%s 500 \"%s\" ip:%s mirror:%s error:%s", typ, p.FileInfo.Path, p.IP, mirrorID, err.Error())
	} else {
		dlogger.l.Printf("%s %d \"%s\" ip:%s error:%s", typ, statuscode, p.FileInfo.Path, p.IP, err.Error())
	}
}
<|endoftext|>


<commit_before>// Package contains the entry point for the commitfmt command.
package main

import (
	"fmt"
	"github.com/gcurtis/commitfmt/rules"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, "You must provide a path to a file containing"+
			" the commit message.")
		os.Exit(1)
	}

	path := os.Args[1]
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't open file \"%s\".\n", path)
		os.Exit(1)
	}
	msg := string(bytes)

	report := runRules(msg)
	fmt.Println(report.string())
	if len(report.violations) > 0 {
		os.Exit(1)
	}
}

// runRules parses a commit message and then enforces every rule found in the
// rules package.
func runRules(msg string) (rep *report) {
	msg = strings.TrimSpace(msg)
	rep = &report{msg: msg}
	subject, body := parseMsg(msg)

	for _, rule := range rules.All {
		violations := rule.Enforce(subject, body)
		rep.append(violations...)
	}

	return
}

// parseMsg parses a message by breaking it up into a subject and a body.
func parseMsg(msg string) (subject string, body string) {
	split := strings.SplitN(msg, "\n\n", 2)
	subject = split[0]
	if len(split) > 1 {
		body = split[1]
	}
	return
}
<commit_msg>Strip comment lines starting with #<commit_after>// Package contains the entry point for the commitfmt command.
package main

import (
	"bytes"
	"fmt"
	"github.com/gcurtis/commitfmt/rules"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, "You must provide a path to a file containing"+
			" the commit message.")
		os.Exit(1)
	}

	path := os.Args[1]
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't open file \"%s\".\n", path)
		os.Exit(1)
	}
	msg := string(bytes)

	report := runRules(msg)
	fmt.Println(report.string())
	if len(report.violations) > 0 {
		os.Exit(1)
	}
}

// runRules parses a commit message and then enforces every rule found in the
// rules package.
func runRules(msg string) (rep *report) {
	msg = strings.TrimSpace(msg)
	rep = &report{msg: msg}
	subject, body := parseMsg(msg)

	for _, rule := range rules.All {
		violations := rule.Enforce(subject, body)
		rep.append(violations...)
	}

	return
}

// parseMsg parses a message by breaking it up into a subject and a body.
func parseMsg(msg string) (subject string, body string) {
	remComments := bytes.Buffer{}
	split := strings.SplitAfter(msg, "\n")
	for _, line := range split {
		trim := strings.TrimSpace(line)
		if !strings.HasPrefix(trim, "#") {
			remComments.WriteString(line)
		}
	}

	split = strings.SplitN(strings.TrimSpace(remComments.String()), "\n\n", 2)
	subject = split[0]
	if len(split) > 1 {
		body = split[1]
	}
	return
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	resp, err := http.PostForm("https://hr.hiyes.tw:443/getMessage.php",
        url.Values{"mid": {event.ReplyToken}, "message": {message.Text}})
    if err != nil {
        fmt.Println(err)
    } else {
        body, _ := ioutil.ReadAll(resp.Body)
        fmt.Println("POST OK: ", string(body), resp)
    }

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(event.ReplyToken+"---"+message.Text+" OK!")).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<commit_msg>add URL<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"net/url"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	resp, err := http.PostForm("https://hr.hiyes.tw:443/getMessage.php",
        url.Values{"mid": {event.ReplyToken}, "message": {message.Text}})
    if err != nil {
        fmt.Println(err)
    } else {
        body, _ := ioutil.ReadAll(resp.Body)
        fmt.Println("POST OK: ", string(body), resp)
    }

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(event.ReplyToken+"---"+message.Text+" OK!")).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	_ "net/http/pprof"
	"os"
	"os/signal"

	"github.com/Sirupsen/logrus"
	"github.com/redBorder/rbforwarder"
	"github.com/redBorder/rbforwarder/senders/httpsender"
	"github.com/x-cray/logrus-prefixed-formatter"
)

const (
	defaultQueueSize = 10000
	defaultWorkers   = 1
	defaultRetries   = 0
	defaultBackoff   = 2
)

var (
	configFile *string
	debug      *bool
	logger     *logrus.Entry
)

func init() {
	configFile = flag.String("config", "", "Config file")
	debug = flag.Bool("debug", false, "Show debug info")

	flag.Parse()

	if len(*configFile) == 0 {
		fmt.Println("No config file provided")
		flag.Usage()
		os.Exit(1)
	}

	log := logrus.New()
	// Show debug info if required
	if *debug {
		log.Level = logrus.DebugLevel
	}
	log.Formatter = new(prefixed.TextFormatter)

	logger = log.WithFields(logrus.Fields{
		"prefix": "k2http",
	})
}

func main() {

	// Load the configuration from file
	configData, err := LoadConfigFile(*configFile)
	if err != nil {
		logger.Fatal(err)
	}

	logger.Debug("Showing debug info")

	// Capture ctrl-c
	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	// Parse the backend configuration
	rbForwarderConfig := rbforwarder.Config{}

	// Get number of workers
	if workers, ok := configData.Backend["workers"].(int); ok {
		rbForwarderConfig.Workers = workers
	} else {
		rbForwarderConfig.Workers = defaultWorkers
	}

	// Get number of retries per message
	if retries, ok := configData.Backend["retries"].(int); ok {
		rbForwarderConfig.Retries = retries
	} else {
		rbForwarderConfig.Retries = defaultRetries
	}

	// Time to wait between retries
	if backoff, ok := configData.Backend["backoff"].(int); ok {
		rbForwarderConfig.Backoff = backoff
	} else {
		rbForwarderConfig.Backoff = defaultBackoff
	}

	// Get queue size
	if queue, ok := configData.Backend["queue"].(int); ok {
		rbForwarderConfig.QueueSize = queue
	} else {
		rbForwarderConfig.QueueSize = defaultQueueSize
	}

	// Get max message rate
	if maxMessages, ok := configData.Backend["max_messages"].(int); ok {
		rbForwarderConfig.MaxMessages = maxMessages
	}

	// Get max bytes rate
	if maxBytes, ok := configData.Backend["max_bytes"].(int); ok {
		rbForwarderConfig.MaxBytes = maxBytes
	}

	// Show debug info
	if *debug {
		rbForwarderConfig.Debug = true
	}

	// Get the interval to show message rate
	if interval, ok := configData.Backend["showcounter"].(int); ok {
		rbForwarderConfig.ShowCounter = interval
	}

	// Create forwarder
	forwarder := rbforwarder.NewRBForwarder(rbForwarderConfig)

	// Initialize kafka
	kafka := new(KafkaConsumer)
	kafka.ParseKafkaConfig(configData.Kafka)
	kafka.backend = forwarder

	// Get the HTTP sender helper
	httpSenderHelper := httpsender.NewHelper(configData.HTTP)
	forwarder.SetSenderHelper(httpSenderHelper)

	// Start the backend
	forwarder.Start()

	// Wait for ctrl-c to close the consumer
	go func() {
		<-ctrlc
		forwarder.Close()
		kafka.Close()
	}()

	// Start getting messages
	kafka.Start()

	defer recoverPanic()
}
<commit_msg>Close Kafka Consumer prior to rbforwarder<commit_after>package main

import (
	"flag"
	"fmt"
	_ "net/http/pprof"
	"os"
	"os/signal"

	"github.com/Sirupsen/logrus"
	"github.com/redBorder/rbforwarder"
	"github.com/redBorder/rbforwarder/senders/httpsender"
	"github.com/x-cray/logrus-prefixed-formatter"
)

const (
	defaultQueueSize = 10000
	defaultWorkers   = 1
	defaultRetries   = 0
	defaultBackoff   = 2
)

var (
	configFile *string
	debug      *bool
	logger     *logrus.Entry
)

func init() {
	configFile = flag.String("config", "", "Config file")
	debug = flag.Bool("debug", false, "Show debug info")

	flag.Parse()

	if len(*configFile) == 0 {
		fmt.Println("No config file provided")
		flag.Usage()
		os.Exit(1)
	}

	log := logrus.New()
	// Show debug info if required
	if *debug {
		log.Level = logrus.DebugLevel
	}
	log.Formatter = new(prefixed.TextFormatter)

	logger = log.WithFields(logrus.Fields{
		"prefix": "k2http",
	})
}

func main() {

	// Load the configuration from file
	configData, err := LoadConfigFile(*configFile)
	if err != nil {
		logger.Fatal(err)
	}

	logger.Debug("Showing debug info")

	// Capture ctrl-c
	ctrlc := make(chan os.Signal, 1)
	signal.Notify(ctrlc, os.Interrupt)

	// Parse the backend configuration
	rbForwarderConfig := rbforwarder.Config{}

	// Get number of workers
	if workers, ok := configData.Backend["workers"].(int); ok {
		rbForwarderConfig.Workers = workers
	} else {
		rbForwarderConfig.Workers = defaultWorkers
	}

	// Get number of retries per message
	if retries, ok := configData.Backend["retries"].(int); ok {
		rbForwarderConfig.Retries = retries
	} else {
		rbForwarderConfig.Retries = defaultRetries
	}

	// Time to wait between retries
	if backoff, ok := configData.Backend["backoff"].(int); ok {
		rbForwarderConfig.Backoff = backoff
	} else {
		rbForwarderConfig.Backoff = defaultBackoff
	}

	// Get queue size
	if queue, ok := configData.Backend["queue"].(int); ok {
		rbForwarderConfig.QueueSize = queue
	} else {
		rbForwarderConfig.QueueSize = defaultQueueSize
	}

	// Get max message rate
	if maxMessages, ok := configData.Backend["max_messages"].(int); ok {
		rbForwarderConfig.MaxMessages = maxMessages
	}

	// Get max bytes rate
	if maxBytes, ok := configData.Backend["max_bytes"].(int); ok {
		rbForwarderConfig.MaxBytes = maxBytes
	}

	// Show debug info
	if *debug {
		rbForwarderConfig.Debug = true
	}

	// Get the interval to show message rate
	if interval, ok := configData.Backend["showcounter"].(int); ok {
		rbForwarderConfig.ShowCounter = interval
	}

	// Create forwarder
	forwarder := rbforwarder.NewRBForwarder(rbForwarderConfig)

	// Initialize kafka
	kafka := new(KafkaConsumer)
	kafka.ParseKafkaConfig(configData.Kafka)
	kafka.backend = forwarder

	// Get the HTTP sender helper
	httpSenderHelper := httpsender.NewHelper(configData.HTTP)
	forwarder.SetSenderHelper(httpSenderHelper)

	// Start the backend
	forwarder.Start()

	// Wait for ctrl-c to close the consumer
	go func() {
		<-ctrlc
		kafka.Close()
		forwarder.Close()
	}()

	// Start getting messages
	kafka.Start()

	defer recoverPanic()
}
<|endoftext|>


<commit_before>package main

import (
	"log"
)

func main() {
	res, err := ping("google.com", 5)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("Min: %f ms\n", res.Min)
	log.Printf("Avg: %f ms\n", res.Avg)
	log.Printf("Max: %f ms\n", res.Max)
	log.Printf("Mdev: %f ms\n", res.Mdev)
}
<commit_msg>Use a ticker to schedule pings<commit_after>package main

import (
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"
)

func main() {
	ticker := time.NewTicker(10 * time.Second)
	go func() {
		for _ = range ticker.C {
			log.Println("ping google.com -c 5")
			res, err := ping("google.com", 5)
			if err != nil {
				log.Fatal(err)
			}
			log.Printf("Min: %f ms\n", res.Min)
			log.Printf("Avg: %f ms\n", res.Avg)
			log.Printf("Max: %f ms\n", res.Max)
			log.Printf("Mdev: %f ms\n", res.Mdev)
		}
	}()

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	log.Printf("Received signal: %v\n", <-ch)
	log.Println("Shutting down")
	ticker.Stop()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/RangelReale/osin"
	"github.com/Sirupsen/logrus"
	"github.com/buger/goterm"
	"github.com/docopt/docopt.go"
	"github.com/justinas/alice"
	"github.com/rcrowley/goagain"
	"html/template"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"
)

var log = logrus.New()
var authManager = AuthorisationManager{}
var config = Config{}
var templates = &template.Template{}
var analytics = RedisAnalyticsHandler{}
var profileFile = &os.File{}
var doMemoryProfile bool
var genericOsinStorage *RedisOsinStorageInterface

// Generic system error
const (
	E_SYSTEM_ERROR          string = "{\"status\": \"system error, please contact administrator\"}"
	OAUTH_AUTH_CODE_TIMEOUT int    = 60 * 60
	OAUTH_PREFIX            string = "oauth-data."
)

// Display introductory details
func intro() {
	fmt.Print("\n\n")
	fmt.Println(goterm.Bold(goterm.Color("Tyk.io Gateway API v1.0", goterm.GREEN)))
	fmt.Println(goterm.Bold(goterm.Color("=======================", goterm.GREEN)))
	fmt.Print("Copyright Jively Ltd. 2014")
	fmt.Print("\nhttp://www.tyk.io\n\n")
}

// Display configuration options
func displayConfig() {
	configTable := goterm.NewTable(0, 10, 5, ' ', 0)
	fmt.Fprintf(configTable, "Listening on port:\t%d\n", config.ListenPort)

	fmt.Println(configTable)
	fmt.Println("")
}

// Create all globals and init connection handlers
func setupGlobals() {
	if config.Storage.Type == "memory" {
		log.Warning("Using in-memory storage. Warning: this is not scalable.")
		authManager = AuthorisationManager{
			&InMemoryStorageManager{
				map[string]string{}}}
	} else if config.Storage.Type == "redis" {
		log.Info("Using Redis storage manager.")
		authManager = AuthorisationManager{
			&RedisStorageManager{KeyPrefix: "apikey-"}}

		authManager.Store.Connect()
	}

	if (config.EnableAnalytics == true) && (config.Storage.Type != "redis") {
		log.Panic("Analytics requires Redis Storage backend, please enable Redis in the tyk.conf file.")
	}

	if config.EnableAnalytics {
		AnalyticsStore := RedisStorageManager{KeyPrefix: "analytics-"}
		log.Info("Setting up analytics DB connection")

		if config.AnalyticsConfig.Type == "csv" {
			log.Info("Using CSV cache purge")
			analytics = RedisAnalyticsHandler{
				Store: &AnalyticsStore,
				Clean: &CSVPurger{&AnalyticsStore}}

		} else if config.AnalyticsConfig.Type == "mongo" {
			log.Info("Using MongoDB cache purge")
			analytics = RedisAnalyticsHandler{
				Store: &AnalyticsStore,
				Clean: &MongoPurger{&AnalyticsStore, nil}}
		}

		analytics.Store.Connect()
		go analytics.Clean.StartPurgeLoop(config.AnalyticsConfig.PurgeDelay)
	}

	genericOsinStorage = MakeNewOsinServer()

	templateFile := fmt.Sprintf("%s/error.json", config.TemplatePath)
	templates = template.Must(template.ParseFiles(templateFile))
}

// Pull API Specs from configuration
func getAPISpecs() []APISpec {
	var APISpecs []APISpec
	thisAPILoader := APIDefinitionLoader{}

	if config.UseDBAppConfigs {
		log.Info("Using App Configuration from Mongo DB")
		APISpecs = thisAPILoader.LoadDefinitionsFromMongo()
	} else {
		APISpecs = thisAPILoader.LoadDefinitions(config.AppPath)
	}

	return APISpecs
}

// Set up default Tyk control API endpoints - these are global, so need to be added first
func loadAPIEndpoints(Muxer *http.ServeMux) {
	// set up main API handlers
	Muxer.HandleFunc("/tyk/keys/create", CheckIsAPIOwner(createKeyHandler))
	Muxer.HandleFunc("/tyk/keys/", CheckIsAPIOwner(keyHandler))
	Muxer.HandleFunc("/tyk/reload/", CheckIsAPIOwner(resetHandler))
	Muxer.HandleFunc("/tyk/oauth/clients/create", CheckIsAPIOwner(createOauthClient))
	Muxer.HandleFunc("/tyk/oauth/clients/", CheckIsAPIOwner(oAuthClientHandler))
}

// Create API-specific OAuth handlers and respective auth servers
func addOAuthHandlers(spec APISpec, Muxer *http.ServeMux, test bool) {
	apiAuthorizePath := spec.Proxy.ListenPath + "tyk/oauth/authorize-client/"
	clientAuthPath := spec.Proxy.ListenPath + "oauth/authorize/"
	clientAccessPath := spec.Proxy.ListenPath + "oauth/token/"

	serverConfig := osin.NewServerConfig()
	serverConfig.ErrorStatusCode = 403
	serverConfig.AllowedAccessTypes = spec.Oauth2Meta.AllowedAccessTypes
	serverConfig.AllowedAuthorizeTypes = spec.Oauth2Meta.AllowedAuthorizeTypes

	OAuthPrefix := OAUTH_PREFIX + spec.APIID + "."
	storageManager := RedisStorageManager{KeyPrefix: OAuthPrefix}
	storageManager.Connect()
	osinStorage := RedisOsinStorageInterface{&storageManager}

	if test {
		log.Warning("Adding test client")
		testClient := osin.DefaultClient{
			Id:          "1234",
			Secret:      "aabbccdd",
			RedirectUri: "http://client.oauth.com",
		}
		osinStorage.SetClient(testClient.Id, &testClient, false)
		log.Warning("Test client added")
	}
	osinServer := osin.NewServer(serverConfig, osinStorage)
	osinServer.AccessTokenGen = &AccessTokenGenTyk{}

	oauthManager := OAuthManager{spec, osinServer}
	oauthHandlers := OAuthHandlers{oauthManager}

	Muxer.HandleFunc(apiAuthorizePath, CheckIsAPIOwner(oauthHandlers.HandleGenerateAuthCodeData))
	Muxer.HandleFunc(clientAuthPath, oauthHandlers.HandleAuthorizePassthrough)
	Muxer.HandleFunc(clientAccessPath, oauthHandlers.HandleAccessRequest)
}

// Create the individual API (app) specs based on live configurations and assign middleware
func loadApps(APISpecs []APISpec, Muxer *http.ServeMux) {
	// load the APi defs
	log.Info("Loading API configurations.")

	for _, spec := range APISpecs {
		// Create a new handler for each API spec
		remote, err := url.Parse(spec.APIDefinition.Proxy.TargetURL)
		if err != nil {
			log.Error("Culdn't parse target URL")
			log.Error(err)
		}

		if spec.UseOauth2 {
			addOAuthHandlers(spec, Muxer, false)
		}

		proxy := TykNewSingleHostReverseProxy(remote)
		spec.target = remote

		proxyHandler := http.HandlerFunc(ProxyHandler(proxy, spec))
		tykMiddleware := TykMiddleware{spec, proxy}

		if spec.APIDefinition.UseKeylessAccess {
			// for KeyLessAccess we can't support rate limiting, versioning or access rules
			chain := alice.New().Then(proxyHandler)
			Muxer.Handle(spec.Proxy.ListenPath, chain)

		} else {

			// Select the keying method to use for setting session states
			var keyCheck func(http.Handler) http.Handler

			if spec.APIDefinition.UseOauth2 {
				// Oauth2
				keyCheck = CreateMiddleware(&Oauth2KeyExists{tykMiddleware}, tykMiddleware)
			} else if spec.APIDefinition.UseBasicAuth {
				// Basic Auth
				keyCheck = CreateMiddleware(&BasicAuthKeyIsValid{tykMiddleware}, tykMiddleware)
			} else if spec.EnableSignatureChecking {
				// HMAC Auth
				keyCheck = CreateMiddleware(&HMACMiddleware{tykMiddleware}, tykMiddleware)
			} else {
				// Auth key
				keyCheck = CreateMiddleware(&AuthKey{tykMiddleware}, tykMiddleware)
			}

			// Use CreateMiddleware(&ModifiedMiddleware{tykMiddleware}, tykMiddleware)  to run custom middleware
			chain := alice.New(
				keyCheck,
				CreateMiddleware(&KeyExpired{tykMiddleware}, tykMiddleware),
				CreateMiddleware(&VersionCheck{tykMiddleware}, tykMiddleware),
				CreateMiddleware(&AccessRightsCheck{tykMiddleware}, tykMiddleware),
				CreateMiddleware(&RateLimitAndQuotaCheck{tykMiddleware}, tykMiddleware)).Then(proxyHandler)

			Muxer.Handle(spec.Proxy.ListenPath, chain)
		}

	}
}

// ReloadURLStructure will create a new muxer, reload all the app configs for an
// instance and then replace the DefaultServeMux with the new one, this enables a
// reconfiguration to take place without stopping any requests from being handled.
func ReloadURLStructure() {
	newMuxes := http.NewServeMux()
	loadAPIEndpoints(newMuxes)
	specs := getAPISpecs()
	loadApps(specs, newMuxes)

	http.DefaultServeMux = newMuxes
	log.Info("Reload complete")
}

func init() {
	intro()

	usage := `Tyk API Gateway.

	Usage:
		tyk [options]

	Options:
		-h --help      Show this screen
		--conf=FILE    Load a named configuration file
		--port=PORT    Listen on PORT (overrides confg file)
		--memprofile   Generate a memory profile
		--debug		   Enable Debug output

	`

	arguments, err := docopt.Parse(usage, nil, true, "v1.0", false)
	if err != nil {
		log.Println("Error while parsing arguments.")
		log.Fatal(err)
	}

	filename := "/etc/tyk/tyk.conf"
	value, _ := arguments["--conf"]
	if value != nil {
		log.Info(fmt.Sprintf("Using %s for configuration", value.(string)))
		filename = arguments["--conf"].(string)
	} else {
		log.Info("No configuration file defined, will try to use default (./tyk.conf)")
	}

	loadConfig(filename, &config)

	if config.Storage.Type != "redis" {
		log.Fatal("Redis connection details not set, please ensure that the storage type is set to Redis and that the connection parameters are correct.")
	}

	setupGlobals()

	port, _ := arguments["--port"]
	if port != nil {
		portNum, err := strconv.Atoi(port.(string))
		if err != nil {
			log.Error("Port specified in flags must be a number!")
			log.Error(err)
		} else {
			config.ListenPort = portNum
		}
	}

	doMemoryProfile, _ = arguments["--memprofile"].(bool)

	doDebug, _ := arguments["--debug"]
	log.Level = logrus.Info
	if doDebug == true {
		log.Level = logrus.Debug
		log.Debug("Enabling debug-level output")
	}

}

func main() {
	displayConfig()

	if doMemoryProfile {
		log.Info("Memory profiling active")
		profileFile, _ = os.Create("tyk.mprof")
		defer profileFile.Close()
	}

	targetPort := fmt.Sprintf(":%d", config.ListenPort)
	loadAPIEndpoints(http.DefaultServeMux)

	// Handle reload when SIGUSR2 is received
	l, err := goagain.Listener()
	if nil != err {

		// Listen on a TCP or a UNIX domain socket (TCP here).
		l, err = net.Listen("tcp", targetPort)
		if nil != err {
			log.Fatalln(err)
		}
		log.Println("Listening on", l.Addr())

		// Accept connections in a new goroutine.
		specs := getAPISpecs()
		loadApps(specs, http.DefaultServeMux)
		go http.Serve(l, nil)

	} else {

		// Resume accepting connections in a new goroutine.
		log.Println("Resuming listening on", l.Addr())
		specs := getAPISpecs()
		loadApps(specs, http.DefaultServeMux)
		go http.Serve(l, nil)

		// Kill the parent, now that the child has started successfully.
		if err := goagain.Kill(); nil != err {
			log.Fatalln(err)
		}

	}

	// Block the main goroutine awaiting signals.
	if _, err := goagain.Wait(l); nil != err {
		log.Fatalln(err)
	}

	// Do whatever's necessary to ensure a graceful exit like waiting for
	// goroutines to terminate or a channel to become closed.
	//
	// In this case, we'll simply stop listening and wait one second.
	if err := l.Close(); nil != err {
		log.Fatalln(err)
	}
	time.Sleep(1e9)
}
<commit_msg>Update logrus usage to latest API<commit_after>package main

import (
	"fmt"
	"github.com/RangelReale/osin"
	"github.com/Sirupsen/logrus"
	"github.com/buger/goterm"
	"github.com/docopt/docopt.go"
	"github.com/justinas/alice"
	"github.com/rcrowley/goagain"
	"html/template"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"
)

var log = logrus.New()
var authManager = AuthorisationManager{}
var config = Config{}
var templates = &template.Template{}
var analytics = RedisAnalyticsHandler{}
var profileFile = &os.File{}
var doMemoryProfile bool
var genericOsinStorage *RedisOsinStorageInterface

// Generic system error
const (
	E_SYSTEM_ERROR          string = "{\"status\": \"system error, please contact administrator\"}"
	OAUTH_AUTH_CODE_TIMEOUT int    = 60 * 60
	OAUTH_PREFIX            string = "oauth-data."
)

// Display introductory details
func intro() {
	fmt.Print("\n\n")
	fmt.Println(goterm.Bold(goterm.Color("Tyk.io Gateway API v1.0", goterm.GREEN)))
	fmt.Println(goterm.Bold(goterm.Color("=======================", goterm.GREEN)))
	fmt.Print("Copyright Jively Ltd. 2014")
	fmt.Print("\nhttp://www.tyk.io\n\n")
}

// Display configuration options
func displayConfig() {
	configTable := goterm.NewTable(0, 10, 5, ' ', 0)
	fmt.Fprintf(configTable, "Listening on port:\t%d\n", config.ListenPort)

	fmt.Println(configTable)
	fmt.Println("")
}

// Create all globals and init connection handlers
func setupGlobals() {
	if config.Storage.Type == "memory" {
		log.Warning("Using in-memory storage. Warning: this is not scalable.")
		authManager = AuthorisationManager{
			&InMemoryStorageManager{
				map[string]string{}}}
	} else if config.Storage.Type == "redis" {
		log.Info("Using Redis storage manager.")
		authManager = AuthorisationManager{
			&RedisStorageManager{KeyPrefix: "apikey-"}}

		authManager.Store.Connect()
	}

	if (config.EnableAnalytics == true) && (config.Storage.Type != "redis") {
		log.Panic("Analytics requires Redis Storage backend, please enable Redis in the tyk.conf file.")
	}

	if config.EnableAnalytics {
		AnalyticsStore := RedisStorageManager{KeyPrefix: "analytics-"}
		log.Info("Setting up analytics DB connection")

		if config.AnalyticsConfig.Type == "csv" {
			log.Info("Using CSV cache purge")
			analytics = RedisAnalyticsHandler{
				Store: &AnalyticsStore,
				Clean: &CSVPurger{&AnalyticsStore}}

		} else if config.AnalyticsConfig.Type == "mongo" {
			log.Info("Using MongoDB cache purge")
			analytics = RedisAnalyticsHandler{
				Store: &AnalyticsStore,
				Clean: &MongoPurger{&AnalyticsStore, nil}}
		}

		analytics.Store.Connect()
		go analytics.Clean.StartPurgeLoop(config.AnalyticsConfig.PurgeDelay)
	}

	genericOsinStorage = MakeNewOsinServer()

	templateFile := fmt.Sprintf("%s/error.json", config.TemplatePath)
	templates = template.Must(template.ParseFiles(templateFile))
}

// Pull API Specs from configuration
func getAPISpecs() []APISpec {
	var APISpecs []APISpec
	thisAPILoader := APIDefinitionLoader{}

	if config.UseDBAppConfigs {
		log.Info("Using App Configuration from Mongo DB")
		APISpecs = thisAPILoader.LoadDefinitionsFromMongo()
	} else {
		APISpecs = thisAPILoader.LoadDefinitions(config.AppPath)
	}

	return APISpecs
}

// Set up default Tyk control API endpoints - these are global, so need to be added first
func loadAPIEndpoints(Muxer *http.ServeMux) {
	// set up main API handlers
	Muxer.HandleFunc("/tyk/keys/create", CheckIsAPIOwner(createKeyHandler))
	Muxer.HandleFunc("/tyk/keys/", CheckIsAPIOwner(keyHandler))
	Muxer.HandleFunc("/tyk/reload/", CheckIsAPIOwner(resetHandler))
	Muxer.HandleFunc("/tyk/oauth/clients/create", CheckIsAPIOwner(createOauthClient))
	Muxer.HandleFunc("/tyk/oauth/clients/", CheckIsAPIOwner(oAuthClientHandler))
}

// Create API-specific OAuth handlers and respective auth servers
func addOAuthHandlers(spec APISpec, Muxer *http.ServeMux, test bool) {
	apiAuthorizePath := spec.Proxy.ListenPath + "tyk/oauth/authorize-client/"
	clientAuthPath := spec.Proxy.ListenPath + "oauth/authorize/"
	clientAccessPath := spec.Proxy.ListenPath + "oauth/token/"

	serverConfig := osin.NewServerConfig()
	serverConfig.ErrorStatusCode = 403
	serverConfig.AllowedAccessTypes = spec.Oauth2Meta.AllowedAccessTypes
	serverConfig.AllowedAuthorizeTypes = spec.Oauth2Meta.AllowedAuthorizeTypes

	OAuthPrefix := OAUTH_PREFIX + spec.APIID + "."
	storageManager := RedisStorageManager{KeyPrefix: OAuthPrefix}
	storageManager.Connect()
	osinStorage := RedisOsinStorageInterface{&storageManager}

	if test {
		log.Warning("Adding test client")
		testClient := osin.DefaultClient{
			Id:          "1234",
			Secret:      "aabbccdd",
			RedirectUri: "http://client.oauth.com",
		}
		osinStorage.SetClient(testClient.Id, &testClient, false)
		log.Warning("Test client added")
	}
	osinServer := osin.NewServer(serverConfig, osinStorage)
	osinServer.AccessTokenGen = &AccessTokenGenTyk{}

	oauthManager := OAuthManager{spec, osinServer}
	oauthHandlers := OAuthHandlers{oauthManager}

	Muxer.HandleFunc(apiAuthorizePath, CheckIsAPIOwner(oauthHandlers.HandleGenerateAuthCodeData))
	Muxer.HandleFunc(clientAuthPath, oauthHandlers.HandleAuthorizePassthrough)
	Muxer.HandleFunc(clientAccessPath, oauthHandlers.HandleAccessRequest)
}

// Create the individual API (app) specs based on live configurations and assign middleware
func loadApps(APISpecs []APISpec, Muxer *http.ServeMux) {
	// load the APi defs
	log.Info("Loading API configurations.")

	for _, spec := range APISpecs {
		// Create a new handler for each API spec
		remote, err := url.Parse(spec.APIDefinition.Proxy.TargetURL)
		if err != nil {
			log.Error("Culdn't parse target URL")
			log.Error(err)
		}

		if spec.UseOauth2 {
			addOAuthHandlers(spec, Muxer, false)
		}

		proxy := TykNewSingleHostReverseProxy(remote)
		spec.target = remote

		proxyHandler := http.HandlerFunc(ProxyHandler(proxy, spec))
		tykMiddleware := TykMiddleware{spec, proxy}

		if spec.APIDefinition.UseKeylessAccess {
			// for KeyLessAccess we can't support rate limiting, versioning or access rules
			chain := alice.New().Then(proxyHandler)
			Muxer.Handle(spec.Proxy.ListenPath, chain)

		} else {

			// Select the keying method to use for setting session states
			var keyCheck func(http.Handler) http.Handler

			if spec.APIDefinition.UseOauth2 {
				// Oauth2
				keyCheck = CreateMiddleware(&Oauth2KeyExists{tykMiddleware}, tykMiddleware)
			} else if spec.APIDefinition.UseBasicAuth {
				// Basic Auth
				keyCheck = CreateMiddleware(&BasicAuthKeyIsValid{tykMiddleware}, tykMiddleware)
			} else if spec.EnableSignatureChecking {
				// HMAC Auth
				keyCheck = CreateMiddleware(&HMACMiddleware{tykMiddleware}, tykMiddleware)
			} else {
				// Auth key
				keyCheck = CreateMiddleware(&AuthKey{tykMiddleware}, tykMiddleware)
			}

			// Use CreateMiddleware(&ModifiedMiddleware{tykMiddleware}, tykMiddleware)  to run custom middleware
			chain := alice.New(
				keyCheck,
				CreateMiddleware(&KeyExpired{tykMiddleware}, tykMiddleware),
				CreateMiddleware(&VersionCheck{tykMiddleware}, tykMiddleware),
				CreateMiddleware(&AccessRightsCheck{tykMiddleware}, tykMiddleware),
				CreateMiddleware(&RateLimitAndQuotaCheck{tykMiddleware}, tykMiddleware)).Then(proxyHandler)

			Muxer.Handle(spec.Proxy.ListenPath, chain)
		}

	}
}

// ReloadURLStructure will create a new muxer, reload all the app configs for an
// instance and then replace the DefaultServeMux with the new one, this enables a
// reconfiguration to take place without stopping any requests from being handled.
func ReloadURLStructure() {
	newMuxes := http.NewServeMux()
	loadAPIEndpoints(newMuxes)
	specs := getAPISpecs()
	loadApps(specs, newMuxes)

	http.DefaultServeMux = newMuxes
	log.Info("Reload complete")
}

func init() {
	intro()

	usage := `Tyk API Gateway.

	Usage:
		tyk [options]

	Options:
		-h --help      Show this screen
		--conf=FILE    Load a named configuration file
		--port=PORT    Listen on PORT (overrides confg file)
		--memprofile   Generate a memory profile
		--debug		   Enable Debug output

	`

	arguments, err := docopt.Parse(usage, nil, true, "v1.0", false)
	if err != nil {
		log.Println("Error while parsing arguments.")
		log.Fatal(err)
	}

	filename := "/etc/tyk/tyk.conf"
	value, _ := arguments["--conf"]
	if value != nil {
		log.Info(fmt.Sprintf("Using %s for configuration", value.(string)))
		filename = arguments["--conf"].(string)
	} else {
		log.Info("No configuration file defined, will try to use default (./tyk.conf)")
	}

	loadConfig(filename, &config)

	if config.Storage.Type != "redis" {
		log.Fatal("Redis connection details not set, please ensure that the storage type is set to Redis and that the connection parameters are correct.")
	}

	setupGlobals()

	port, _ := arguments["--port"]
	if port != nil {
		portNum, err := strconv.Atoi(port.(string))
		if err != nil {
			log.Error("Port specified in flags must be a number!")
			log.Error(err)
		} else {
			config.ListenPort = portNum
		}
	}

	doMemoryProfile, _ = arguments["--memprofile"].(bool)

	doDebug, _ := arguments["--debug"]
	log.Level = logrus.InfoLevel
	if doDebug == true {
		log.Level = logrus.DebugLevel
		log.Debug("Enabling debug-level output")
	}

}

func main() {
	displayConfig()

	if doMemoryProfile {
		log.Info("Memory profiling active")
		profileFile, _ = os.Create("tyk.mprof")
		defer profileFile.Close()
	}

	targetPort := fmt.Sprintf(":%d", config.ListenPort)
	loadAPIEndpoints(http.DefaultServeMux)

	// Handle reload when SIGUSR2 is received
	l, err := goagain.Listener()
	if nil != err {

		// Listen on a TCP or a UNIX domain socket (TCP here).
		l, err = net.Listen("tcp", targetPort)
		if nil != err {
			log.Fatalln(err)
		}
		log.Println("Listening on", l.Addr())

		// Accept connections in a new goroutine.
		specs := getAPISpecs()
		loadApps(specs, http.DefaultServeMux)
		go http.Serve(l, nil)

	} else {

		// Resume accepting connections in a new goroutine.
		log.Println("Resuming listening on", l.Addr())
		specs := getAPISpecs()
		loadApps(specs, http.DefaultServeMux)
		go http.Serve(l, nil)

		// Kill the parent, now that the child has started successfully.
		if err := goagain.Kill(); nil != err {
			log.Fatalln(err)
		}

	}

	// Block the main goroutine awaiting signals.
	if _, err := goagain.Wait(l); nil != err {
		log.Fatalln(err)
	}

	// Do whatever's necessary to ensure a graceful exit like waiting for
	// goroutines to terminate or a channel to become closed.
	//
	// In this case, we'll simply stop listening and wait one second.
	if err := l.Close(); nil != err {
		log.Fatalln(err)
	}
	time.Sleep(1e9)
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/yapdns/yapdns-client/beater"
	"github.com/yapdns/yapdns-client/outputs/http"
	"github.com/elastic/beats/libbeat/beat"
	"github.com/elastic/beats/libbeat/outputs"
)

var Name = "filebeat"

// The basic model of execution:
// - prospector: finds files in paths/globs to harvest, starts harvesters
// - harvester: reads a file, sends events to the spooler
// - spooler: buffers events until ready to flush to the publisher
// - publisher: writes to the network, notifies registrar
// - registrar: records positions of files read
// Finally, prospector uses the registrar information, on restart, to
// determine where in each file to restart a harvester.

func init() {
	outputs.RegisterOutputPlugin("http", http.New)
}

func main() {
	if err := beat.Run(Name, "", beater.New()); err != nil {
		os.Exit(1)
	}
}
<commit_msg>create libbeat output plugin the nice way<commit_after>package main

import (
	"os"

	"github.com/yapdns/yapdns-client/beater"
	"github.com/elastic/beats/libbeat/beat"
	_ "github.com/yapdns/yapdns-client/outputs/http"
)

var Name = "filebeat"

// The basic model of execution:
// - prospector: finds files in paths/globs to harvest, starts harvesters
// - harvester: reads a file, sends events to the spooler
// - spooler: buffers events until ready to flush to the publisher
// - publisher: writes to the network, notifies registrar
// - registrar: records positions of files read
// Finally, prospector uses the registrar information, on restart, to
// determine where in each file to restart a harvester.

func main() {
	if err := beat.Run(Name, "", beater.New()); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main
import (
    "bytes"
    "crypto/sha1"
    "encoding/hex"
    "flag"
    "fmt"
    "image"
    "image/gif"
    "image/png"
    "image/jpeg"
    "io"
    "io/ioutil"
    "net/http"
    "os"
    "strconv"
    "strings"
    "time"
    "github.com/nfnt/resize"
    "github.com/bradfitz/gomemcache/memcache"
)

type JobDescription struct {
    Url string
    Key string
    Format string
    MaxHeight uint64
    MaxWidth uint64
}

var (
    globalMaxWidth uint64
    globalMaxHeight uint64
    maxInDim uint64
    maxInSize uint64
    errorPicName string
    standByPicName string
    listenAddr string
    memcacheAddr string
    jpegQuality int

    marshaledTimeLength int
    errorPic image.Image
    standByPic image.Image
    mc *memcache.Client
    hc http.Client
    workChan chan *JobDescription = make(chan *JobDescription, 100)
)
var formatsMime = map[string]string{
        "gif": "image/gif",
        "png": "image/png",
        "jpg": "image/jpeg",
}

func init() {
    flag.Uint64Var(&globalMaxWidth, "maxwidth", 512, "max thumbnail width in pixels")
    flag.Uint64Var(&globalMaxHeight, "maxheight", 512, "max thumbnail height in pixels")
    flag.Uint64Var(&maxInDim, "maxindim", 1024, "max input image height or width in pixels")
    flag.Uint64Var(&maxInSize, "maxinsize", 20*1024*1024, "max input image size in bytes")
    flag.IntVar(&jpegQuality, "quality", 80, "jpeg quality, 0 to 100")
    flag.StringVar(&errorPicName, "errorpic", "", "error picture")
    flag.StringVar(&memcacheAddr, "memcache", "127.0.0.1:11211", "comma-separated list of memcache servers")
    flag.StringVar(&listenAddr, "listen", "127.0.0.1:8080", "address and port to listen on")
}

func parseOrDefault(s string, min uint64, max uint64, def uint64) uint64 {
    if s == "" {
        return def
    }
    n, err := strconv.ParseUint(s, 10, 64)
    switch {
        case err != nil:
            return def
        case n < min:
            return min
        case n > max:
            return max
    }
    return n
}

func encodeImage(w io.Writer, img image.Image, fmt string) error {
    switch fmt {
        case "png": return png.Encode(w, img)
        case "gif": return gif.Encode(w, img, &gif.Options{NumColors: 256})
    }
    return jpeg.Encode(w, img, &jpeg.Options{Quality: jpegQuality} )
}

func renderThumbnail(img image.Image, key string, lastMod time.Time,
                        maxWidth uint64, maxHeight uint64, fmt string) {
    thumb := resize.Thumbnail(uint(maxWidth), uint(maxHeight), img, resize.NearestNeighbor)
    var outbuf bytes.Buffer
    encodeImage(&outbuf, thumb, fmt)
    thumbHash := sha1.New()
    bytes.NewReader(outbuf.Bytes()).WriteTo(thumbHash)
    etag := hex.EncodeToString(thumbHash.Sum(nil))
    if len(etag) != 40 { return }
    saveThumb(key, []byte(etag), []byte{}, lastMod, outbuf.Bytes())
    return
}

func thumbHandler(w http.ResponseWriter, r *http.Request) {
    if r.Method != "GET" {
        http.Error(w,"Method not allowed", http.StatusMethodNotAllowed)
        return
    }
    r.ParseForm()
    var maxWidth uint64
    var maxHeight uint64
    argUrl := r.Form.Get("img")
    argFmt := r.Form.Get("fmt")
    mimeType,ok := formatsMime[argFmt]
    if !ok {
        argFmt = "png"
        mimeType = formatsMime["png"]
    }
    hdr := w.Header()
    hdr["Content-Type"] = []string{mimeType}
    argMax := r.Form.Get("max")
    if argMax != "" {
        maxWidth = parseOrDefault(argMax, 1, globalMaxWidth, globalMaxHeight)
        maxHeight = parseOrDefault(argMax, 1, globalMaxHeight, globalMaxHeight)
    } else {
        maxWidth = parseOrDefault(r.Form.Get("mx"), 1, globalMaxWidth, globalMaxWidth)
        maxHeight = parseOrDefault(r.Form.Get("my"), 1, globalMaxHeight, globalMaxHeight)
    }
    if mc != nil {
        h := sha1.New()
        io.WriteString(h, argUrl)
        key := fmt.Sprintf("%s-%dx%d-%s", argFmt, maxWidth, maxHeight, hex.EncodeToString(h.Sum(nil)))[:250]
        item, err := mc.Get(key)
        if err == nil {
            if len(item.Value) >= 86 {
                thumbEtag := item.Value[0:40]
                //originalEtag := item.Value[40:80]
                lastModBytes := item.Value[80:80+marshaledTimeLength]
                lastMod := time.Time{}
                lastMod.UnmarshalBinary(lastModBytes)
                dataBytes := item.Value[80+marshaledTimeLength:]
                hdr.Set("Last-Modified", lastMod.Format(http.TimeFormat))
                hdr.Set("Etag", string(thumbEtag))
                hdr.Set("Content-Length", strconv.Itoa(len(dataBytes)))
                w.Write(dataBytes)
                return
            }
        } else if err == memcache.ErrCacheMiss {
            err = mc.Add(&memcache.Item{Key: key, Value: []byte("X")})
            if err != memcache.ErrNotStored {
                workChan <- &JobDescription{
                    Url: argUrl, Key: key, Format: argFmt,
                    MaxHeight: maxHeight, MaxWidth: maxWidth,
                }
            }
        }
    }

    thumb := resize.Thumbnail(uint(maxWidth), uint(maxHeight), standByPic, resize.NearestNeighbor)
    hdr.Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
    hdr.Set("Cache-control", "max-age=5, public")
    hdr.Set("Etag", fmt.Sprintf("standby-%d-%d-%s", maxWidth, maxHeight, argFmt))
    encodeImage(w, thumb, argFmt)
    return
}

func saveThumb(Key string, Etag []byte, OriginEtag []byte, LastMod time.Time, data []byte) error {
    if mc != nil {
        value := make([]byte, 0, 128)
        lastModBytes, _ := LastMod.MarshalBinary()
        OriginEtag := []byte("0000000000000000000000000000000000000000") // not used yet
        value = append(value, Etag...)
        value = append(value, OriginEtag...)
        value = append(value, lastModBytes...)
        value = append(value, data...)
        return mc.Set(&memcache.Item{Key: Key, Value: value })
    }
    return nil
}

func renderWorker() {
    for job := range workChan {
        closeBody := false
        req, err := http.NewRequest("GET", job.Url, nil)
        if err != nil { goto Error }
        req.Header.Set("User-Agent", "bnw-thumb/1.0 (http://github.com/stiletto/bnw-thumb)")
        resp, err := hc.Do(req)
        if err != nil { goto Error }
        closeBody = true
        if resp.StatusCode != 200 || uint64(resp.ContentLength) > maxInSize { goto Error }
        bytebuf, err := ioutil.ReadAll(resp.Body)
        if err != nil { goto Error }
        bufreader := bytes.NewReader(bytebuf)
        config, _, err := image.DecodeConfig(bufreader)
        if err != nil || uint64(config.Width) > maxInDim || uint64(config.Height) > maxInDim { goto Error }
        bufreader.Seek(0,0)
        img, _, err := image.Decode(bufreader)
        if err != nil { goto Error }
        if closeBody { resp.Body.Close() }
        renderThumbnail(img, job.Key, time.Now(), job.MaxWidth, job.MaxHeight, job.Format)
        return
    Error:
        if closeBody { resp.Body.Close() }
        renderThumbnail(errorPic, job.Key, time.Now(), job.MaxWidth, job.MaxHeight, job.Format)
        return
    }
}

func loadPicOrEmpty(name string) image.Image {
    var pic image.Image
    if name != "" {
        var fmtinfo string
        img, err := os.Open(name)
        defer img.Close()
        if err == nil {
            pic, fmtinfo, err = image.Decode(img)
        }
        if err != nil {
            pic = nil
            fmt.Fprintf(os.Stderr, "Unable to load picture %s: %s\n", name, err)
        } else {
            ebounds := pic.Bounds()
            fmt.Fprintf(os.Stderr, "Loaded picture %s (%s, %dx%d)\n",
                name, fmtinfo, ebounds.Max.X-ebounds.Min.X,
                ebounds.Max.Y-ebounds.Min.Y)
        }
    }
    if pic == nil {
        pic = image.NewRGBA(image.Rect(0, 0, 1, 1))
    }
    return pic
}

func main() {
    flag.Parse()
    mc = nil
    if memcacheAddr != "" {
        mc = memcache.New(strings.Split(memcacheAddr, ",")...)
    } else {
        fmt.Fprintf(os.Stderr, "Warning: Memcache is required for bnw-thumb to work.\n")
        fmt.Fprintf(os.Stderr, "Warning: Without memcache no thumbnails will be actually generated. Use only for testing.\n")
    }
    marshaledNow, _ := time.Now().MarshalBinary()
    // we know that time.Time always has the same length when marshaled
    // but this is kinda implementation dependant hack
    marshaledTimeLength = len(marshaledNow)
    errorPic = loadPicOrEmpty(errorPicName)
    standByPic = loadPicOrEmpty(standByPicName)
    http.HandleFunc("/", thumbHandler)
    http.ListenAndServe(listenAddr, nil)
}
<commit_msg>Somewhat working<commit_after>package main
import (
    "bytes"
    "crypto/sha1"
    "encoding/hex"
    "flag"
    "fmt"
    "image"
    "image/gif"
    "image/png"
    "image/jpeg"
    "io"
    "io/ioutil"
    "net/http"
    "os"
    "strconv"
    "strings"
    "time"
    "github.com/nfnt/resize"
    "github.com/bradfitz/gomemcache/memcache"
)

type JobDescription struct {
    Url string
    Key string
    Format string
    MaxHeight uint64
    MaxWidth uint64
}

var (
    globalMaxWidth uint64
    globalMaxHeight uint64
    maxInDim uint64
    maxInSize uint64
    errorPicName string
    standByPicName string
    listenAddr string
    memcacheAddr string
    jpegQuality int

    marshaledTimeLength int
    errorPic image.Image
    standByPic image.Image
    mc *memcache.Client
    hc http.Client
    workChan chan *JobDescription = make(chan *JobDescription, 100)
)
var formatsMime = map[string]string{
        "gif": "image/gif",
        "png": "image/png",
        "jpg": "image/jpeg",
}

func init() {
    flag.Uint64Var(&globalMaxWidth, "maxwidth", 512, "max thumbnail width in pixels")
    flag.Uint64Var(&globalMaxHeight, "maxheight", 512, "max thumbnail height in pixels")
    flag.Uint64Var(&maxInDim, "maxindim", 1024, "max input image height or width in pixels")
    flag.Uint64Var(&maxInSize, "maxinsize", 20*1024*1024, "max input image size in bytes")
    flag.IntVar(&jpegQuality, "quality", 80, "jpeg quality, 0 to 100")
    flag.StringVar(&errorPicName, "errorpic", "", "error picture")
    flag.StringVar(&standByPicName, "standbypic", "", "standby picture")
    flag.StringVar(&memcacheAddr, "memcache", "127.0.0.1:11211", "comma-separated list of memcache servers")
    flag.StringVar(&listenAddr, "listen", "127.0.0.1:8080", "address and port to listen on")
}

func parseOrDefault(s string, min uint64, max uint64, def uint64) uint64 {
    if s == "" {
        return def
    }
    n, err := strconv.ParseUint(s, 10, 64)
    switch {
        case err != nil:
            return def
        case n < min:
            return min
        case n > max:
            return max
    }
    return n
}

func encodeImage(w io.Writer, img image.Image, fmt string) error {
    switch fmt {
        case "png": return png.Encode(w, img)
        case "gif": return gif.Encode(w, img, &gif.Options{NumColors: 256})
    }
    return jpeg.Encode(w, img, &jpeg.Options{Quality: jpegQuality} )
}

func renderThumbnail(img image.Image, key string, lastMod time.Time,
                        maxWidth uint64, maxHeight uint64, fmt string) {
    thumb := resize.Thumbnail(uint(maxWidth), uint(maxHeight), img, resize.NearestNeighbor)
    var outbuf bytes.Buffer
    encodeImage(&outbuf, thumb, fmt)
    thumbHash := sha1.New()
    bytes.NewReader(outbuf.Bytes()).WriteTo(thumbHash)
    etag := hex.EncodeToString(thumbHash.Sum(nil))
    if len(etag) != 40 { return }
    saveThumb(key, []byte(etag), []byte{}, lastMod, outbuf.Bytes())
    return
}

func thumbHandler(w http.ResponseWriter, r *http.Request) {
    if r.Method != "GET" {
        http.Error(w,"Method not allowed", http.StatusMethodNotAllowed)
        return
    }
    r.ParseForm()
    var maxWidth uint64
    var maxHeight uint64
    argUrl := r.Form.Get("img")
    argFmt := r.Form.Get("fmt")
    mimeType,ok := formatsMime[argFmt]
    if !ok {
        argFmt = "png"
        mimeType = formatsMime["png"]
    }
    hdr := w.Header()
    hdr["Content-Type"] = []string{mimeType}
    argMax := r.Form.Get("max")
    if argMax != "" {
        maxWidth = parseOrDefault(argMax, 1, globalMaxWidth, globalMaxHeight)
        maxHeight = parseOrDefault(argMax, 1, globalMaxHeight, globalMaxHeight)
    } else {
        maxWidth = parseOrDefault(r.Form.Get("mx"), 1, globalMaxWidth, globalMaxWidth)
        maxHeight = parseOrDefault(r.Form.Get("my"), 1, globalMaxHeight, globalMaxHeight)
    }
    if mc != nil {
        h := sha1.New()
        io.WriteString(h, argUrl)
        key := fmt.Sprintf("%s-%dx%d-%s", argFmt, maxWidth, maxHeight, hex.EncodeToString(h.Sum(nil)))
        if len(key)> 250 { key=key[:250] }
        item, err := mc.Get(key)
        if err == nil {
            if len(item.Value) >= 86 {
                thumbEtag := item.Value[0:40]
                //originalEtag := item.Value[40:80]
                lastModBytes := item.Value[80:80+marshaledTimeLength]
                lastMod := time.Time{}
                lastMod.UnmarshalBinary(lastModBytes)
                dataBytes := item.Value[80+marshaledTimeLength:]
                hdr.Set("Last-Modified", lastMod.Format(http.TimeFormat))
                hdr.Set("Etag", string(thumbEtag))
                hdr.Set("Content-Length", strconv.Itoa(len(dataBytes)))
                w.Write(dataBytes)
                return
            }
        } else if err == memcache.ErrCacheMiss {
            err = mc.Add(&memcache.Item{Key: key, Value: []byte("X")})
            if err != memcache.ErrNotStored {
                workChan <- &JobDescription{
                    Url: argUrl, Key: key, Format: argFmt,
                    MaxHeight: maxHeight, MaxWidth: maxWidth,
                }
            }
        }
    }

    thumb := resize.Thumbnail(uint(maxWidth), uint(maxHeight), standByPic, resize.NearestNeighbor)
    hdr.Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
    hdr.Set("Cache-control", "max-age=5, public")
    hdr.Set("Etag", fmt.Sprintf("standby-%d-%d-%s", maxWidth, maxHeight, argFmt))
    encodeImage(w, thumb, argFmt)
    return
}

func saveThumb(Key string, Etag []byte, OriginEtag []byte, LastMod time.Time, data []byte) error {
    if mc != nil {
        value := make([]byte, 0, 128)
        lastModBytes, _ := LastMod.MarshalBinary()
        OriginEtag := []byte("0000000000000000000000000000000000000000") // not used yet
        value = append(value, Etag...)
        value = append(value, OriginEtag...)
        value = append(value, lastModBytes...)
        value = append(value, data...)
        return mc.Set(&memcache.Item{Key: Key, Value: value })
    }
    return nil
}

func renderWorker() {
    for job := range workChan {
        fmt.Fprintf(os.Stderr, "Generating \"%s\" %s %dx%d\n", job.Url, job.Format, job.MaxWidth, job.MaxHeight)
        req, err := http.NewRequest("GET", job.Url, nil)
        if err == nil {
            req.Header.Set("User-Agent", "bnw-thumb/1.0 (http://github.com/stiletto/bnw-thumb)")
            closeBody := false
            resp, err := hc.Do(req)
            if err == nil {
                closeBody = true
                if resp.StatusCode == 200 || uint64(resp.ContentLength) <= maxInSize {
                    bytebuf, err := ioutil.ReadAll(resp.Body)
                    if err == nil {
                        bufreader := bytes.NewReader(bytebuf)
                        config, _, err := image.DecodeConfig(bufreader)
                        if err == nil && uint64(config.Width) <= maxInDim && uint64(config.Height) <= maxInDim {
                            bufreader.Seek(0,0)
                            img, _, err := image.Decode(bufreader)
                            if err == nil {
                                if closeBody { resp.Body.Close() }
                                fmt.Fprintf(os.Stderr, "Generated \"%s\" %s %dx%d\n", job.Url, job.Format, job.MaxWidth, job.MaxHeight)
                                renderThumbnail(img, job.Key, time.Now(), job.MaxWidth, job.MaxHeight, job.Format)
                                continue
                            }
                        }
                    }
                }
            }
            fmt.Fprintf(os.Stderr, "Failed to generate \"%s\" %s %dx%d (%d, %d)\n", job.Url, job.Format,
                job.MaxWidth, job.MaxHeight, resp.StatusCode, resp.ContentLength )
            if closeBody { resp.Body.Close() }
        }
        renderThumbnail(errorPic, job.Key, time.Now(), job.MaxWidth, job.MaxHeight, job.Format)
    }
}

func loadPicOrEmpty(name string) image.Image {
    var pic image.Image
    if name != "" {
        var fmtinfo string
        img, err := os.Open(name)
        defer img.Close()
        if err == nil {
            pic, fmtinfo, err = image.Decode(img)
        }
        if err != nil {
            pic = nil
            fmt.Fprintf(os.Stderr, "Unable to load picture %s: %s\n", name, err)
        } else {
            ebounds := pic.Bounds()
            fmt.Fprintf(os.Stderr, "Loaded picture %s (%s, %dx%d)\n",
                name, fmtinfo, ebounds.Max.X-ebounds.Min.X,
                ebounds.Max.Y-ebounds.Min.Y)
        }
    }
    if pic == nil {
        pic = image.NewRGBA(image.Rect(0, 0, 1, 1))
    }
    return pic
}

func main() {
    flag.Parse()
    mc = nil
    if memcacheAddr != "" {
        mc = memcache.New(strings.Split(memcacheAddr, ",")...)
    } else {
        fmt.Fprintf(os.Stderr, "Warning: Memcache is required for bnw-thumb to work.\n")
        fmt.Fprintf(os.Stderr, "Warning: Without memcache no thumbnails will be actually generated. Use only for testing.\n")
    }
    marshaledNow, _ := time.Now().MarshalBinary()
    // we know that time.Time always has the same length when marshaled
    // but this is kinda implementation dependant hack
    marshaledTimeLength = len(marshaledNow)
    errorPic = loadPicOrEmpty(errorPicName)
    standByPic = loadPicOrEmpty(standByPicName)
    http.HandleFunc("/", thumbHandler)
    go renderWorker()
    fmt.Fprintf(os.Stderr, "Going to listen on %s\n", listenAddr)
    http.ListenAndServe(listenAddr, nil)
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	//"net/url"
	//"io/ioutil"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

    /*resp, err := http.Get("https://hr.hiyes.tw:443/getMessage.php?mid=Kordan&message=Ou")
    if err != nil {
        fmt.Println(err)
    } else {
        body, _ := ioutil.ReadAll(resp.Body)
        fmt.Println("GET OK: ", string(body), resp)
    }*/

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}
    var msg string
	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				//if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(event.ReplyToken+":"+message.ID+"-"+message.Text+" OK!")).Do(); err != nil {
				if message.Text == "勤耕延吉"{
					msg = "勤耕延吉"
				}
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(msg)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<commit_msg>add link<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	//"net/url"
	//"io/ioutil"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

    /*resp, err := http.Get("https://hr.hiyes.tw:443/getMessage.php?mid=Kordan&message=Ou")
    if err != nil {
        fmt.Println(err)
    } else {
        body, _ := ioutil.ReadAll(resp.Body)
        fmt.Println("GET OK: ", string(body), resp)
    }*/

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}
    var msg string
	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				//if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(event.ReplyToken+":"+message.ID+"-"+message.Text+" OK!")).Do(); err != nil {
				if message.Text == "勤耕延吉"{
					msg = "勤耕延吉 http://www.hiyes.tw/allcase/yanji/index.html"
				}
				if message.Text == "幸福莊園"{
					msg = "幸福莊園 http://www.hiyes.tw/allcase/happymanor/index.html"
				}
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(msg)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<|endoftext|>