<commit_before>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate protoc -I ./cni --gogo_out=plugins=grpc:./cni ./cni/cni.proto

package podmanager

import (
	"context"
	"fmt"
	"strings"

	"github.com/fsouza/go-dockerclient"

	"github.com/ligato/cn-infra/infra"
	grpcplugin "github.com/ligato/cn-infra/rpc/grpc"

	controller "github.com/contiv/vpp/plugins/controller/api"
	podmodel "github.com/contiv/vpp/plugins/ksr/model/pod"
	"github.com/contiv/vpp/plugins/podmanager/cni"
)

const (
	// name of the CNI request argument that stores pod name
	podNameExtraArg = "K8S_POD_NAME"

	// name of the CNI request argument that stores pod namespace
	podNamespaceExtraArg = "K8S_POD_NAMESPACE"

	// label attached to the sandbox container of every pod
	k8sLabelForSandboxContainer = "io.kubernetes.docker.type=podsandbox"

	// labels attached to (not only sandbox) container to identify the pod it belongs to
	k8sLabelForPodName      = "io.kubernetes.pod.name"
	k8sLabelForPodNamespace = "io.kubernetes.pod.namespace"

	// state value of running pods
	runningPodState = "running"

	// possible return value for CNI requests
	cniResultOk  uint32 = 0
	cniResultErr uint32 = 1
)

// PodManager plugin manages pods deployed on this node. It serves Add/Delete CNI
// requests, converts them to AddPod and DeletePod events, and maintains a map
// of metadata for all locally deployed pods, with enough information for other
// plugins to be able to (re)construct connectivity between pods and the vswitch.
type PodManager struct {
	Deps
	dockerClient DockerClient

	// map of locally deployed pods
	pods LocalPods
}

// Deps lists dependencies of PodManager.
type Deps struct {
	infra.PluginDeps
	EventLoop controller.EventLoop
	GRPC      grpcplugin.Server
}

// DockerClient defines API of a Docker client needed by PodManager.
// The interface allows to inject mock Docker client in the unit tests.
type DockerClient interface {
	// Ping pings the docker server.
	Ping() error
	// ListContainers returns a slice of containers matching the given criteria.
	ListContainers(opts docker.ListContainersOptions) ([]docker.APIContainers, error)
	// InspectContainer returns information about a container by its ID.
	InspectContainer(id string) (*docker.Container, error)
}

var (
	// Error thrown by PodManager.Update (main event loop) when Kubernetes asks
	// to configure pod which is already configured but with different parameters.
	// PodManager.Add() will act by sending an event to remove the obsolete pod
	// connectivity.
	errObsoletePod = fmt.Errorf("obsolete pod configuration detected")
)

// Init connects to Docker server and also registers the plugin to serve
// Add/Delete CNI requests.
func (pm *PodManager) Init() (err error) {
	// init attributes
	pm.pods = make(LocalPods)

	// connect to Docker server
	pm.dockerClient, err = docker.NewClientFromEnv()
	if err != nil {
		return err
	}

	cni.RegisterRemoteCNIServer(pm.GRPC.GetServer(), pm)
	return nil
}

// GetLocalPods returns all currently locally deployed pods.
// The method should be called only from within the main event loop
// (not thread safe) and not before the startup resync.
func (pm *PodManager) GetLocalPods() LocalPods {
	return pm.pods
}

// HandlesEvent select AddPod, DeletePod and any resync events.
func (pm *PodManager) HandlesEvent(event controller.Event) bool {
	if event.Method() != controller.Update {
		return true
	}
	if _, isAddPod := event.(*AddPod); isAddPod {
		return true
	}
	if _, isDeletePod := event.(*DeletePod); isDeletePod {
		return true
	}
	return false
}

// Resync re-synchronizes the map of local pods using information provided
// by Docker server.
func (pm *PodManager) Resync(_ controller.Event, _ controller.KubeStateData,
	resyncCount int, _ controller.ResyncOperations) error {

	// No need to resync the state of running pods in the run-time - local pod
	// will not be added/deleted without the agent knowing about it.
	if resyncCount > 1 {
		return nil
	}

	// list all sandbox containers
	listOpts := docker.ListContainersOptions{
		All: true,
		Filters: map[string][]string{
			"label": {k8sLabelForSandboxContainer},
		},
	}
	containers, err := pm.dockerClient.ListContainers(listOpts)
	if err != nil {
		return controller.NewFatalError(
			fmt.Errorf("failed to list sandbox containers: %v", err))
	}

	// inspect every sandbox to re-construct the pod metadata
	for _, container := range containers {
		if container.State != runningPodState {
			pm.Log.Debugf("Ignoring non-running sandbox container: %v", container.ID)
			continue
		}
		// read pod identifier from labels
		podName, hasPodName := container.Labels[k8sLabelForPodName]
		podNamespace, hasPodNamespace := container.Labels[k8sLabelForPodNamespace]
		podID := podmodel.ID{Name: podName, Namespace: podNamespace}
		if !hasPodName || !hasPodNamespace {
			pm.Log.Warnf("Sandbox container '%s' is missing pod identification\n",
				container.ID)
			continue
		}
		// inspect every sandbox container to obtain the PID, which is used in the network
		// namespace reference
		details, err := pm.dockerClient.InspectContainer(container.ID)
		if err != nil {
			pm.Log.Warnf("Failed to inspect sandbox container '%s': %v\n",
				container.ID, err)
			continue
		}
		// ignore bare (without process) sandbox containers
		if details.State.Pid == 0 {
			continue
		}
		// add pod into the set of running pods
		pm.pods[podID] = &LocalPod{
			ID:               podID,
			ContainerID:      container.ID,
			NetworkNamespace: fmt.Sprintf("/proc/%d/ns/net", details.State.Pid),
		}
		pm.Log.Debugf("Found locally running Pod: %+v", pm.pods[podID])
	}

	pm.Log.Debugf("PodManager state after resync: pods=%s", pm.pods.String())
	return nil
}

// Update handles AddPod and DeletePod events.
func (pm *PodManager) Update(event controller.Event, _ controller.UpdateOperations) (changeDescription string, err error) {
	if addPod, isAddPod := event.(*AddPod); isAddPod {
		// check for obsolete pod entry
		if pod, hasPod := pm.pods[addPod.Pod]; hasPod {
			if pod.ContainerID != addPod.ContainerID ||
				pod.NetworkNamespace != addPod.NetworkNamespace {
				return "", errObsoletePod
			}
		}

		// add pod into the map of local pods
		pm.pods[addPod.Pod] = &LocalPod{
			ID:               addPod.Pod,
			ContainerID:      addPod.ContainerID,
			NetworkNamespace: addPod.NetworkNamespace,
		}
	}
	if deletePod, isDeletePod := event.(*DeletePod); isDeletePod {
		_, hasPod := pm.pods[deletePod.Pod]
		if !hasPod {
			pm.Log.Warnf("Unknown pod to delete: %v", deletePod.Pod)
		} else {
			delete(pm.pods, deletePod.Pod)
		}
	}
	return "", nil
}

// Revert is used to remove a pod entry from the map of local pods when AddPod event fails.
func (pm *PodManager) Revert(event controller.Event) error {
	if addPod, isAddPod := event.(*AddPod); isAddPod {
		delete(pm.pods, addPod.Pod)
	}
	return nil
}

// Add converts CNI Add request to AddPod event.
func (pm *PodManager) Add(ctx context.Context, request *cni.CNIRequest) (reply *cni.CNIReply, err error) {
	pm.Log.Info("Add pod request received ", *request)

	// push AddPod event and wait for the result
	event := NewAddPodEvent(request)
	err = pm.EventLoop.PushEvent(event)
	if err == nil {
		err = event.Wait()
	}

	if err == errObsoletePod {
		// remove the obsolete pod first
		delEvent := NewDeletePodEvent(request)
		err = pm.EventLoop.PushEvent(delEvent)
		if err == nil {
			err = event.Wait()
		}
		if err != nil {
			// treat error as warning
			pm.Log.Warnf("Error while removing obsolete pod container: %v", err)
			err = nil
		}
		// retry AddPod event
		event := NewAddPodEvent(request)
		err = pm.EventLoop.PushEvent(event)
		if err == nil {
			err = event.Wait()
		}
	}

	reply = pm.cniReplyForAddPod(request, event, err)
	return reply, err
}

// Delete converts CNI Delete request to DeletePod event.
func (pm *PodManager) Delete(ctx context.Context, request *cni.CNIRequest) (reply *cni.CNIReply, err error) {
	pm.Log.Info("Delete pod request received ", *request)

	// push DeletePod event and wait for the result
	event := NewDeletePodEvent(request)
	err = pm.EventLoop.PushEvent(event)
	if err == nil {
		err = event.Wait()
	}

	return pm.cniReplyForDeletePod(err), err
}

// cniReplyForAddPod builds CNI reply for processed AddPod event.
func (pm *PodManager) cniReplyForAddPod(request *cni.CNIRequest, event *AddPod, err error) (reply *cni.CNIReply) {
	if err != nil {
		reply = pm.cniErrorReply(err)
		pm.Log.Debugf("CNI Add request reply: %+v", *reply)
		return reply
	}

	reply = &cni.CNIReply{
		Result: cniResultOk,
	}

	// collect interfaces defined by event handlers
	for _, iface := range event.Interfaces {
		cniIface := &cni.CNIReply_Interface{
			Name:    iface.HostName,
			Sandbox: request.NetworkNamespace,
		}
		for _, ipAddr := range iface.IPAddresses {
			cniIface.IpAddresses = append(cniIface.IpAddresses, &cni.CNIReply_Interface_IP{
				Version: cniIPVersion(ipAddr.Version),
				Address: ipAddr.Address.String(),
				Gateway: ipAddr.Gateway.String(),
			})
		}
		reply.Interfaces = append(reply.Interfaces, cniIface)
	}

	// collect routes defined by event handlers
	for _, route := range event.Routes {
		reply.Routes = append(reply.Routes, &cni.CNIReply_Route{
			Dst: route.Network.String(),
			Gw:  route.Gateway.String(),
		})
	}
	pm.Log.Debugf("CNI Add request reply: %+v", *reply)
	return reply
}

// cniReplyForDeletePod builds CNI reply for processed DeletePod event.
func (pm *PodManager) cniReplyForDeletePod(err error) (reply *cni.CNIReply) {
	if err != nil {
		reply = pm.cniErrorReply(err)
	} else {
		reply = &cni.CNIReply{
			Result: cniResultOk,
		}
	}
	pm.Log.Debugf("CNI Delete request reply: %+v", *reply)
	return reply
}

// cniErrorReply returns CNI reply for failed request.
func (pm *PodManager) cniErrorReply(err error) *cni.CNIReply {
	return &cni.CNIReply{
		Result: cniResultErr,
		Error:  err.Error(),
	}
}

// parseCniExtraArgs parses CNI extra arguments from a string into a map.
func parseCniExtraArgs(input string) map[string]string {
	res := map[string]string{}

	pairs := strings.Split(input, ";")
	for i := range pairs {
		kv := strings.Split(pairs[i], "=")
		if len(kv) == 2 {
			res[kv[0]] = kv[1]
		}
	}
	return res
}

func cniIPVersion(version IPVersion) cni.CNIReply_Interface_IP_Version {
	if version == IPv6 {
		return cni.CNIReply_Interface_IP_IPV6
	}
	return cni.CNIReply_Interface_IP_IPV4
}
<commit_msg>Properly handle missing GRPC conf<commit_after>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate protoc -I ./cni --gogo_out=plugins=grpc:./cni ./cni/cni.proto

package podmanager

import (
	"context"
	"fmt"
	"strings"

	"github.com/fsouza/go-dockerclient"

	"github.com/ligato/cn-infra/infra"
	grpcplugin "github.com/ligato/cn-infra/rpc/grpc"

	controller "github.com/contiv/vpp/plugins/controller/api"
	podmodel "github.com/contiv/vpp/plugins/ksr/model/pod"
	"github.com/contiv/vpp/plugins/podmanager/cni"
)

const (
	// name of the CNI request argument that stores pod name
	podNameExtraArg = "K8S_POD_NAME"

	// name of the CNI request argument that stores pod namespace
	podNamespaceExtraArg = "K8S_POD_NAMESPACE"

	// label attached to the sandbox container of every pod
	k8sLabelForSandboxContainer = "io.kubernetes.docker.type=podsandbox"

	// labels attached to (not only sandbox) container to identify the pod it belongs to
	k8sLabelForPodName      = "io.kubernetes.pod.name"
	k8sLabelForPodNamespace = "io.kubernetes.pod.namespace"

	// state value of running pods
	runningPodState = "running"

	// possible return value for CNI requests
	cniResultOk  uint32 = 0
	cniResultErr uint32 = 1
)

// PodManager plugin manages pods deployed on this node. It serves Add/Delete CNI
// requests, converts them to AddPod and DeletePod events, and maintains a map
// of metadata for all locally deployed pods, with enough information for other
// plugins to be able to (re)construct connectivity between pods and the vswitch.
type PodManager struct {
	Deps
	dockerClient DockerClient

	// map of locally deployed pods
	pods LocalPods
}

// Deps lists dependencies of PodManager.
type Deps struct {
	infra.PluginDeps
	EventLoop controller.EventLoop
	GRPC      grpcplugin.Server
}

// DockerClient defines API of a Docker client needed by PodManager.
// The interface allows to inject mock Docker client in the unit tests.
type DockerClient interface {
	// Ping pings the docker server.
	Ping() error
	// ListContainers returns a slice of containers matching the given criteria.
	ListContainers(opts docker.ListContainersOptions) ([]docker.APIContainers, error)
	// InspectContainer returns information about a container by its ID.
	InspectContainer(id string) (*docker.Container, error)
}

var (
	// Error thrown by PodManager.Update (main event loop) when Kubernetes asks
	// to configure pod which is already configured but with different parameters.
	// PodManager.Add() will act by sending an event to remove the obsolete pod
	// connectivity.
	errObsoletePod = fmt.Errorf("obsolete pod configuration detected")
	errMissingDep  = fmt.Errorf("missing mandatory dependency")
)

// Init connects to Docker server and also registers the plugin to serve
// Add/Delete CNI requests.
func (pm *PodManager) Init() (err error) {
	// init attributes
	pm.pods = make(LocalPods)

	// connect to Docker server
	pm.dockerClient, err = docker.NewClientFromEnv()
	if err != nil {
		return err
	}

	if pm.GRPC == nil || pm.GRPC.IsDisabled() {
		return errMissingDep
	}
	cni.RegisterRemoteCNIServer(pm.GRPC.GetServer(), pm)
	return nil
}

// GetLocalPods returns all currently locally deployed pods.
// The method should be called only from within the main event loop
// (not thread safe) and not before the startup resync.
func (pm *PodManager) GetLocalPods() LocalPods {
	return pm.pods
}

// HandlesEvent select AddPod, DeletePod and any resync events.
func (pm *PodManager) HandlesEvent(event controller.Event) bool {
	if event.Method() != controller.Update {
		return true
	}
	if _, isAddPod := event.(*AddPod); isAddPod {
		return true
	}
	if _, isDeletePod := event.(*DeletePod); isDeletePod {
		return true
	}
	return false
}

// Resync re-synchronizes the map of local pods using information provided
// by Docker server.
func (pm *PodManager) Resync(_ controller.Event, _ controller.KubeStateData,
	resyncCount int, _ controller.ResyncOperations) error {

	// No need to resync the state of running pods in the run-time - local pod
	// will not be added/deleted without the agent knowing about it.
	if resyncCount > 1 {
		return nil
	}

	// list all sandbox containers
	listOpts := docker.ListContainersOptions{
		All: true,
		Filters: map[string][]string{
			"label": {k8sLabelForSandboxContainer},
		},
	}
	containers, err := pm.dockerClient.ListContainers(listOpts)
	if err != nil {
		return controller.NewFatalError(
			fmt.Errorf("failed to list sandbox containers: %v", err))
	}

	// inspect every sandbox to re-construct the pod metadata
	for _, container := range containers {
		if container.State != runningPodState {
			pm.Log.Debugf("Ignoring non-running sandbox container: %v", container.ID)
			continue
		}
		// read pod identifier from labels
		podName, hasPodName := container.Labels[k8sLabelForPodName]
		podNamespace, hasPodNamespace := container.Labels[k8sLabelForPodNamespace]
		podID := podmodel.ID{Name: podName, Namespace: podNamespace}
		if !hasPodName || !hasPodNamespace {
			pm.Log.Warnf("Sandbox container '%s' is missing pod identification\n",
				container.ID)
			continue
		}
		// inspect every sandbox container to obtain the PID, which is used in the network
		// namespace reference
		details, err := pm.dockerClient.InspectContainer(container.ID)
		if err != nil {
			pm.Log.Warnf("Failed to inspect sandbox container '%s': %v\n",
				container.ID, err)
			continue
		}
		// ignore bare (without process) sandbox containers
		if details.State.Pid == 0 {
			continue
		}
		// add pod into the set of running pods
		pm.pods[podID] = &LocalPod{
			ID:               podID,
			ContainerID:      container.ID,
			NetworkNamespace: fmt.Sprintf("/proc/%d/ns/net", details.State.Pid),
		}
		pm.Log.Debugf("Found locally running Pod: %+v", pm.pods[podID])
	}

	pm.Log.Debugf("PodManager state after resync: pods=%s", pm.pods.String())
	return nil
}

// Update handles AddPod and DeletePod events.
func (pm *PodManager) Update(event controller.Event, _ controller.UpdateOperations) (changeDescription string, err error) {
	if addPod, isAddPod := event.(*AddPod); isAddPod {
		// check for obsolete pod entry
		if pod, hasPod := pm.pods[addPod.Pod]; hasPod {
			if pod.ContainerID != addPod.ContainerID ||
				pod.NetworkNamespace != addPod.NetworkNamespace {
				return "", errObsoletePod
			}
		}

		// add pod into the map of local pods
		pm.pods[addPod.Pod] = &LocalPod{
			ID:               addPod.Pod,
			ContainerID:      addPod.ContainerID,
			NetworkNamespace: addPod.NetworkNamespace,
		}
	}
	if deletePod, isDeletePod := event.(*DeletePod); isDeletePod {
		_, hasPod := pm.pods[deletePod.Pod]
		if !hasPod {
			pm.Log.Warnf("Unknown pod to delete: %v", deletePod.Pod)
		} else {
			delete(pm.pods, deletePod.Pod)
		}
	}
	return "", nil
}

// Revert is used to remove a pod entry from the map of local pods when AddPod event fails.
func (pm *PodManager) Revert(event controller.Event) error {
	if addPod, isAddPod := event.(*AddPod); isAddPod {
		delete(pm.pods, addPod.Pod)
	}
	return nil
}

// Add converts CNI Add request to AddPod event.
func (pm *PodManager) Add(ctx context.Context, request *cni.CNIRequest) (reply *cni.CNIReply, err error) {
	pm.Log.Info("Add pod request received ", *request)

	// push AddPod event and wait for the result
	event := NewAddPodEvent(request)
	err = pm.EventLoop.PushEvent(event)
	if err == nil {
		err = event.Wait()
	}

	if err == errObsoletePod {
		// remove the obsolete pod first
		delEvent := NewDeletePodEvent(request)
		err = pm.EventLoop.PushEvent(delEvent)
		if err == nil {
			err = event.Wait()
		}
		if err != nil {
			// treat error as warning
			pm.Log.Warnf("Error while removing obsolete pod container: %v", err)
			err = nil
		}
		// retry AddPod event
		event := NewAddPodEvent(request)
		err = pm.EventLoop.PushEvent(event)
		if err == nil {
			err = event.Wait()
		}
	}

	reply = pm.cniReplyForAddPod(request, event, err)
	return reply, err
}

// Delete converts CNI Delete request to DeletePod event.
func (pm *PodManager) Delete(ctx context.Context, request *cni.CNIRequest) (reply *cni.CNIReply, err error) {
	pm.Log.Info("Delete pod request received ", *request)

	// push DeletePod event and wait for the result
	event := NewDeletePodEvent(request)
	err = pm.EventLoop.PushEvent(event)
	if err == nil {
		err = event.Wait()
	}

	return pm.cniReplyForDeletePod(err), err
}

// cniReplyForAddPod builds CNI reply for processed AddPod event.
func (pm *PodManager) cniReplyForAddPod(request *cni.CNIRequest, event *AddPod, err error) (reply *cni.CNIReply) {
	if err != nil {
		reply = pm.cniErrorReply(err)
		pm.Log.Debugf("CNI Add request reply: %+v", *reply)
		return reply
	}

	reply = &cni.CNIReply{
		Result: cniResultOk,
	}

	// collect interfaces defined by event handlers
	for _, iface := range event.Interfaces {
		cniIface := &cni.CNIReply_Interface{
			Name:    iface.HostName,
			Sandbox: request.NetworkNamespace,
		}
		for _, ipAddr := range iface.IPAddresses {
			cniIface.IpAddresses = append(cniIface.IpAddresses, &cni.CNIReply_Interface_IP{
				Version: cniIPVersion(ipAddr.Version),
				Address: ipAddr.Address.String(),
				Gateway: ipAddr.Gateway.String(),
			})
		}
		reply.Interfaces = append(reply.Interfaces, cniIface)
	}

	// collect routes defined by event handlers
	for _, route := range event.Routes {
		reply.Routes = append(reply.Routes, &cni.CNIReply_Route{
			Dst: route.Network.String(),
			Gw:  route.Gateway.String(),
		})
	}
	pm.Log.Debugf("CNI Add request reply: %+v", *reply)
	return reply
}

// cniReplyForDeletePod builds CNI reply for processed DeletePod event.
func (pm *PodManager) cniReplyForDeletePod(err error) (reply *cni.CNIReply) {
	if err != nil {
		reply = pm.cniErrorReply(err)
	} else {
		reply = &cni.CNIReply{
			Result: cniResultOk,
		}
	}
	pm.Log.Debugf("CNI Delete request reply: %+v", *reply)
	return reply
}

// cniErrorReply returns CNI reply for failed request.
func (pm *PodManager) cniErrorReply(err error) *cni.CNIReply {
	return &cni.CNIReply{
		Result: cniResultErr,
		Error:  err.Error(),
	}
}

// parseCniExtraArgs parses CNI extra arguments from a string into a map.
func parseCniExtraArgs(input string) map[string]string {
	res := map[string]string{}

	pairs := strings.Split(input, ";")
	for i := range pairs {
		kv := strings.Split(pairs[i], "=")
		if len(kv) == 2 {
			res[kv[0]] = kv[1]
		}
	}
	return res
}

func cniIPVersion(version IPVersion) cni.CNIReply_Interface_IP_Version {
	if version == IPv6 {
		return cni.CNIReply_Interface_IP_IPV6
	}
	return cni.CNIReply_Interface_IP_IPV4
}
<|endoftext|>


<commit_before>package models

import (
	"time"

	"github.com/jinzhu/now"
	"github.com/qor/qor-example/db"
)

type Chart struct {
	Total string
	Date  time.Time
}

/*
date format 2015-01-23
*/
func GetChartData(table, start, end string) (res []Chart) {
	startdate, err := time.Parse("2006-01-02", start)
	if err != nil {
		return
	}
	enddate, err := time.Parse("2006-01-02", end)
	if err != nil || enddate.UnixNano() < startdate.UnixNano() {
		enddate = now.EndOfDay()
	} else {
		enddate.AddDate(0, 0, 1)
	}
	db.DB.Table(table).Where("created_at > ? AND created_at < ?", startdate, enddate).Select("date(created_at) as date, count(1) as total").Group("date(created_at)").Scan(&res)
	return
}
<commit_msg>Fix time zone problem by using now parse time<commit_after>package models

import (
	"time"

	"github.com/jinzhu/now"
	"github.com/qor/qor-example/db"
)

type Chart struct {
	Total string
	Date  time.Time
}

/*
date format 2015-01-23
*/
func GetChartData(table, start, end string) (res []Chart) {
	startdate, err := now.Parse(start)
	if err != nil {
		return
	}

	enddate, err := now.Parse(end)
	if err != nil || enddate.UnixNano() < startdate.UnixNano() {
		enddate = now.EndOfDay()
	} else {
		enddate.AddDate(0, 0, 1)
	}

	db.DB.Table(table).Where("created_at > ? AND created_at < ?", startdate, enddate).Select("date(created_at) as date, count(*) as total").Group("date(created_at)").Scan(&res)
	return
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/domain/host"
	"github.com/zenoss/glog"
)

const monitorSubDir string = "/monitor"

// Monitor monitors the exporting of a file system to clients.
type Monitor struct {
	driver StorageDriver

	monitoredHosts     map[string]bool
	monitoredHostsLock sync.RWMutex

	shouldRestart   bool
	monitorInterval time.Duration
	previousRestart time.Time

	conn               client.Connection
	storageClientsPath string
}

// NewMonitor returns a Monitor object to monitor the exported file system
func NewMonitor(driver StorageDriver, monitorInterval time.Duration) (*Monitor, error) {
	m := &Monitor{
		driver:          driver,
		monitoredHosts:  make(map[string]bool),
		shouldRestart:   getShouldRestartDFSOnFailure(),
		monitorInterval: monitorInterval,
	}

	return m, nil
}

// UpdateRemoteMonitorFile is used by remote clients to write a tiny file to the DFS volume at the given cycle
func UpdateRemoteMonitorFile(localPath string, writeInterval time.Duration, ipAddr string, shutdown <-chan interface{}) {
	remoteFile := path.Join(localPath, monitorSubDir, ipAddr)
	glog.Infof("updating DFS volume monitor file %s at write interval: %s", remoteFile, writeInterval)

	for {
		glog.V(2).Infof("writing DFS file %s", remoteFile)
		if err := ioutil.WriteFile(remoteFile, []byte(ipAddr), 0600); err != nil {
			glog.Warningf("unable to write DFS file %s: %s", remoteFile, err)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(writeInterval):

		case <-shutdown:
			glog.Infof("no longer writing remote monitor status for DFS volume %s to %s", localPath, remoteFile)
			return
		}
	}
}

// SetMonitorStorageClients sets the monitored remote IPs that are active
func (m *Monitor) SetMonitorStorageClients(conn client.Connection, storageClientsPath string, clients ...string) {
	m.conn = conn
	m.storageClientsPath = storageClientsPath
	m.SetMonitorRemoteHosts(getActiveRemoteHosts(conn, storageClientsPath, m.monitorInterval, clients...)...)
}

// SetMonitorRemoteHosts sets the remote hosts to monitor
func (m *Monitor) SetMonitorRemoteHosts(ipAddrs ...string) {
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	for _, ipAddr := range ipAddrs {
		m.monitoredHosts[ipAddr] = true
	}
}

// setMonitorRemoteHost updates the status of the monitored remote
func (m *Monitor) setMonitorRemoteHost(ipAddr string, hasUpdated bool) {
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	m.monitoredHosts[ipAddr] = hasUpdated
}

// getMonitorRemoteHosts returns the statuses of the monitored remotes
func (m *Monitor) getMonitorRemoteHosts() map[string]bool {
	m.monitoredHostsLock.RLock()
	defer m.monitoredHostsLock.RUnlock()
	return m.monitoredHosts
}

// ProcessDFSVolumePollUpdateFunc is called by MonitorDFSVolume when updates are detected
type DFSVolumeMonitorPollUpdateFunc func(mountpoint, remoteIP string, isUpdated bool)

// DFSVolumeMonitorPollUpdateFunc restarts nfs based on status of monitored remotes
func (m *Monitor) DFSVolumeMonitorPollUpdateFunc(mountpoint, remoteIP string, hasUpdatedFile bool) {
	// monitor dfs; log warnings each cycle; restart dfs if needed

	if hasUpdatedFile {
		return
	} else if len(m.getMonitorRemoteHosts()) == 0 {
		return
	}

	glog.Warningf("DFS NFS volume %s is not seen by remoteIP:%s - further action may be needed i.e: restart nfs", mountpoint, remoteIP)
	now := time.Now()
	since := now.Sub(m.previousRestart)
	if !m.shouldRestart {
		glog.Warningf("Not restarting DFS NFS service due to configuration setting: SERVICED_MONITOR_DFS_MASTER_RESTART=0")
		return
	} else if since < m.monitorInterval {
		glog.Warningf("Not restarting DFS NFS service - have not surpassed interval: %s since last restart", m.monitorInterval)
		return
	}

	m.previousRestart = now
	if err := m.driver.Restart(); err != nil {
		glog.Errorf("Error restarting DFS NFS service: %s", err)
	}
}

// MonitorVolume monitors the DFS volume - logs on failure and calls pollUpdateFunc
func (m *Monitor) MonitorDFSVolume(mountpoint string, shutdown <-chan interface{}, pollUpdateFunc DFSVolumeMonitorPollUpdateFunc) {
	glog.Infof("monitoring DFS export info for DFS volume %s at polling interval: %s", mountpoint, m.monitorInterval)

	m.previousRestart = time.Now()

	monitorPath := path.Join(mountpoint, monitorSubDir)
	os.RemoveAll(monitorPath)
	if err := os.MkdirAll(monitorPath, 0755); err != nil {
		glog.Errorf("no longer monitoring status for DFS volume %s - unable to mkdir %+v: %s", mountpoint, monitorPath, err)
		return
	}

	for {
		// check all active remotes
		remotes := m.getMonitorRemoteHosts()
		remoteIPs := make([]string, 0, len(remotes))
		for k := range remotes {
			remoteIPs = append(remoteIPs, k)
		}

		activeRemotes := remoteIPs
		if m.conn != nil && len(m.storageClientsPath) != 0 {
			activeRemotes = getActiveRemoteHosts(m.conn, m.storageClientsPath, m.monitorInterval, remoteIPs...)
			glog.V(0).Infof("==== active remotes: %s checked from list of remotes: %s", activeRemotes, remoteIPs) // V(2)
		}
		// TODO: remoteIPs = getAllRemoteHostsFromZookeeper()
		for _, remoteIP := range activeRemotes {
			remoteFile := path.Join(monitorPath, remoteIP)
			sinceModified, err := fileSinceModified(remoteFile)
			hasUpdatedFile := false
			if err != nil {
				glog.Warningf("remote agent %s is not synced on DFS volume %s for file %s: %s", remoteIP, mountpoint, remoteFile, err)
			} else if sinceModified > m.monitorInterval {
				glog.Warningf("remote agent %s is not synced DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
			} else {
				glog.V(4).Infof("remote agent %s is synced on DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
				hasUpdatedFile = true
			}

			m.setMonitorRemoteHost(remoteIP, hasUpdatedFile)
			pollUpdateFunc(mountpoint, remoteIP, hasUpdatedFile)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(m.monitorInterval):

		case <-shutdown:
			glog.Infof("no longer monitoring status for DFS volume %s", mountpoint)
			return
		}
	}
}

// fileSinceModified returns the duration of the file since last modified
func fileSinceModified(filename string) (time.Duration, error) {
	fileinfo, err := os.Stat(filename)
	if err != nil {
		glog.V(2).Infof("unable to stat file: %s %s", filename, err)
		return 0, err
	}

	timeSince := time.Now().Sub(fileinfo.ModTime())
	glog.V(4).Infof("file %s was modified %s ago", filename, timeSince)
	return timeSince, nil
}

// Get the variable determining if we will restart NFS when we detect a failure
func getShouldRestartDFSOnFailure() bool {
	shouldRestart := os.Getenv("SERVICED_MONITOR_DFS_MASTER_RESTART")
	if len(shouldRestart) == 0 {
		glog.Infof("defaulting SERVICED_MONITOR_DFS_MASTER_RESTART to true")
		return true
	} else if shouldRestart == "1" {
		return true
	} else {
		return false
	}
}

// getDefaultDFSMonitorRemoteInterval returns the duration specified in seconds for the env var SERVICED_DFS_MONITOR_REMOTE_UPDATE_INTERVAL
func getDefaultDFSMonitorRemoteInterval() time.Duration {
	return getEnvMinDuration("SERVICED_MONITOR_DFS_REMOTE_UPDATE_INTERVAL", 60, 60)
}

func getDefaultNFSMonitorMasterInterval() time.Duration {
	min := getDefaultDFSMonitorRemoteInterval()
	return getEnvMinDuration("SERVICED_MONITOR_DFS_MASTER_INTERVAL", 900, int32(min.Seconds())*2)
}

// getEnvMinDuration returns the time.Duration env var meeting minimum and default duration
func getEnvMinDuration(envvar string, def, min int32) time.Duration {
	duration := def
	envval := os.Getenv(envvar)
	if len(strings.TrimSpace(envval)) == 0 {
		// ignore unset envvar
	} else if intVal, intErr := strconv.ParseInt(envval, 0, 32); intErr != nil {
		glog.Warningf("ignoring invalid %s of '%s': %s", envvar, envval, intErr)
		duration = min
	} else if int32(intVal) < min {
		glog.Warningf("ignoring invalid %s of '%s' < minimum:%v seconds", envvar, envval, min)
	} else {
		duration = int32(intVal)
	}

	return time.Duration(duration) * time.Second
}

// StorageClientHostNode is the zookeeper node for a host
type StorageClientHostNode struct {
	host.Host
	version interface{}
}

// Version is an implementation of client.Node
func (v *StorageClientHostNode) Version() interface{} { return v.version }

// SetVersion is an implementation of client.Node
func (v *StorageClientHostNode) SetVersion(version interface{}) { v.version = version }

// getActiveRemoteHosts returns a slice of activeClientIPs
func getActiveRemoteHosts(conn client.Connection, zpath string, monitorInterval time.Duration, clients ...string) []string {
	var activeClientIPs []string
	now := time.Now()
	for _, clnt := range clients {
		cp := path.Join(zpath, clnt)
		glog.V(0).Infof("retrieving info for DFS for remoteIP %s at zookeeper node %s", clnt, cp) // V2

		hnode := StorageClientHostNode{}
		err := conn.Get(cp, &hnode)
		if err != nil && err != client.ErrEmptyNode {
			glog.Errorf("DFS could not get zookeeper hostnode %s: %s", cp, err)
			continue
		}
		if hnode.Host.UpdatedAt.IsZero() {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v", clnt, hnode.ID, hnode.UpdatedAt)
			continue
		}

		elapsed := now.Sub(hnode.Host.UpdatedAt)
		glog.V(0).Infof("retrieved info for DFS for remoteIP %s  UpdatedAt:%s  elapsed:%s  monitorInterval:%s", clnt, hnode.Host.UpdatedAt, elapsed, monitorInterval) // V2
		if elapsed > monitorInterval {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v  lastseen:%s ago", clnt, hnode.ID, hnode.UpdatedAt, elapsed)
			continue
		}

		activeClientIPs = append(activeClientIPs, clnt)
	}

	glog.V(0).Infof("DFS remote active IPs: %+v", activeClientIPs) // V2
	return activeClientIPs
}
<commit_msg>reduced verbose log messages in storage/monitor.go<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/domain/host"
	"github.com/zenoss/glog"
)

const monitorSubDir string = "/monitor"

// Monitor monitors the exporting of a file system to clients.
type Monitor struct {
	driver StorageDriver

	monitoredHosts     map[string]bool
	monitoredHostsLock sync.RWMutex

	shouldRestart   bool
	monitorInterval time.Duration
	previousRestart time.Time

	conn               client.Connection
	storageClientsPath string
}

// NewMonitor returns a Monitor object to monitor the exported file system
func NewMonitor(driver StorageDriver, monitorInterval time.Duration) (*Monitor, error) {
	m := &Monitor{
		driver:          driver,
		monitoredHosts:  make(map[string]bool),
		shouldRestart:   getShouldRestartDFSOnFailure(),
		monitorInterval: monitorInterval,
	}

	return m, nil
}

// UpdateRemoteMonitorFile is used by remote clients to write a tiny file to the DFS volume at the given cycle
func UpdateRemoteMonitorFile(localPath string, writeInterval time.Duration, ipAddr string, shutdown <-chan interface{}) {
	remoteFile := path.Join(localPath, monitorSubDir, ipAddr)
	glog.Infof("updating DFS volume monitor file %s at write interval: %s", remoteFile, writeInterval)

	for {
		glog.V(2).Infof("writing DFS file %s", remoteFile)
		if err := ioutil.WriteFile(remoteFile, []byte(ipAddr), 0600); err != nil {
			glog.Warningf("unable to write DFS file %s: %s", remoteFile, err)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(writeInterval):

		case <-shutdown:
			glog.Infof("no longer writing remote monitor status for DFS volume %s to %s", localPath, remoteFile)
			return
		}
	}
}

// SetMonitorStorageClients sets the monitored remote IPs that are active
func (m *Monitor) SetMonitorStorageClients(conn client.Connection, storageClientsPath string, clients ...string) {
	m.conn = conn
	m.storageClientsPath = storageClientsPath
	m.SetMonitorRemoteHosts(getActiveRemoteHosts(conn, storageClientsPath, m.monitorInterval, clients...)...)
}

// SetMonitorRemoteHosts sets the remote hosts to monitor
func (m *Monitor) SetMonitorRemoteHosts(ipAddrs ...string) {
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	for _, ipAddr := range ipAddrs {
		m.monitoredHosts[ipAddr] = true
	}
}

// setMonitorRemoteHost updates the status of the monitored remote
func (m *Monitor) setMonitorRemoteHost(ipAddr string, hasUpdated bool) {
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	m.monitoredHosts[ipAddr] = hasUpdated
}

// getMonitorRemoteHosts returns the statuses of the monitored remotes
func (m *Monitor) getMonitorRemoteHosts() map[string]bool {
	m.monitoredHostsLock.RLock()
	defer m.monitoredHostsLock.RUnlock()
	return m.monitoredHosts
}

// ProcessDFSVolumePollUpdateFunc is called by MonitorDFSVolume when updates are detected
type DFSVolumeMonitorPollUpdateFunc func(mountpoint, remoteIP string, isUpdated bool)

// DFSVolumeMonitorPollUpdateFunc restarts nfs based on status of monitored remotes
func (m *Monitor) DFSVolumeMonitorPollUpdateFunc(mountpoint, remoteIP string, hasUpdatedFile bool) {
	// monitor dfs; log warnings each cycle; restart dfs if needed

	if hasUpdatedFile {
		return
	} else if len(m.getMonitorRemoteHosts()) == 0 {
		return
	}

	glog.Warningf("DFS NFS volume %s is not seen by remoteIP:%s - further action may be needed i.e: restart nfs", mountpoint, remoteIP)
	now := time.Now()
	since := now.Sub(m.previousRestart)
	if !m.shouldRestart {
		glog.Warningf("Not restarting DFS NFS service due to configuration setting: SERVICED_MONITOR_DFS_MASTER_RESTART=0")
		return
	} else if since < m.monitorInterval {
		glog.Warningf("Not restarting DFS NFS service - have not surpassed interval: %s since last restart", m.monitorInterval)
		return
	}

	m.previousRestart = now
	if err := m.driver.Restart(); err != nil {
		glog.Errorf("Error restarting DFS NFS service: %s", err)
	}
}

// MonitorVolume monitors the DFS volume - logs on failure and calls pollUpdateFunc
func (m *Monitor) MonitorDFSVolume(mountpoint string, shutdown <-chan interface{}, pollUpdateFunc DFSVolumeMonitorPollUpdateFunc) {
	glog.Infof("monitoring DFS export info for DFS volume %s at polling interval: %s", mountpoint, m.monitorInterval)

	m.previousRestart = time.Now()

	monitorPath := path.Join(mountpoint, monitorSubDir)
	os.RemoveAll(monitorPath)
	if err := os.MkdirAll(monitorPath, 0755); err != nil {
		glog.Errorf("no longer monitoring status for DFS volume %s - unable to mkdir %+v: %s", mountpoint, monitorPath, err)
		return
	}

	for {
		// check all active remotes
		remotes := m.getMonitorRemoteHosts()
		remoteIPs := make([]string, 0, len(remotes))
		for k := range remotes {
			remoteIPs = append(remoteIPs, k)
		}

		activeRemotes := remoteIPs
		if m.conn != nil && len(m.storageClientsPath) != 0 {
			// TODO: remoteIPs = getAllRemoteHostsFromZookeeper() when deletion of zookeeper nodes are implemented in server.go
			activeRemotes = getActiveRemoteHosts(m.conn, m.storageClientsPath, m.monitorInterval, remoteIPs...)
			glog.V(2).Infof("==== active remotes: %s checked from list of remotes: %s", activeRemotes, remoteIPs)
		}

		for _, remoteIP := range activeRemotes {
			remoteFile := path.Join(monitorPath, remoteIP)
			sinceModified, err := fileSinceModified(remoteFile)
			hasUpdatedFile := false
			if err != nil {
				glog.Warningf("remote agent %s is not synced on DFS volume %s for file %s: %s", remoteIP, mountpoint, remoteFile, err)
			} else if sinceModified > m.monitorInterval {
				glog.Warningf("remote agent %s is not synced DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
			} else {
				glog.V(4).Infof("remote agent %s is synced on DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
				hasUpdatedFile = true
			}

			m.setMonitorRemoteHost(remoteIP, hasUpdatedFile)
			pollUpdateFunc(mountpoint, remoteIP, hasUpdatedFile)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(m.monitorInterval):

		case <-shutdown:
			glog.Infof("no longer monitoring status for DFS volume %s", mountpoint)
			return
		}
	}
}

// fileSinceModified returns the duration of the file since last modified
func fileSinceModified(filename string) (time.Duration, error) {
	fileinfo, err := os.Stat(filename)
	if err != nil {
		glog.V(2).Infof("unable to stat file: %s %s", filename, err)
		return 0, err
	}

	timeSince := time.Now().Sub(fileinfo.ModTime())
	glog.V(4).Infof("file %s was modified %s ago", filename, timeSince)
	return timeSince, nil
}

// Get the variable determining if we will restart NFS when we detect a failure
func getShouldRestartDFSOnFailure() bool {
	shouldRestart := os.Getenv("SERVICED_MONITOR_DFS_MASTER_RESTART")
	if len(shouldRestart) == 0 {
		glog.Infof("defaulting SERVICED_MONITOR_DFS_MASTER_RESTART to true")
		return true
	} else if shouldRestart == "1" {
		return true
	} else {
		return false
	}
}

// getDefaultDFSMonitorRemoteInterval returns the duration specified in seconds for the env var SERVICED_DFS_MONITOR_REMOTE_UPDATE_INTERVAL
func getDefaultDFSMonitorRemoteInterval() time.Duration {
	return getEnvMinDuration("SERVICED_MONITOR_DFS_REMOTE_UPDATE_INTERVAL", 60, 60)
}

func getDefaultNFSMonitorMasterInterval() time.Duration {
	min := getDefaultDFSMonitorRemoteInterval()
	return getEnvMinDuration("SERVICED_MONITOR_DFS_MASTER_INTERVAL", 900, int32(min.Seconds())*2)
}

// getEnvMinDuration returns the time.Duration env var meeting minimum and default duration
func getEnvMinDuration(envvar string, def, min int32) time.Duration {
	duration := def
	envval := os.Getenv(envvar)
	if len(strings.TrimSpace(envval)) == 0 {
		// ignore unset envvar
	} else if intVal, intErr := strconv.ParseInt(envval, 0, 32); intErr != nil {
		glog.Warningf("ignoring invalid %s of '%s': %s", envvar, envval, intErr)
		duration = min
	} else if int32(intVal) < min {
		glog.Warningf("ignoring invalid %s of '%s' < minimum:%v seconds", envvar, envval, min)
	} else {
		duration = int32(intVal)
	}

	return time.Duration(duration) * time.Second
}

// StorageClientHostNode is the zookeeper node for a host
type StorageClientHostNode struct {
	host.Host
	version interface{}
}

// Version is an implementation of client.Node
func (v *StorageClientHostNode) Version() interface{} { return v.version }

// SetVersion is an implementation of client.Node
func (v *StorageClientHostNode) SetVersion(version interface{}) { v.version = version }

// getActiveRemoteHosts returns a slice of activeClientIPs
func getActiveRemoteHosts(conn client.Connection, zpath string, monitorInterval time.Duration, clients ...string) []string {
	var activeClientIPs []string
	now := time.Now()
	for _, clnt := range clients {
		cp := path.Join(zpath, clnt)
		glog.V(2).Infof("retrieving info for DFS for remoteIP %s at zookeeper node %s", clnt, cp)

		hnode := StorageClientHostNode{}
		err := conn.Get(cp, &hnode)
		if err != nil && err != client.ErrEmptyNode {
			glog.Errorf("DFS could not get zookeeper hostnode %s: %s", cp, err)
			continue
		}
		if hnode.Host.UpdatedAt.IsZero() {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v", clnt, hnode.ID, hnode.UpdatedAt)
			continue
		}

		elapsed := now.Sub(hnode.Host.UpdatedAt)
		glog.V(2).Infof("retrieved info for DFS for remoteIP %s  UpdatedAt:%s  elapsed:%s  monitorInterval:%s", clnt, hnode.Host.UpdatedAt, elapsed, monitorInterval)
		if elapsed > monitorInterval {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v  lastseen:%s ago", clnt, hnode.ID, hnode.UpdatedAt, elapsed)
			continue
		}

		activeClientIPs = append(activeClientIPs, clnt)
	}

	glog.V(2).Infof("DFS remote active IPs: %+v", activeClientIPs)
	return activeClientIPs
}
<|endoftext|>


<commit_before>package kuzzle

import (
  "errors"
  "github.com/kuzzleio/sdk-go/types"
  "encoding/json"
  "github.com/kuzzleio/sdk-go/event"
)

func (k *Kuzzle) Login(strategy string, credentials interface{}, expiresIn *int) (string, error) {
  if strategy == "" {
    return "", errors.New("Kuzzle.Login: cannot authenticate to Kuzzle without an authentication strategy")
  }

  type loginResult struct {
    Jwt string `json:"jwt"`
  }

  var token loginResult
  var body interface{}

  if credentials != nil {
    body = credentials
  }

  q := types.KuzzleRequest{
    Controller: "auth",
    Action:     "login",
    Body:       body,
    Strategy:   strategy,
  }

  if expiresIn != nil {
    q.ExpiresIn = *expiresIn
  }

  result := make(chan types.KuzzleResponse)

  go k.Query(q, nil, result)

  res := <-result

  json.Unmarshal(res.Result, &token)

  if res.Error.Message != "" {
    err := errors.New(res.Error.Message)
    k.socket.EmitEvent(event.LoginAttempt, types.LoginAttempt{Success: true, Error: err})
    return "", err
  }

  if token.Jwt != "" {
    // todo renew subscriptions
    k.socket.EmitEvent(event.LoginAttempt, types.LoginAttempt{Success: true})
  }

  return token.Jwt, nil
}
<commit_msg>Update login.go<commit_after>package kuzzle

import (
  "errors"
  "github.com/kuzzleio/sdk-go/types"
  "encoding/json"
  "github.com/kuzzleio/sdk-go/event"
)

func (k *Kuzzle) Login(strategy string, credentials interface{}, expiresIn *int) (string, error) {
  if strategy == "" {
    return "", errors.New("Kuzzle.Login: cannot authenticate to Kuzzle without an authentication strategy")
  }

  type loginResult struct {
    Jwt string `json:"jwt"`
  }

  var token loginResult
  var body interface{}

  if credentials != nil {
    body = credentials
  }

  q := types.KuzzleRequest{
    Controller: "auth",
    Action:     "login",
    Body:       body,
    Strategy:   strategy,
  }

  if expiresIn != nil {
    q.ExpiresIn = *expiresIn
  }

  result := make(chan types.KuzzleResponse)

  go k.Query(q, nil, result)

  res := <-result

  json.Unmarshal(res.Result, &token)

  if res.Error.Message != "" {
    err := errors.New(res.Error.Message)
    k.socket.EmitEvent(event.LoginAttempt, types.LoginAttempt{Success: false, Error: err})
    return "", err
  }

  if token.Jwt != "" {
    // todo renew subscriptions
    k.socket.EmitEvent(event.LoginAttempt, types.LoginAttempt{Success: true})
  }

  return token.Jwt, nil
}
<|endoftext|>


<commit_before>// Package config contains the model and loader of the goreleaser configuration
// file.
package config

import (
	"io/ioutil"

	yaml "gopkg.in/yaml.v1"
)

// Repo represents any kind of repo (github, gitlab, etc)
type Repo struct {
	Owner string `yaml:",omitempty"`
	Name  string `yaml:",omitempty"`
}

// String of the repo, e.g. owner/name
func (r Repo) String() string {
	return r.Owner + "/" + r.Name
}

// Homebrew contains the brew section
type Homebrew struct {
	GitHub       Repo     `yaml:",omitempty"`
	Folder       string   `yaml:",omitempty"`
	Caveats      string   `yaml:",omitempty"`
	Plist        string   `yaml:",omitempty"`
	Install      string   `yaml:",omitempty"`
	Dependencies []string `yaml:",omitempty"`
	Conflicts    []string `yaml:",omitempty"`
	Description  string   `yaml:",omitempty"`
	Homepage     string   `yaml:",omitempty"`
}

// Hooks define actions to run before and/or after something
type Hooks struct {
	Pre  string `yaml:",omitempty"`
	Post string `yaml:",omitempty"`
}

// IgnoredBuild represents a build ignored by the user
type IgnoredBuild struct {
	Goos, Goarch, Goarm string
}

// Build contains the build configuration section
type Build struct {
	Goos    []string       `yaml:",omitempty"`
	Goarch  []string       `yaml:",omitempty"`
	Goarm   []string       `yaml:",omitempty"`
	Ignore  []IgnoredBuild `yaml:",omitempty"`
	Main    string         `yaml:",omitempty"`
	Ldflags string         `yaml:",omitempty"`
	Flags   string         `yaml:",omitempty"`
	Binary  string         `yaml:",omitempty"`
	Hooks   Hooks          `yaml:",omitempty"`
}

// FormatOverride is used to specify a custom format for a specific GOOS.
type FormatOverride struct {
	Goos   string `yaml:",omitempty"`
	Format string `yaml:",omitempty"`
}

// Archive config used for the archive
type Archive struct {
	Format          string            `yaml:",omitempty"`
	FormatOverrides []FormatOverride  `yaml:"format_overrides,omitempty"`
	NameTemplate    string            `yaml:"name_template,omitempty"`
	Replacements    map[string]string `yaml:",omitempty"`
	Files           []string          `yaml:",omitempty"`
}

// Release config used for the GitHub release
type Release struct {
	GitHub Repo `yaml:",omitempty"`
	Draft  bool `yaml:",omitempty"`
}

// FPM config
type FPM struct {
	Formats      []string `yaml:",omitempty"`
	Dependencies []string `yaml:",omitempty"`
	Conflicts    []string `yaml:",omitempty"`
	Vendor       string   `yaml:",omitempty"`
	Homepage     string   `yaml:",omitempty"`
	Maintainer   string   `yaml:",omitempty"`
	Description  string   `yaml:",omitempty"`
	License      string   `yaml:",omitempty"`
}

// Snapshot config
type Snapshot struct {
	NameTemplate string `yaml:"name_template,omitempty"`
}

// Project includes all project configuration
type Project struct {
	Release  Release  `yaml:",omitempty"`
	Brew     Homebrew `yaml:",omitempty"`
	Build    Build    `yaml:",omitempty"`
	Archive  Archive  `yaml:",omitempty"`
	FPM      FPM      `yaml:",omitempty"`
	Snapshot Snapshot `yaml:",omitempty"`

	// test only property indicating the path to the dist folder
	Dist string `yaml:"-"`
}

// Load config file
func Load(file string) (config Project, err error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return config, err
	}
	err = yaml.Unmarshal(data, &config)
	return
}
<commit_msg>Add LoadReader function<commit_after>// Package config contains the model and loader of the goreleaser configuration
// file.
package config

import (
	"io"
	"io/ioutil"

	yaml "gopkg.in/yaml.v1"
)

// Repo represents any kind of repo (github, gitlab, etc)
type Repo struct {
	Owner string `yaml:",omitempty"`
	Name  string `yaml:",omitempty"`
}

// String of the repo, e.g. owner/name
func (r Repo) String() string {
	return r.Owner + "/" + r.Name
}

// Homebrew contains the brew section
type Homebrew struct {
	GitHub       Repo     `yaml:",omitempty"`
	Folder       string   `yaml:",omitempty"`
	Caveats      string   `yaml:",omitempty"`
	Plist        string   `yaml:",omitempty"`
	Install      string   `yaml:",omitempty"`
	Dependencies []string `yaml:",omitempty"`
	Conflicts    []string `yaml:",omitempty"`
	Description  string   `yaml:",omitempty"`
	Homepage     string   `yaml:",omitempty"`
}

// Hooks define actions to run before and/or after something
type Hooks struct {
	Pre  string `yaml:",omitempty"`
	Post string `yaml:",omitempty"`
}

// IgnoredBuild represents a build ignored by the user
type IgnoredBuild struct {
	Goos, Goarch, Goarm string
}

// Build contains the build configuration section
type Build struct {
	Goos    []string       `yaml:",omitempty"`
	Goarch  []string       `yaml:",omitempty"`
	Goarm   []string       `yaml:",omitempty"`
	Ignore  []IgnoredBuild `yaml:",omitempty"`
	Main    string         `yaml:",omitempty"`
	Ldflags string         `yaml:",omitempty"`
	Flags   string         `yaml:",omitempty"`
	Binary  string         `yaml:",omitempty"`
	Hooks   Hooks          `yaml:",omitempty"`
}

// FormatOverride is used to specify a custom format for a specific GOOS.
type FormatOverride struct {
	Goos   string `yaml:",omitempty"`
	Format string `yaml:",omitempty"`
}

// Archive config used for the archive
type Archive struct {
	Format          string            `yaml:",omitempty"`
	FormatOverrides []FormatOverride  `yaml:"format_overrides,omitempty"`
	NameTemplate    string            `yaml:"name_template,omitempty"`
	Replacements    map[string]string `yaml:",omitempty"`
	Files           []string          `yaml:",omitempty"`
}

// Release config used for the GitHub release
type Release struct {
	GitHub Repo `yaml:",omitempty"`
	Draft  bool `yaml:",omitempty"`
}

// FPM config
type FPM struct {
	Formats      []string `yaml:",omitempty"`
	Dependencies []string `yaml:",omitempty"`
	Conflicts    []string `yaml:",omitempty"`
	Vendor       string   `yaml:",omitempty"`
	Homepage     string   `yaml:",omitempty"`
	Maintainer   string   `yaml:",omitempty"`
	Description  string   `yaml:",omitempty"`
	License      string   `yaml:",omitempty"`
}

// Snapshot config
type Snapshot struct {
	NameTemplate string `yaml:"name_template,omitempty"`
}

// Project includes all project configuration
type Project struct {
	Release  Release  `yaml:",omitempty"`
	Brew     Homebrew `yaml:",omitempty"`
	Build    Build    `yaml:",omitempty"`
	Archive  Archive  `yaml:",omitempty"`
	FPM      FPM      `yaml:",omitempty"`
	Snapshot Snapshot `yaml:",omitempty"`

	// test only property indicating the path to the dist folder
	Dist string `yaml:"-"`
}

// Load config file
func Load(file string) (config Project, err error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return config, err
	}
	err = yaml.Unmarshal(data, &config)
	return
}

// Load config via io.Reader
func LoadReader(fd io.Reader) (config Project, err error) {
	data, err := ioutil.ReadAll(fd)
	if err != nil {
                return config, err
        }
        err = yaml.Unmarshal(data, &config)
        return
}
<|endoftext|>


<commit_before>package v13

import (
	"github.com/amsokol/go-ignite-client/http/types"
	"github.com/amsokol/go-ignite-client/http/v1/client"
	"github.com/amsokol/go-ignite-client/http/v1/v10"
)

// Version command shows current Ignite version.
// See https://apacheignite.readme.io/v1.3/docs/rest-api#section-version for more details
func Version(c client.Client) (types.Version, types.SessionToken, error) {
	return v10.Version(c)
}
<commit_msg>removed useless file<commit_after><|endoftext|>


<commit_before>package httpread

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/bpicode/fritzctl/logger"
	"github.com/pkg/errors"
)

var (
	httpStatusBuzzwords = map[string]int{"500 Internal Server Error": 500}
)

// HTTPStatusCodeError represents an 4xx client or a 5xx server error.
type HTTPStatusCodeError struct {
	error
}

func statusCodeError(code int, phrase string) *HTTPStatusCodeError {
	return &HTTPStatusCodeError{error: fmt.Errorf("HTTP status code error (%d): remote replied with '%s'", code, phrase)}
}

// ReadFullyString reads a http response into a string.
// The response is checked for its status code and the http.Response.Body is closed.
func ReadFullyString(f func() (*http.Response, error)) (string, error) {
	response, err := f()
	if err != nil {
		return "", err
	}
	defer response.Body.Close()
	bytesRead, err := ioutil.ReadAll(response.Body)
	body := string(bytesRead)
	logger.Debug("DATA:", body)
	statusCode, statusPhrase := guessStatusCode(response.StatusCode, response.Status, body)
	if statusCode >= 400 {
		return body, statusCodeError(statusCode, statusPhrase)
	}
	return body, err
}

func guessStatusCode(claimedCode int, claimedPhrase, body string) (int, string) {
	if claimedCode >= 400 {
		return claimedCode, claimedPhrase // This is already bad enough.
	}
	// There are web servers that send the wrong status code, but provide some hint in the text/html.
	for k, v := range httpStatusBuzzwords {
		if strings.Contains(strings.ToLower(body), strings.ToLower(k)) {
			return v, k
		}
	}
	return claimedCode, claimedPhrase
}

type decoder interface {
	Decode(v interface{}) error
}

type decoderFactory func(io.Reader) decoder

// ReadFullyXML reads a http response into a data container using an XML decoder.
// The response is checked for its status code and the http.Response.Body is closed.
func ReadFullyXML(f func() (*http.Response, error), v interface{}) error {
	return readDecode(f, func(r io.Reader) decoder {
		return xml.NewDecoder(r)
	}, v)
}

// ReadFullyJSON reads a http response into a data container using a json decoder.
// The response is checked for its status code and the http.Response.Body is closed.
func ReadFullyJSON(f func() (*http.Response, error), v interface{}) error {
	return readDecode(f, func(r io.Reader) decoder {
		return json.NewDecoder(r)
	}, v)
}

func readDecode(f func() (*http.Response, error), df decoderFactory, v interface{}) error {
	response, err := f()
	if err != nil {
		return errors.Wrap(err, "error obtaining HTTP response from remote")
	}
	defer response.Body.Close()
	if response.StatusCode >= 400 {
		return statusCodeError(response.StatusCode, response.Status)
	}
	return decode(response.Body, df, v)
}

func decode(r io.Reader, df decoderFactory, v interface{}) error {
	buf := new(bytes.Buffer)
	tee := io.TeeReader(r, buf)
	defer func() { logger.Debug("DATA:", buf) }()
	err := df(tee).Decode(v)
	if err != nil {
		return errors.Wrapf(err, "unable to decode remote response")
	}
	return nil
}
<commit_msg>remove HTTPStatusCodeError; align string reading with json/xml ansatz<commit_after>package httpread

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/bpicode/fritzctl/logger"
	"github.com/pkg/errors"
)

var (
	httpStatusBuzzwords = map[string]int{"500 Internal Server Error": 500}
)

type stringDecoder struct {
	reader io.Reader
}

func (s *stringDecoder) Decode(v interface{}) error {
	bytesRead, err := ioutil.ReadAll(s.reader)
	sp := v.(*string)
	*sp = string(bytesRead)
	return err
}

// ReadFullyString reads a http response into a string.
// The response is checked for its status code and the http.Response.Body is closed.
func ReadFullyString(f func() (*http.Response, error)) (string, error) {
	body := ""
	err := readDecode(f, func(r io.Reader) decoder {
		return &stringDecoder{reader: r}
	}, &body)
	if err != nil {
		return "", err
	}
	sc, sp := guessStatusCode(body)
	if sc >= 400 {
		return "", fmt.Errorf("HTTP status code error (%d, guessed): remote replied with '%s'", sc, sp)
	}
	return body, nil
}

func guessStatusCode(body string) (int, string) {
	// There are web servers that send the wrong status code, but provide some hint in the text/html.
	for k, v := range httpStatusBuzzwords {
		if strings.Contains(strings.ToLower(body), strings.ToLower(k)) {
			return v, k
		}
	}
	return 0, ""
}

type decoder interface {
	Decode(v interface{}) error
}

type decoderFactory func(io.Reader) decoder

// ReadFullyXML reads a http response into a data container using an XML decoder.
// The response is checked for its status code and the http.Response.Body is closed.
func ReadFullyXML(f func() (*http.Response, error), v interface{}) error {
	return readDecode(f, func(r io.Reader) decoder {
		return xml.NewDecoder(r)
	}, v)
}

// ReadFullyJSON reads a http response into a data container using a json decoder.
// The response is checked for its status code and the http.Response.Body is closed.
func ReadFullyJSON(f func() (*http.Response, error), v interface{}) error {
	return readDecode(f, func(r io.Reader) decoder {
		return json.NewDecoder(r)
	}, v)
}

func readDecode(f func() (*http.Response, error), df decoderFactory, v interface{}) error {
	response, err := f()
	if err != nil {
		return errors.Wrap(err, "error obtaining HTTP response from remote")
	}
	defer response.Body.Close()
	if response.StatusCode >= 400 {
		return fmt.Errorf("HTTP status code error (%d): remote replied with '%s'", response.StatusCode, response.Status)
	}
	return decode(response.Body, df, v)
}

func decode(r io.Reader, df decoderFactory, v interface{}) error {
	buf := new(bytes.Buffer)
	tee := io.TeeReader(r, buf)
	defer func() { logger.Debug("DATA:", buf) }()
	err := df(tee).Decode(v)
	if err != nil {
		return errors.Wrapf(err, "unable to decode remote response")
	}
	return nil
}
<|endoftext|>


<commit_before>package generatethumbnail

import (
	"image"
	"image/jpeg"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/kevintavog/findaphoto/common"

	"github.com/ian-kent/go-log/log"
	"github.com/nfnt/resize"
	"github.com/twinj/uuid"
)

var GeneratedImage int64
var FailedImage int64
var GeneratedVideo int64
var FailedVideo int64

type ThumbnailInfo struct {
	FullPath    string
	AliasedPath string
	MimeType    string
}

const thumbnailMaxHeightDimension = 170

var queue chan *ThumbnailInfo
var waitGroup sync.WaitGroup

func Start() {
	numConsumers := common.RatioNumCpus(4)
	queue = make(chan *ThumbnailInfo, 10000)
	waitGroup.Add(numConsumers)

	log.Info("Thumbnail generate using %d consumers", numConsumers)
	for idx := 0; idx < numConsumers; idx++ {
		go func() {
			dequeue()
			waitGroup.Done()
		}()
	}
}

func Done() {
	close(queue)
}

func Wait() {
	waitGroup.Wait()
}

func Enqueue(fullPath, aliasedPath, mimeType string) {
	thumbnailInfo := &ThumbnailInfo{
		FullPath:    fullPath,
		AliasedPath: aliasedPath,
		MimeType:    mimeType,
	}
	queue <- thumbnailInfo
}

func dequeue() {
	var mediaType []string
	var thumbPath string
	var err error
	var lastHeapSys uint64

	for thumbnailInfo := range queue {
		mediaType = strings.Split(thumbnailInfo.MimeType, "/")
		thumbPath = common.ToThumbPath(thumbnailInfo.AliasedPath)
		if len(mediaType) < 1 {
			log.Error("Invalid media type: '%s' for %s", thumbnailInfo.MimeType, thumbnailInfo.FullPath)
			continue
		}

		err = common.CreateDirectory(path.Dir(thumbPath))
		if err != nil {
			log.Error("Unable to create directory for '%s'", thumbPath)
			continue
		}

		switch strings.ToLower(mediaType[0]) {
		case "video":
			generateVideo(thumbnailInfo.FullPath, thumbPath)
		case "image":
			generateImage(thumbnailInfo.FullPath, thumbPath)
		default:
			log.Error("Unhandled mediaType: %s (%s) for %s", thumbnailInfo.MimeType, mediaType, thumbnailInfo.FullPath)
		}

		var memStats runtime.MemStats
		runtime.ReadMemStats(&memStats)
		if lastHeapSys != (memStats.HeapSys / 1024) {
			lastHeapSys = memStats.HeapSys / 1024
			log.Info("%d from OS; %d in use; %d idle; %d num objects; %d released (%s)",
				memStats.HeapSys/1024,
				memStats.HeapInuse/1024,
				memStats.HeapIdle/1024,
				memStats.HeapObjects,
				memStats.HeapReleased/1024,
				thumbPath)
		}
	}
}

func generateImage(fullPath, thumbPath string) {
	if createThumbnail(fullPath, thumbPath) != nil {
		atomic.AddInt64(&FailedImage, 1)
	} else {
		atomic.AddInt64(&GeneratedImage, 1)
	}
}

func generateVideo(fullPath, thumbPath string) {
	tmpFilename := path.Join(os.TempDir(), "findAPhoto", "thumbnails", uuid.NewV4().String()+".JPG")
	defer os.Remove(tmpFilename)

	err := common.CreateDirectory(path.Dir(tmpFilename))
	if err != nil {
		log.Fatal("Unable to create temporary directory for thumbnail generation (%s): %s", tmpFilename, err.Error())
	}

	out, err := exec.Command(common.FfmpegPath, "-i", fullPath, "-ss", "00:00:01.0", "-vframes", "1", tmpFilename).Output()
	if err != nil {
		atomic.AddInt64(&FailedVideo, 1)
		log.Fatal("Failed executing ffmpeg for '%s': %s (%s)", fullPath, err.Error(), out)
	}

	if exists, _ := common.PathExists(tmpFilename); !exists {
		// The video may not be long enough to grab a frame at the 1 second point...
		out, err = exec.Command(common.FfmpegPath, "-i", fullPath, "-ss", "00:00:00.0", "-vframes", "1", tmpFilename).Output()
		if err != nil {
			atomic.AddInt64(&FailedVideo, 1)
			log.Error("Failed executing ffmpeg for '%s': %s (%s)", fullPath, err.Error(), out)
		}
	}

	//	if resizeWithImaging(tmpFilename, thumbPath) != nil {
	if createThumbnail(tmpFilename, thumbPath) != nil {
		atomic.AddInt64(&FailedVideo, 1)
	} else {
		atomic.AddInt64(&GeneratedVideo, 1)
	}
}

func createThumbnail(imageFilename, thumbFilename string) error {
	file, err := os.Open(imageFilename)
	if err != nil {
		return err
	}
	defer file.Close()

	image, _, err := image.Decode(file)
	if err != nil {
		return err
	}

	thumb := resize.Resize(0, thumbnailMaxHeightDimension, image, resize.NearestNeighbor)
	savedThumbnailFile, err := os.Create(thumbFilename)
	if err != nil {
		return err
	}
	defer savedThumbnailFile.Close()

	jpeg.Encode(savedThumbnailFile, thumb, &jpeg.Options{Quality: 85})
	return nil
}
<commit_msg>Fix a typo<commit_after>package generatethumbnail

import (
	"image"
	"image/jpeg"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/kevintavog/findaphoto/common"

	"github.com/ian-kent/go-log/log"
	"github.com/nfnt/resize"
	"github.com/twinj/uuid"
)

var GeneratedImage int64
var FailedImage int64
var GeneratedVideo int64
var FailedVideo int64

type ThumbnailInfo struct {
	FullPath    string
	AliasedPath string
	MimeType    string
}

const thumbnailMaxHeightDimension = 170

var queue chan *ThumbnailInfo
var waitGroup sync.WaitGroup

func Start() {
	numConsumers := common.RatioNumCpus(4)
	queue = make(chan *ThumbnailInfo, 10000)
	waitGroup.Add(numConsumers)

	log.Info("Thumbnail generation using %d consumers", numConsumers)
	for idx := 0; idx < numConsumers; idx++ {
		go func() {
			dequeue()
			waitGroup.Done()
		}()
	}
}

func Done() {
	close(queue)
}

func Wait() {
	waitGroup.Wait()
}

func Enqueue(fullPath, aliasedPath, mimeType string) {
	thumbnailInfo := &ThumbnailInfo{
		FullPath:    fullPath,
		AliasedPath: aliasedPath,
		MimeType:    mimeType,
	}
	queue <- thumbnailInfo
}

func dequeue() {
	var mediaType []string
	var thumbPath string
	var err error
	var lastHeapSys uint64

	for thumbnailInfo := range queue {
		mediaType = strings.Split(thumbnailInfo.MimeType, "/")
		thumbPath = common.ToThumbPath(thumbnailInfo.AliasedPath)
		if len(mediaType) < 1 {
			log.Error("Invalid media type: '%s' for %s", thumbnailInfo.MimeType, thumbnailInfo.FullPath)
			continue
		}

		err = common.CreateDirectory(path.Dir(thumbPath))
		if err != nil {
			log.Error("Unable to create directory for '%s'", thumbPath)
			continue
		}

		switch strings.ToLower(mediaType[0]) {
		case "video":
			generateVideo(thumbnailInfo.FullPath, thumbPath)
		case "image":
			generateImage(thumbnailInfo.FullPath, thumbPath)
		default:
			log.Error("Unhandled mediaType: %s (%s) for %s", thumbnailInfo.MimeType, mediaType, thumbnailInfo.FullPath)
		}

		var memStats runtime.MemStats
		runtime.ReadMemStats(&memStats)
		if lastHeapSys != (memStats.HeapSys / 1024) {
			lastHeapSys = memStats.HeapSys / 1024
			log.Info("Heap info: %d from OS; %d in use; %d idle; %d objects (%s)",
				memStats.HeapSys/1024,
				memStats.HeapInuse/1024,
				memStats.HeapIdle/1024,
				memStats.HeapObjects,
				thumbPath)
		}
	}
}

func generateImage(fullPath, thumbPath string) {
	if createThumbnail(fullPath, thumbPath) != nil {
		atomic.AddInt64(&FailedImage, 1)
	} else {
		atomic.AddInt64(&GeneratedImage, 1)
	}
}

func generateVideo(fullPath, thumbPath string) {
	tmpFilename := path.Join(os.TempDir(), "findAPhoto", "thumbnails", uuid.NewV4().String()+".JPG")
	defer os.Remove(tmpFilename)

	err := common.CreateDirectory(path.Dir(tmpFilename))
	if err != nil {
		log.Fatal("Unable to create temporary directory for thumbnail generation (%s): %s", tmpFilename, err.Error())
	}

	out, err := exec.Command(common.FfmpegPath, "-i", fullPath, "-ss", "00:00:01.0", "-vframes", "1", tmpFilename).Output()
	if err != nil {
		atomic.AddInt64(&FailedVideo, 1)
		log.Fatal("Failed executing ffmpeg for '%s': %s (%s)", fullPath, err.Error(), out)
	}

	if exists, _ := common.PathExists(tmpFilename); !exists {
		// The video may not be long enough to grab a frame at the 1 second point...
		out, err = exec.Command(common.FfmpegPath, "-i", fullPath, "-ss", "00:00:00.0", "-vframes", "1", tmpFilename).Output()
		if err != nil {
			atomic.AddInt64(&FailedVideo, 1)
			log.Error("Failed executing ffmpeg for '%s': %s (%s)", fullPath, err.Error(), out)
		}
	}

	//	if resizeWithImaging(tmpFilename, thumbPath) != nil {
	if createThumbnail(tmpFilename, thumbPath) != nil {
		atomic.AddInt64(&FailedVideo, 1)
	} else {
		atomic.AddInt64(&GeneratedVideo, 1)
	}
}

func createThumbnail(imageFilename, thumbFilename string) error {
	file, err := os.Open(imageFilename)
	if err != nil {
		return err
	}
	defer file.Close()

	image, _, err := image.Decode(file)
	if err != nil {
		return err
	}

	thumb := resize.Resize(0, thumbnailMaxHeightDimension, image, resize.NearestNeighbor)
	savedThumbnailFile, err := os.Create(thumbFilename)
	if err != nil {
		return err
	}
	defer savedThumbnailFile.Close()

	jpeg.Encode(savedThumbnailFile, thumb, &jpeg.Options{Quality: 85})
	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	cmds "github.com/ipfs/go-ipfs/commands"
	"strings"
	"testing"
)

func checkHelptextRecursive(t *testing.T, name []string, c *cmds.Command) {
	if c.Helptext.Tagline == "" {
		t.Errorf("%s has no tagline!", strings.Join(name, " "))
	}

	if c.Helptext.LongDescription == "" {
		t.Errorf("%s has no long description!", strings.Join(name, " "))
	}

	if c.Helptext.ShortDescription == "" {
		t.Errorf("%s has no short description!", strings.Join(name, " "))
	}

	if c.Helptext.Synopsis == "" {
		t.Errorf("%s has no synopsis!", strings.Join(name, " "))
	}

	for subname, sub := range c.Subcommands {
		checkHelptextRecursive(t, append(name, subname), sub)
	}
}

func TestHelptexts(t *testing.T) {
	checkHelptextRecursive(t, []string{"ipfs"}, Root)
}
<commit_msg>Add ProcessHelp call to Helptext test.<commit_after>package commands

import (
	cmds "github.com/ipfs/go-ipfs/commands"
	"strings"
	"testing"
)

func checkHelptextRecursive(t *testing.T, name []string, c *cmds.Command) {
	if c.Helptext.Tagline == "" {
		t.Errorf("%s has no tagline!", strings.Join(name, " "))
	}

	if c.Helptext.LongDescription == "" {
		t.Errorf("%s has no long description!", strings.Join(name, " "))
	}

	if c.Helptext.ShortDescription == "" {
		t.Errorf("%s has no short description!", strings.Join(name, " "))
	}

	if c.Helptext.Synopsis == "" {
		t.Errorf("%s has no synopsis!", strings.Join(name, " "))
	}

	for subname, sub := range c.Subcommands {
		checkHelptextRecursive(t, append(name, subname), sub)
	}
}

func TestHelptexts(t *testing.T) {
	Root.ProcessHelp()
	checkHelptextRecursive(t, []string{"ipfs"}, Root)
}
<|endoftext|>


<commit_before>// Copyright 2020 Georg Großberger <contact@grossberger-ge.org>
// This is free software; it is provided under the terms of the MIT License
// See the file LICENSE or <https://opensource.org/licenses/MIT> for details

package labels

import (
	"bytes"
	"encoding/xml"
	"io/ioutil"
	"path/filepath"
	"time"

	"github.com/garfieldius/t3ll/log"

	"github.com/jinzhu/copier"
	"github.com/kr/pretty"
)

// Xliff is the root of all languages and labels
type Xliff struct {
	SourceFile string
	Files      []*XliffRoot
}

// Labels converts a Xliff tree into a Labels tree
func (x *Xliff) Labels() *Labels {
	codes := make(map[string]bool)
	labels := &Labels{
		Type:      XMLXliffv1,
		SrcXlif:   x,
		File:      extPathOfFile(x.SourceFile),
		FromFile:  x.SourceFile,
		Languages: make([]string, 0),
		Data:      make([]*Label, 0),
	}

	for _, file := range x.Files {
		langCode := file.File.ToLang

		if langCode == "" {
			langCode = file.File.SrcLang
		}

		codes[langCode] = true

		for _, unit := range file.File.Body.Units {
			t := &Translation{
				Language: langCode,
				Content:  unit.To,
			}

			if langCode == "en" {
				t.Content = unit.Src
				labels.File = extPathOfFile(file.SourceFile)
			}

			found := false

			for _, label := range labels.Data {
				if label.ID == unit.ID {
					found = true
					label.Translations = append(label.Translations, t)
				}
			}

			if !found {
				labels.Data = append(labels.Data, &Label{
					ID:           unit.ID,
					Translations: []*Translation{t},
				})
			}
		}
	}

	for langCode := range codes {
		labels.Languages = append(labels.Languages, langCode)
	}

	log.Msg("Converted xlif into %# v", pretty.Formatter(labels))

	return labels
}

// XliffRoot is the virtual root node of a XML document
type XliffRoot struct {
	XMLName    string     `xml:"xliff"`
	Version    string     `xml:"version,attr,omitempty"`
	XMLNST3    string     `xml:"xmlns:t3,attr,omitempty"`
	XMLNS      string     `xml:"xmlns,attr,omitempty"`
	File       *XliffFile `xml:"file,omitempty"`
	Language   string     `xml:"-"`
	SourceFile string     `xml:"-"`
}

// Labels in this implementation has no function, because the
// calling party handles the conversion
func (x *XliffRoot) Labels() *Labels {
	return nil // Noop
}

// IndentChar determines the string for indentatio or
// XML tags
func (x *XliffRoot) IndentChar() string {
	return indentOfFile(x.SourceFile)
}

// XliffFile is the actual root node of a Xliff document
type XliffFile struct {
	SrcLang string       `xml:"source-language,attr,omitempty"`
	ToLang  string       `xml:"target-language,attr,omitempty"`
	Orig    string       `xml:"original,attr,omitempty"`
	Name    string       `xml:"product-name,attr,omitempty"`
	Date    string       `xml:"date,attr,omitempty"`
	Header  *XliffHeader `xml:"header,omitempty"`
	Body    *XliffBody   `xml:"body,omitempty"`
}

// XliffHeader is a collection of common metadata
type XliffHeader struct {
	Skl      string `xml:"skl,omitempty"`
	PhaseGrp string `xml:"phase-group,omitempty"`
	Glossary string `xml:"glossary,omitempty"`
	Ref      string `xml:"reference,omitempty"`
	CountGrp string `xml:"count-group,omitempty"`
	Tool     string `xml:"tool,omitempty"`
	PopGrp   string `xml:"pop-group,omitempty"`
	Note     string `xml:"note,omitempty"`

	// Non-standard, but common in TYPO3
	Type        string `xml:"type,omitempty"`
	Description string `xml:"description,omitempty"`
	AuthName    string `xml:"authorName,omitempty"`
	AuthMail    string `xml:"authorEmail,omitempty"`
	AuthCompany string `xml:"authorCompany,omitempty"`
	Generator   string `xml:"generator,omitempty"`
}

// XliffBody contains the units of a Xliff document
type XliffBody struct {
	Units []*XliffUnit `xml:"trans-unit,omitempty"`
}

// XliffUnit is a single ID + Content unit of a Xliff document
type XliffUnit struct {
	ID      string `xml:"id,attr,omitempty"`
	ResName string `xml:"resname,attr,omitempty"`
	Src     string `xml:"source,omitempty"`
	To      string `xml:"target,omitempty"`
}

// XliffConverter will convert a Xliff structure to struct Lables
type XliffConverter struct {
	src  *Labels
	file *XliffFile
	lang string
}

// XML will return the Xliff structure of a Labels structure
func (c *XliffConverter) XML() LangFile {
	b := &XliffBody{
		Units: make([]*XliffUnit, 0, len(c.src.Data)),
	}
	f := &XliffFile{
		SrcLang: "en",
		Date:    time.Now().Format(time.RFC3339),
		Body:    b,
	}

	if c.file != nil {
		f.Header = c.file.Header
		f.Orig = c.file.Orig
		f.Name = c.file.Name
	}

	l := "en"

	if c.lang != "en" {
		f.ToLang = c.lang
		l = c.lang
	}

	for _, label := range c.src.Data {
		for _, trans := range label.Translations {
			if trans.Language == l {
				u := &XliffUnit{
					ResName: label.ID,
					// ID is not the correct attribute for this, but is used by TYPO3
					ID: label.ID,
				}

				if c.lang == "en" {
					u.Src = trans.Content
				} else {
					if trans.Content == "" {
						continue
					}
					u.To = trans.Content
					for _, orig := range label.Translations {
						if orig.Language == "en" {
							u.Src = orig.Content
						}
					}
				}

				b.Units = append(b.Units, u)
			}
		}
	}

	if len(b.Units) < 1 {
		return nil
	}

	x := XliffRoot{
		SourceFile: c.src.FromFile,
		Language:   c.lang,
		Version:    "1.2",
		XMLNST3:    "http://typo3.org/schemas/xliff",
		XMLNS:      "urn:oasis:names:tc:xliff:document:1.2",
		File:       f,
	}

	cp := XliffRoot{}

	if err := copier.Copy(&cp, &x); err == nil {
		if oldFileData, err := ioutil.ReadFile(c.File()); err == nil {
			oldXlif := new(XliffRoot)
			if err = xml.Unmarshal(oldFileData, oldXlif); err == nil {
				if oldXlif.File != nil && oldXlif.File.Date != "" {
					cp.File.Date = oldXlif.File.Date
					oldXml, _ := xml.MarshalIndent(&oldXlif, "", "  ")
					newXml, _ := xml.MarshalIndent(&cp, "", "  ")
					if oldXml != nil && newXml != nil && bytes.Equal(oldXml, newXml) {
						return nil
					}
				}
			}
		}
	}

	return &x
}

// File determines the desired target file of this document
func (c *XliffConverter) File() string {
	d := filepath.Dir(c.src.FromFile)
	f := xliffLangPrefix.ReplaceAllString(filepath.Base(c.src.FromFile), "")
	p := ""

	if c.lang != "en" {
		p = c.lang + "."
	}

	full := d + string(filepath.Separator) + p + f
	log.Msg("Nomalized from %s to %s", c.src.FromFile, full)

	return full
}
<commit_msg>Add "datatype" property to xliff root<commit_after>// Copyright 2020 Georg Großberger <contact@grossberger-ge.org>
// This is free software; it is provided under the terms of the MIT License
// See the file LICENSE or <https://opensource.org/licenses/MIT> for details

package labels

import (
	"bytes"
	"encoding/xml"
	"io/ioutil"
	"path/filepath"
	"time"

	"github.com/garfieldius/t3ll/log"

	"github.com/jinzhu/copier"
	"github.com/kr/pretty"
)

// Xliff is the root of all languages and labels
type Xliff struct {
	SourceFile string
	Files      []*XliffRoot
}

// Labels converts a Xliff tree into a Labels tree
func (x *Xliff) Labels() *Labels {
	codes := make(map[string]bool)
	labels := &Labels{
		Type:      XMLXliffv1,
		SrcXlif:   x,
		File:      extPathOfFile(x.SourceFile),
		FromFile:  x.SourceFile,
		Languages: make([]string, 0),
		Data:      make([]*Label, 0),
	}

	for _, file := range x.Files {
		langCode := file.File.ToLang

		if langCode == "" {
			langCode = file.File.SrcLang
		}

		codes[langCode] = true

		for _, unit := range file.File.Body.Units {
			t := &Translation{
				Language: langCode,
				Content:  unit.To,
			}

			if langCode == "en" {
				t.Content = unit.Src
				labels.File = extPathOfFile(file.SourceFile)
			}

			found := false

			for _, label := range labels.Data {
				if label.ID == unit.ID {
					found = true
					label.Translations = append(label.Translations, t)
				}
			}

			if !found {
				labels.Data = append(labels.Data, &Label{
					ID:           unit.ID,
					Translations: []*Translation{t},
				})
			}
		}
	}

	for langCode := range codes {
		labels.Languages = append(labels.Languages, langCode)
	}

	log.Msg("Converted xlif into %# v", pretty.Formatter(labels))

	return labels
}

// XliffRoot is the virtual root node of a XML document
type XliffRoot struct {
	XMLName    string     `xml:"xliff"`
	Version    string     `xml:"version,attr,omitempty"`
	XMLNST3    string     `xml:"xmlns:t3,attr,omitempty"`
	XMLNS      string     `xml:"xmlns,attr,omitempty"`
	File       *XliffFile `xml:"file,omitempty"`
	Language   string     `xml:"-"`
	SourceFile string     `xml:"-"`
}

// Labels in this implementation has no function, because the
// calling party handles the conversion
func (x *XliffRoot) Labels() *Labels {
	return nil // Noop
}

// IndentChar determines the string for indentatio or
// XML tags
func (x *XliffRoot) IndentChar() string {
	return indentOfFile(x.SourceFile)
}

// XliffFile is the actual root node of a Xliff document
type XliffFile struct {
	SrcLang  string       `xml:"source-language,attr"`
	ToLang   string       `xml:"target-language,attr,omitempty"`
	Orig     string       `xml:"original,attr"`
	Datatype string       `xml:"datatype,attr"`
	Name     string       `xml:"product-name,attr,omitempty"`
	Date     string       `xml:"date,attr,omitempty"`
	Header   *XliffHeader `xml:"header,omitempty"`
	Body     *XliffBody   `xml:"body,omitempty"`
}

// XliffHeader is a collection of common metadata
type XliffHeader struct {
	Skl      string `xml:"skl,omitempty"`
	PhaseGrp string `xml:"phase-group,omitempty"`
	Glossary string `xml:"glossary,omitempty"`
	Ref      string `xml:"reference,omitempty"`
	CountGrp string `xml:"count-group,omitempty"`
	Tool     string `xml:"tool,omitempty"`
	PopGrp   string `xml:"pop-group,omitempty"`
	Note     string `xml:"note,omitempty"`

	// Non-standard, but common in TYPO3
	Type        string `xml:"type,omitempty"`
	Description string `xml:"description,omitempty"`
	AuthName    string `xml:"authorName,omitempty"`
	AuthMail    string `xml:"authorEmail,omitempty"`
	AuthCompany string `xml:"authorCompany,omitempty"`
	Generator   string `xml:"generator,omitempty"`
}

// XliffBody contains the units of a Xliff document
type XliffBody struct {
	Units []*XliffUnit `xml:"trans-unit,omitempty"`
}

// XliffUnit is a single ID + Content unit of a Xliff document
type XliffUnit struct {
	ID      string `xml:"id,attr,omitempty"`
	ResName string `xml:"resname,attr,omitempty"`
	Src     string `xml:"source,omitempty"`
	To      string `xml:"target,omitempty"`
}

// XliffConverter will convert a Xliff structure to struct Lables
type XliffConverter struct {
	src  *Labels
	file *XliffFile
	lang string
}

// XML will return the Xliff structure of a Labels structure
func (c *XliffConverter) XML() LangFile {
	b := &XliffBody{
		Units: make([]*XliffUnit, 0, len(c.src.Data)),
	}
	f := &XliffFile{
		SrcLang:  "en",
		Date:     time.Now().Format(time.RFC3339),
		Body:     b,
		Datatype: "plaintext",
	}

	if c.file != nil {
		if c.file.Datatype != "" {
			f.Datatype = c.file.Datatype
		}
		if c.file.Orig != "" {
			f.Orig = c.file.Orig
		}
		f.Header = c.file.Header
		f.Name = c.file.Name
	}

	l := "en"

	if c.lang != "en" {
		f.ToLang = c.lang
		l = c.lang
	}

	for _, label := range c.src.Data {
		for _, trans := range label.Translations {
			if trans.Language == l {
				u := &XliffUnit{
					ResName: label.ID,
					// ID is not the correct attribute for this, but is used by TYPO3
					ID: label.ID,
				}

				if c.lang == "en" {
					u.Src = trans.Content
				} else {
					if trans.Content == "" {
						continue
					}
					u.To = trans.Content
					for _, orig := range label.Translations {
						if orig.Language == "en" {
							u.Src = orig.Content
						}
					}
				}

				b.Units = append(b.Units, u)
			}
		}
	}

	if len(b.Units) < 1 {
		return nil
	}

	x := XliffRoot{
		SourceFile: c.src.FromFile,
		Language:   c.lang,
		Version:    "1.2",
		XMLNST3:    "http://typo3.org/schemas/xliff",
		XMLNS:      "urn:oasis:names:tc:xliff:document:1.2",
		File:       f,
	}

	cp := XliffRoot{}

	if err := copier.Copy(&cp, &x); err == nil {
		if oldFileData, err := ioutil.ReadFile(c.File()); err == nil {
			oldXlif := new(XliffRoot)
			if err = xml.Unmarshal(oldFileData, oldXlif); err == nil {
				if oldXlif.File != nil && oldXlif.File.Date != "" {
					cp.File.Date = oldXlif.File.Date
					oldXml, _ := xml.MarshalIndent(&oldXlif, "", "  ")
					newXml, _ := xml.MarshalIndent(&cp, "", "  ")
					if oldXml != nil && newXml != nil && bytes.Equal(oldXml, newXml) {
						return nil
					}
				}
			}
		}
	}

	return &x
}

// File determines the desired target file of this document
func (c *XliffConverter) File() string {
	d := filepath.Dir(c.src.FromFile)
	f := xliffLangPrefix.ReplaceAllString(filepath.Base(c.src.FromFile), "")
	p := ""

	if c.lang != "en" {
		p = c.lang + "."
	}

	full := d + string(filepath.Separator) + p + f
	log.Msg("Nomalized from %s to %s", c.src.FromFile, full)

	return full
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"

	"os"

	"github.com/labstack/echo"
)

type ThumbnailRequest struct {
	Hash string `json:"hash" xml:"hash" form:"hash" query:"hash"`
}

func ThumbnailHandler(c echo.Context) error {
	hash := c.Param("hash")
	fmt.Printf("hash=%s\n", hash)

	thumImagePath := CreateThumFilePathFromHash(hash)
	_, err := os.Stat(thumImagePath)
	if err != nil {
		return c.File("assets/noimage.jpg")
	}
	return c.File(thumImagePath)
}

func ThumbnailBase64Handler(c echo.Context) error {
	hash := c.Param("hash")
	fmt.Printf("hash=%s\n", hash)

	thumImagePath := CreateThumFilePathFromHash(hash)
	_, err := os.Stat(thumImagePath)
	if err != nil {
		thumImagePath = "assets/noimage.jpg"
	}

	data, err := ioutil.ReadFile(thumImagePath)
	if err != nil {
		return err
	}
	imageBase64 := base64.StdEncoding.EncodeToString(data)
	return c.String(http.StatusOK, imageBase64)
}
<commit_msg>サムネイル画像がない時はエラー画像ではなく空文字を返すように変更<commit_after>package main

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"

	"os"

	"github.com/labstack/echo"
)

type ThumbnailRequest struct {
	Hash string `json:"hash" xml:"hash" form:"hash" query:"hash"`
}

func ThumbnailHandler(c echo.Context) error {
	hash := c.Param("hash")
	fmt.Printf("hash=%s\n", hash)

	thumImagePath := CreateThumFilePathFromHash(hash)
	_, err := os.Stat(thumImagePath)
	if err != nil {
		return c.File("assets/noimage.jpg")
	}
	return c.File(thumImagePath)
}

func ThumbnailBase64Handler(c echo.Context) error {
	hash := c.Param("hash")
	fmt.Printf("hash=%s\n", hash)

	thumImagePath := CreateThumFilePathFromHash(hash)
	_, err := os.Stat(thumImagePath)
	if err != nil {
		//画像なしを返却する
		return c.String(http.StatusOK, "")
	}

	data, err := ioutil.ReadFile(thumImagePath)
	if err != nil {
		return err
	}
	imageBase64 := base64.StdEncoding.EncodeToString(data)
	return c.String(http.StatusOK, imageBase64)
}
<|endoftext|>