<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package l2plugin

import (
	"context"
	"sync"
	"time"

	govppapi "git.fd.io/govpp.git/api"
	"github.com/ligato/cn-infra/core"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/vpp-agent/plugins/govppmux"
	l2_api "github.com/ligato/vpp-agent/plugins/vpp/binapi/l2"
	"github.com/ligato/vpp-agent/plugins/vpp/ifplugin/ifaceidx"
	"github.com/ligato/vpp-agent/plugins/vpp/l2plugin/l2idx"
	"github.com/ligato/vpp-agent/plugins/vpp/model/l2"
)

// BridgeDomainStateUpdater holds all data required to handle bridge domain state.
type BridgeDomainStateUpdater struct {
	log    logging.Logger
	mx     sync.Mutex
	cancel context.CancelFunc
	wg     sync.WaitGroup

	// In-memory mappings
	ifIndexes ifaceidx.SwIfIndex
	bdIndexes l2idx.BDIndex

	// State publisher
	publishBdState func(notification *BridgeDomainStateNotification)
	bdState        map[uint32]*l2.BridgeDomainState_BridgeDomain

	// VPP channel
	vppCh *govppapi.Channel

	// Notification subscriptions
	vppNotifSubs            *govppapi.NotifSubscription
	vppCountersSubs         *govppapi.NotifSubscription
	vppCombinedCountersSubs *govppapi.NotifSubscription
	notificationChan        chan BridgeDomainStateMessage
	bdIdxChan               chan l2idx.BdChangeDto
}

// BridgeDomainStateNotification contains bridge domain state object with all data published to ETCD.
type BridgeDomainStateNotification struct {
	State *l2.BridgeDomainState_BridgeDomain
}

// Init bridge domain state updater.
func (plugin *BridgeDomainStateUpdater) Init(logger logging.PluginLogger, goVppMux govppmux.API, ctx context.Context, bdIndexes l2idx.BDIndex, swIfIndexes ifaceidx.SwIfIndex,
	notificationChan chan BridgeDomainStateMessage, publishBdState func(notification *BridgeDomainStateNotification)) (err error) {
	// Logger
	plugin.log = logger.NewLogger("-l2-bd-state")
	plugin.log.Info("Initializing BridgeDomainStateUpdater")

	// Mappings
	plugin.bdIndexes = bdIndexes
	plugin.ifIndexes = swIfIndexes

	// State publisher
	plugin.notificationChan = notificationChan
	plugin.publishBdState = publishBdState
	plugin.bdState = make(map[uint32]*l2.BridgeDomainState_BridgeDomain)

	// VPP channel
	plugin.vppCh, err = goVppMux.NewAPIChannel()
	if err != nil {
		return err
	}

	// Name-to-index watcher
	plugin.bdIdxChan = make(chan l2idx.BdChangeDto, 100)
	bdIndexes.WatchNameToIdx(core.PluginName("bdplugin_bdstate"), plugin.bdIdxChan)

	var childCtx context.Context
	childCtx, plugin.cancel = context.WithCancel(ctx)

	// Bridge domain notification watcher
	go plugin.watchVPPNotifications(childCtx)

	return nil
}

// watchVPPNotifications watches for delivery of notifications from VPP.
func (plugin *BridgeDomainStateUpdater) watchVPPNotifications(ctx context.Context) {
	plugin.wg.Add(1)
	defer plugin.wg.Done()

	if plugin.notificationChan != nil {
		plugin.log.Info("watchVPPNotifications for bridge domain state started")
	} else {
		plugin.log.Error("failed to start watchVPPNotifications for bridge domain state")
		return
	}

	for {
		select {
		case notif := <-plugin.notificationChan:
			msg := notif.Message
			bdName := notif.Name
			switch msg := msg.(type) {
			case *l2_api.BridgeDomainDetails:
				bdState := plugin.processBridgeDomainDetailsNotification(msg, bdName)
				if bdState != nil {
					plugin.publishBdState(&BridgeDomainStateNotification{
						State: bdState,
					})
				}
			default:
				plugin.log.WithFields(logging.Fields{"MessageName": msg.GetMessageName()}).Debug("L2Plugin: Ignoring unknown VPP notification")
			}
		case bdIdxDto := <-plugin.bdIdxChan:
			bdIdxDto.Done()

		case <-ctx.Done():
			// Stop watching for notifications.
			return
		}
	}
}

func (plugin *BridgeDomainStateUpdater) processBridgeDomainDetailsNotification(msg *l2_api.BridgeDomainDetails, name string) *l2.BridgeDomainState_BridgeDomain {
	bdState := &l2.BridgeDomainState_BridgeDomain{}
	// Delete case.
	if msg.BdID == 0 {
		if name == "" {
			plugin.log.Debugf("invalid bridge domain received: %+v", msg)
			return bdState
		}
		// Mark index to 0 to be removed, but pass name so that the key can be constructed.
		bdState.Index = 0
		bdState.InternalName = name
		return bdState
	}
	bdState.Index = msg.BdID
	name, _, found := plugin.bdIndexes.LookupName(msg.BdID)
	if !found {
		plugin.log.Warnf("bridge domain index not found, index %v is not in the mapping", msg.BdID)
		return bdState
	}
	bdState.InternalName = name
	bdState.InterfaceCount = msg.NSwIfs
	name, _, found = plugin.ifIndexes.LookupName(msg.BviSwIfIndex)
	if found {
		bdState.BviInterface = name
		bdState.BviInterfaceIndex = msg.BviSwIfIndex
	} else {
		bdState.BviInterface = "not_set"
	}
	bdState.L2Params = getBridgeDomainStateParams(msg)
	bdState.Interfaces = plugin.getBridgeDomainInterfaces(msg)
	bdState.LastChange = time.Now().Unix()

	return bdState
}

func (plugin *BridgeDomainStateUpdater) getBridgeDomainInterfaces(msg *l2_api.BridgeDomainDetails) []*l2.BridgeDomainState_BridgeDomain_Interfaces {
	var bdStateInterfaces []*l2.BridgeDomainState_BridgeDomain_Interfaces
	for _, swIfaceDetails := range msg.SwIfDetails {
		bdIfaceState := &l2.BridgeDomainState_BridgeDomain_Interfaces{}
		name, _, found := plugin.ifIndexes.LookupName(swIfaceDetails.SwIfIndex)
		if !found {
			plugin.log.Debugf("Interface name with index %v not found for bridge domain status", swIfaceDetails.SwIfIndex)
			bdIfaceState.Name = "unknown"
		} else {
			bdIfaceState.Name = name
		}
		bdIfaceState.SwIfIndex = swIfaceDetails.SwIfIndex
		bdIfaceState.SplitHorizonGroup = uint32(swIfaceDetails.Shg)
		bdStateInterfaces = append(bdStateInterfaces, bdIfaceState)
	}
	return bdStateInterfaces
}

func getBridgeDomainStateParams(msg *l2_api.BridgeDomainDetails) *l2.BridgeDomainState_BridgeDomain_L2Params {
	params := &l2.BridgeDomainState_BridgeDomain_L2Params{}
	params.Flood = intToBool(msg.Flood)
	params.UnknownUnicastFlood = intToBool(msg.UuFlood)
	params.Forward = intToBool(msg.Forward)
	params.Learn = intToBool(msg.Learn)
	params.ArpTermination = intToBool(msg.ArpTerm)
	params.MacAge = uint32(msg.MacAge)
	return params
}

func intToBool(num uint8) bool {
	if num == 1 {
		return true
	}
	return false
}
<commit_msg>Fix nil pointer access and check if channel has closed<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package l2plugin

import (
	"context"
	"sync"
	"time"

	govppapi "git.fd.io/govpp.git/api"
	"github.com/ligato/cn-infra/core"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/vpp-agent/plugins/govppmux"
	l2_api "github.com/ligato/vpp-agent/plugins/vpp/binapi/l2"
	"github.com/ligato/vpp-agent/plugins/vpp/ifplugin/ifaceidx"
	"github.com/ligato/vpp-agent/plugins/vpp/l2plugin/l2idx"
	"github.com/ligato/vpp-agent/plugins/vpp/model/l2"
)

// BridgeDomainStateUpdater holds all data required to handle bridge domain state.
type BridgeDomainStateUpdater struct {
	log    logging.Logger
	mx     sync.Mutex
	cancel context.CancelFunc
	wg     sync.WaitGroup

	// In-memory mappings
	ifIndexes ifaceidx.SwIfIndex
	bdIndexes l2idx.BDIndex

	// State publisher
	publishBdState func(notification *BridgeDomainStateNotification)
	bdState        map[uint32]*l2.BridgeDomainState_BridgeDomain

	// VPP channel
	vppCh *govppapi.Channel

	// Notification subscriptions
	vppNotifSubs            *govppapi.NotifSubscription
	vppCountersSubs         *govppapi.NotifSubscription
	vppCombinedCountersSubs *govppapi.NotifSubscription
	notificationChan        chan BridgeDomainStateMessage
	bdIdxChan               chan l2idx.BdChangeDto
}

// BridgeDomainStateNotification contains bridge domain state object with all data published to ETCD.
type BridgeDomainStateNotification struct {
	State *l2.BridgeDomainState_BridgeDomain
}

// Init bridge domain state updater.
func (plugin *BridgeDomainStateUpdater) Init(logger logging.PluginLogger, goVppMux govppmux.API, ctx context.Context, bdIndexes l2idx.BDIndex, swIfIndexes ifaceidx.SwIfIndex,
	notificationChan chan BridgeDomainStateMessage, publishBdState func(notification *BridgeDomainStateNotification)) (err error) {
	// Logger
	plugin.log = logger.NewLogger("-l2-bd-state")
	plugin.log.Info("Initializing BridgeDomainStateUpdater")

	// Mappings
	plugin.bdIndexes = bdIndexes
	plugin.ifIndexes = swIfIndexes

	// State publisher
	plugin.notificationChan = notificationChan
	plugin.publishBdState = publishBdState
	plugin.bdState = make(map[uint32]*l2.BridgeDomainState_BridgeDomain)

	// VPP channel
	plugin.vppCh, err = goVppMux.NewAPIChannel()
	if err != nil {
		return err
	}

	// Name-to-index watcher
	plugin.bdIdxChan = make(chan l2idx.BdChangeDto, 100)
	bdIndexes.WatchNameToIdx(core.PluginName("bdplugin_bdstate"), plugin.bdIdxChan)

	var childCtx context.Context
	childCtx, plugin.cancel = context.WithCancel(ctx)

	// Bridge domain notification watcher
	go plugin.watchVPPNotifications(childCtx)

	return nil
}

// watchVPPNotifications watches for delivery of notifications from VPP.
func (plugin *BridgeDomainStateUpdater) watchVPPNotifications(ctx context.Context) {
	plugin.wg.Add(1)
	defer plugin.wg.Done()

	if plugin.notificationChan != nil {
		plugin.log.Info("watchVPPNotifications for bridge domain state started")
	} else {
		plugin.log.Error("failed to start watchVPPNotifications for bridge domain state")
		return
	}

	for {
		select {
		case notif, ok := <-plugin.notificationChan:
			if !ok {
				continue
			}
			bdName := notif.Name
			switch msg := notif.Message.(type) {
			case *l2_api.BridgeDomainDetails:
				bdState := plugin.processBridgeDomainDetailsNotification(msg, bdName)
				if bdState != nil {
					plugin.publishBdState(&BridgeDomainStateNotification{
						State: bdState,
					})
				}
			default:
				plugin.log.Debugf("L2Plugin: Ignoring unknown VPP notification: %v", msg)
			}

		case bdIdxDto := <-plugin.bdIdxChan:
			bdIdxDto.Done()

		case <-ctx.Done():
			// Stop watching for notifications.
			return
		}
	}
}

func (plugin *BridgeDomainStateUpdater) processBridgeDomainDetailsNotification(msg *l2_api.BridgeDomainDetails, name string) *l2.BridgeDomainState_BridgeDomain {
	bdState := &l2.BridgeDomainState_BridgeDomain{}
	// Delete case.
	if msg.BdID == 0 {
		if name == "" {
			plugin.log.Debugf("invalid bridge domain received: %+v", msg)
			return bdState
		}
		// Mark index to 0 to be removed, but pass name so that the key can be constructed.
		bdState.Index = 0
		bdState.InternalName = name
		return bdState
	}
	bdState.Index = msg.BdID
	name, _, found := plugin.bdIndexes.LookupName(msg.BdID)
	if !found {
		plugin.log.Warnf("bridge domain index not found, index %v is not in the mapping", msg.BdID)
		return bdState
	}
	bdState.InternalName = name
	bdState.InterfaceCount = msg.NSwIfs
	name, _, found = plugin.ifIndexes.LookupName(msg.BviSwIfIndex)
	if found {
		bdState.BviInterface = name
		bdState.BviInterfaceIndex = msg.BviSwIfIndex
	} else {
		bdState.BviInterface = "not_set"
	}
	bdState.L2Params = getBridgeDomainStateParams(msg)
	bdState.Interfaces = plugin.getBridgeDomainInterfaces(msg)
	bdState.LastChange = time.Now().Unix()

	return bdState
}

func (plugin *BridgeDomainStateUpdater) getBridgeDomainInterfaces(msg *l2_api.BridgeDomainDetails) []*l2.BridgeDomainState_BridgeDomain_Interfaces {
	var bdStateInterfaces []*l2.BridgeDomainState_BridgeDomain_Interfaces
	for _, swIfaceDetails := range msg.SwIfDetails {
		bdIfaceState := &l2.BridgeDomainState_BridgeDomain_Interfaces{}
		name, _, found := plugin.ifIndexes.LookupName(swIfaceDetails.SwIfIndex)
		if !found {
			plugin.log.Debugf("Interface name with index %v not found for bridge domain status", swIfaceDetails.SwIfIndex)
			bdIfaceState.Name = "unknown"
		} else {
			bdIfaceState.Name = name
		}
		bdIfaceState.SwIfIndex = swIfaceDetails.SwIfIndex
		bdIfaceState.SplitHorizonGroup = uint32(swIfaceDetails.Shg)
		bdStateInterfaces = append(bdStateInterfaces, bdIfaceState)
	}
	return bdStateInterfaces
}

func getBridgeDomainStateParams(msg *l2_api.BridgeDomainDetails) *l2.BridgeDomainState_BridgeDomain_L2Params {
	params := &l2.BridgeDomainState_BridgeDomain_L2Params{}
	params.Flood = intToBool(msg.Flood)
	params.UnknownUnicastFlood = intToBool(msg.UuFlood)
	params.Forward = intToBool(msg.Forward)
	params.Learn = intToBool(msg.Learn)
	params.ArpTermination = intToBool(msg.ArpTerm)
	params.MacAge = uint32(msg.MacAge)
	return params
}

func intToBool(num uint8) bool {
	if num == 1 {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package chroot

import (
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	awscommon "github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/packer"
	"log"
)

// StepCreateVolume creates a new volume from the snapshot of the root
// device of the AMI.
//
// Produces:
//   volume_id string - The ID of the created volume
type StepCreateVolume struct {
	volumeId string
}

func (s *StepCreateVolume) Run(state map[string]interface{}) multistep.StepAction {
	ec2conn := state["ec2"].(*ec2.EC2)
	image := state["source_image"].(*ec2.Image)
	instance := state["instance"].(*ec2.Instance)
	ui := state["ui"].(packer.Ui)

	// Determine the root device snapshot
	log.Printf("Searching for root device of the image (%s)", image.RootDeviceName)
	var rootDevice *ec2.BlockDeviceMapping
	for _, device := range image.BlockDevices {
		if device.DeviceName == image.RootDeviceName {
			rootDevice = &device
			break
		}
	}

	if rootDevice == nil {
		err := fmt.Errorf("Couldn't find root device!")
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	ui.Say("Creating the root volume...")
	createVolume := &ec2.CreateVolume{
		AvailZone:  instance.AvailZone,
		Size:       rootDevice.VolumeSize,
		SnapshotId: rootDevice.SnapshotId,
		VolumeType: rootDevice.VolumeType,
		IOPS:       rootDevice.IOPS,
	}

	createVolumeResp, err := ec2conn.CreateVolume(createVolume)
	if err != nil {
		err := fmt.Errorf("Error creating root volume: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	// Set the volume ID so we remember to delete it later
	s.volumeId = createVolumeResp.VolumeId
	log.Printf("Volume ID: %s", s.volumeId)

	// Wait for the volume to become ready
	stateChange := awscommon.StateChangeConf{
		Conn:      ec2conn,
		Pending:   []string{"creating"},
		StepState: state,
		Target:    "available",
		Refresh: func() (interface{}, string, error) {
			resp, err := ec2conn.Volumes([]string{s.volumeId}, ec2.NewFilter())
			if err != nil {
				return nil, "", err
			}

			return nil, resp.Volumes[0].Status, nil
		},
	}

	_, err = awscommon.WaitForState(&stateChange)
	if err != nil {
		err := fmt.Errorf("Error waiting for volume: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	state["volume_id"] = s.volumeId
	return multistep.ActionContinue
}

func (s *StepCreateVolume) Cleanup(state map[string]interface{}) {
	if s.volumeId == "" {
		return
	}

	ec2conn := state["ec2"].(*ec2.EC2)
	ui := state["ui"].(packer.Ui)

	ui.Say("Deleting the created EBS volume...")
	_, err := ec2conn.DeleteVolume(s.volumeId)
	if err != nil {
		ui.Error(fmt.Sprintf("Error deleting EBS volume: %s", err))
	}
}
<commit_msg>builder/amazon/chroot: improved logging about what volume created<commit_after>package chroot

import (
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	awscommon "github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/packer"
	"log"
)

// StepCreateVolume creates a new volume from the snapshot of the root
// device of the AMI.
//
// Produces:
//   volume_id string - The ID of the created volume
type StepCreateVolume struct {
	volumeId string
}

func (s *StepCreateVolume) Run(state map[string]interface{}) multistep.StepAction {
	ec2conn := state["ec2"].(*ec2.EC2)
	image := state["source_image"].(*ec2.Image)
	instance := state["instance"].(*ec2.Instance)
	ui := state["ui"].(packer.Ui)

	// Determine the root device snapshot
	log.Printf("Searching for root device of the image (%s)", image.RootDeviceName)
	var rootDevice *ec2.BlockDeviceMapping
	for _, device := range image.BlockDevices {
		if device.DeviceName == image.RootDeviceName {
			rootDevice = &device
			break
		}
	}

	if rootDevice == nil {
		err := fmt.Errorf("Couldn't find root device!")
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	ui.Say("Creating the root volume...")
	createVolume := &ec2.CreateVolume{
		AvailZone:  instance.AvailZone,
		Size:       rootDevice.VolumeSize,
		SnapshotId: rootDevice.SnapshotId,
		VolumeType: rootDevice.VolumeType,
		IOPS:       rootDevice.IOPS,
	}
	log.Printf("Create args: %#v", createVolume)

	createVolumeResp, err := ec2conn.CreateVolume(createVolume)
	if err != nil {
		err := fmt.Errorf("Error creating root volume: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	// Set the volume ID so we remember to delete it later
	s.volumeId = createVolumeResp.VolumeId
	log.Printf("Volume ID: %s", s.volumeId)

	// Wait for the volume to become ready
	stateChange := awscommon.StateChangeConf{
		Conn:      ec2conn,
		Pending:   []string{"creating"},
		StepState: state,
		Target:    "available",
		Refresh: func() (interface{}, string, error) {
			resp, err := ec2conn.Volumes([]string{s.volumeId}, ec2.NewFilter())
			if err != nil {
				return nil, "", err
			}

			return nil, resp.Volumes[0].Status, nil
		},
	}

	_, err = awscommon.WaitForState(&stateChange)
	if err != nil {
		err := fmt.Errorf("Error waiting for volume: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	state["volume_id"] = s.volumeId
	return multistep.ActionContinue
}

func (s *StepCreateVolume) Cleanup(state map[string]interface{}) {
	if s.volumeId == "" {
		return
	}

	ec2conn := state["ec2"].(*ec2.EC2)
	ui := state["ui"].(packer.Ui)

	ui.Say("Deleting the created EBS volume...")
	_, err := ec2conn.DeleteVolume(s.volumeId)
	if err != nil {
		ui.Error(fmt.Sprintf("Error deleting EBS volume: %s", err))
	}
}
<|endoftext|>


<commit_before>package manifest

import (
	"github.com/sandstorm/dokku-enterprise-plugin/core/utility"
	"strings"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
)

func CreateManifest(application string) []byte {
	return CreateManifestAndStoreDataIntoTemporaryFolder(application, "")
}
/**
 * Second argument might be NULL
 */
func CreateManifestAndStoreDataIntoTemporaryFolder(application string, temporaryFolder string) []byte {
	applicationConfig := utility.ExecCommand("dokku", "--quiet", "config", application)

	parsedApplicationConfig := parseConfig(applicationConfig)
	manifest := new(manifest)
	manifest.Config = make(map[string]string)

	manifestWrapper := manifestWrapper{
		Version: 1,
		AppName: application,
		Manifest: manifest,
	}

	// Database (Mariadb)
	dbName := extractMariadb(parsedApplicationConfig, "DATABASE_URL", &manifestWrapper)
	if len(dbName) > 0 && len(temporaryFolder) > 0 {
		utility.ExecCommandAndDumpResultToFile(temporaryFolder + "/mariadb/0.sql", "dokku", "mariadb:export", dbName)
	}
	dbIndex := 1
	for configKey, _ := range parsedApplicationConfig {
		switch {
		case strings.HasPrefix(configKey, "DOKKU_MARIADB_"):
			dbName := extractMariadb(parsedApplicationConfig, configKey, &manifestWrapper)
			if len(dbName) > 0 && len(temporaryFolder) > 0 {
				utility.ExecCommandAndDumpResultToFile(temporaryFolder + "/mariadb/" + strconv.Itoa(dbIndex) + ".sql", "dokku", "mariadb:export", dbName)
			}

			dbIndex++
		}
	}

	// Remove defaults from config
	if parsedApplicationConfig["DOKKU_APP_RESTORE"] == "1" {
		delete(parsedApplicationConfig, "DOKKU_APP_RESTORE")
	}
	delete(parsedApplicationConfig, "DOKKU_PROXY_PORT_MAP")
	delete(parsedApplicationConfig, "DOKKU_DOCKERFILE_PORTS")
	delete(parsedApplicationConfig, "NO_VHOST")
	delete(parsedApplicationConfig, "DOKKU_APP_TYPE")

	// Remaining config
	for configKey, configValue := range parsedApplicationConfig {
		manifestWrapper.Manifest.Config[configKey] = configValue
	}

	// Docker options
	extractDockerOptions(&manifestWrapper, "deploy")
	extractDockerOptions(&manifestWrapper, "run")
	extractDockerOptions(&manifestWrapper, "build")

	manifestAsBytes, err := json.MarshalIndent(manifestWrapper, "", "  ")

	if err != nil {
		log.Fatalf("There was an error serializing JSON manifest: %v", err)
	}

	if len(temporaryFolder) > 0 {
		ioutil.WriteFile(temporaryFolder + "/manifest.json", manifestAsBytes, 0644)
	}
	return manifestAsBytes
}

/************************
 EXTRACTORS
 */

// returns the DB name, if found!
func extractMariadb(parsedApplicationConfig map[string]string, configKey string, manifestWrapper *manifestWrapper) string {
	dbUrl, dbUrlExists := parsedApplicationConfig[configKey]
	if dbUrlExists {
		switch {
		case strings.Contains(dbUrl, "mysql://mariadb"):
			manifestWrapper.Manifest.Mariadb = append(manifestWrapper.Manifest.Mariadb, replaceApplicationNameInString(extractDbName(dbUrl), manifestWrapper, "mariadb." + configKey, ""))

			return extractDbName(dbUrl)
		default:
			manifestWrapper.Errors = append(manifestWrapper.Errors, fmt.Sprintf("Could not parse DB URL, which was %v", dbUrl))
		}

		delete(parsedApplicationConfig, configKey);
	} else {
		manifestWrapper.DebugInfo = append(manifestWrapper.DebugInfo, "Did not find DB.")
	}
	return ""
}

func extractDockerOptions(manifestWrapper *manifestWrapper, phase string) {
	dockerOptions := utility.ExecCommand("dokku", "docker-options", manifestWrapper.AppName, phase)

	for _, line := range strings.Split(dockerOptions, "\n") {
		line = strings.TrimSpace(line)
		switch line {
		case "--restart=on-failure:10":
			// Default value; we don't need to include this
			continue
		case "Deploy options:", "Build options:", "Run options:",
			"Deploy options: none", "Build options: none", "Run options: none":
			// first line; we can skip this.
			continue
		default:
			switch (phase) {
			case "deploy":
				manifestWrapper.Manifest.DockerOptions.Deploy = append(manifestWrapper.Manifest.DockerOptions.Deploy, replaceApplicationNameInString(line, manifestWrapper, "dockerOptions.deploy", "-v "))
			case "run":
				manifestWrapper.Manifest.DockerOptions.Run = append(manifestWrapper.Manifest.DockerOptions.Run, replaceApplicationNameInString(line, manifestWrapper, "dockerOptions.run", "-v "))
			case "build":
				manifestWrapper.Manifest.DockerOptions.Build = append(manifestWrapper.Manifest.DockerOptions.Build, replaceApplicationNameInString(line, manifestWrapper, "dockerOptions.build", "-v "))
			default:
				manifestWrapper.Errors = append(manifestWrapper.Errors, fmt.Sprintf("Unknown phase %v given; this error should never happen", phase))
			}
		}
	}
}

/************************
 HELPERS
 */
func replaceApplicationNameInString(s string, manifestWrapper *manifestWrapper, key string, addErrorOnlyIfStringStartsWith string) string {
	r := strings.Replace(s, manifestWrapper.AppName, "[appName]", -1)

	if !strings.Contains(r, "[appName]") && strings.HasPrefix(s, addErrorOnlyIfStringStartsWith) {
		manifestWrapper.Errors = append(manifestWrapper.Errors, fmt.Sprintf("%v: did not find application name '%v' inside string: %v.", key, manifestWrapper.AppName, s))
	}

	return r
}
func extractDbName(dbUrl string) string {
	return dbUrl[strings.LastIndex(dbUrl, "/") + 1:]
}
func parseConfig(applicationConfig string) map[string]string {
	parsed := make(map[string]string)

	for _, line := range strings.Split(applicationConfig, "\n") {
		split := strings.SplitN(line, ":", 2)
		parsed[strings.TrimSpace(split[0])] = strings.TrimSpace(split[1])
	}
	return parsed
}<commit_msg>BUGFIX: fix testcases<commit_after>package manifest

import (
	"github.com/sandstorm/dokku-enterprise-plugin/core/utility"
	"strings"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
)

func CreateManifest(application string) []byte {
	return CreateManifestAndStoreDataIntoTemporaryFolder(application, "")
}
/**
 * Second argument might be NULL
 */
func CreateManifestAndStoreDataIntoTemporaryFolder(application string, temporaryFolder string) []byte {
	applicationConfig := utility.ExecCommand("dokku", "--quiet", "config", application)

	parsedApplicationConfig := parseConfig(applicationConfig)
	manifest := new(manifest)
	manifest.Config = make(map[string]string)

	manifestWrapper := manifestWrapper{
		Version: 1,
		AppName: application,
		Manifest: manifest,
	}

	// Database (Mariadb)
	dbName := extractMariadb(parsedApplicationConfig, "DATABASE_URL", &manifestWrapper)
	if len(dbName) > 0 && len(temporaryFolder) > 0 {
		utility.ExecCommandAndDumpResultToFile(temporaryFolder + "/mariadb/0.sql", "dokku", "mariadb:export", dbName)
	}
	dbIndex := 1
	for configKey, _ := range parsedApplicationConfig {
		switch {
		case strings.HasPrefix(configKey, "DOKKU_MARIADB_"):
			dbName := extractMariadb(parsedApplicationConfig, configKey, &manifestWrapper)
			if len(dbName) > 0 && len(temporaryFolder) > 0 {
				utility.ExecCommandAndDumpResultToFile(temporaryFolder + "/mariadb/" + strconv.Itoa(dbIndex) + ".sql", "dokku", "mariadb:export", dbName)
			}

			dbIndex++
		}
	}

	// Remove defaults from config
	if parsedApplicationConfig["DOKKU_APP_RESTORE"] == "1" {
		delete(parsedApplicationConfig, "DOKKU_APP_RESTORE")
	}
	delete(parsedApplicationConfig, "DOKKU_PROXY_PORT_MAP")
	delete(parsedApplicationConfig, "DOKKU_DOCKERFILE_PORTS")
	delete(parsedApplicationConfig, "NO_VHOST")
	delete(parsedApplicationConfig, "DOKKU_APP_TYPE")

	// Remaining config
	for configKey, configValue := range parsedApplicationConfig {
		manifestWrapper.Manifest.Config[configKey] = configValue
	}

	// Docker options
	extractDockerOptions(&manifestWrapper, "deploy")
	extractDockerOptions(&manifestWrapper, "run")
	extractDockerOptions(&manifestWrapper, "build")

	manifestAsBytes, err := json.MarshalIndent(manifestWrapper, "", "  ")

	if err != nil {
		log.Fatalf("There was an error serializing JSON manifest: %v", err)
	}

	if len(temporaryFolder) > 0 {
		ioutil.WriteFile(temporaryFolder + "/manifest.json", manifestAsBytes, 0644)
	}
	return manifestAsBytes
}

/************************
 EXTRACTORS
 */

// returns the DB name, if found!
func extractMariadb(parsedApplicationConfig map[string]string, configKey string, manifestWrapper *manifestWrapper) string {
	dbUrl, dbUrlExists := parsedApplicationConfig[configKey]
	if dbUrlExists {
		switch {
		case strings.Contains(dbUrl, "mysql://mariadb"):
			manifestWrapper.Manifest.Mariadb = append(manifestWrapper.Manifest.Mariadb, replaceApplicationNameInString(extractDbName(dbUrl), manifestWrapper, "mariadb." + configKey, ""))

			delete(parsedApplicationConfig, configKey)
			return extractDbName(dbUrl)
		default:
			manifestWrapper.Errors = append(manifestWrapper.Errors, fmt.Sprintf("Could not parse DB URL, which was %v", dbUrl))
		}
	} else {
		manifestWrapper.DebugInfo = append(manifestWrapper.DebugInfo, "Did not find DB.")
	}
	return ""
}

func extractDockerOptions(manifestWrapper *manifestWrapper, phase string) {
	dockerOptions := utility.ExecCommand("dokku", "docker-options", manifestWrapper.AppName, phase)

	for _, line := range strings.Split(dockerOptions, "\n") {
		line = strings.TrimSpace(line)
		switch line {
		case "--restart=on-failure:10":
			// Default value; we don't need to include this
			continue
		case "Deploy options:", "Build options:", "Run options:",
			"Deploy options: none", "Build options: none", "Run options: none":
			// first line; we can skip this.
			continue
		default:
			switch (phase) {
			case "deploy":
				manifestWrapper.Manifest.DockerOptions.Deploy = append(manifestWrapper.Manifest.DockerOptions.Deploy, replaceApplicationNameInString(line, manifestWrapper, "dockerOptions.deploy", "-v "))
			case "run":
				manifestWrapper.Manifest.DockerOptions.Run = append(manifestWrapper.Manifest.DockerOptions.Run, replaceApplicationNameInString(line, manifestWrapper, "dockerOptions.run", "-v "))
			case "build":
				manifestWrapper.Manifest.DockerOptions.Build = append(manifestWrapper.Manifest.DockerOptions.Build, replaceApplicationNameInString(line, manifestWrapper, "dockerOptions.build", "-v "))
			default:
				manifestWrapper.Errors = append(manifestWrapper.Errors, fmt.Sprintf("Unknown phase %v given; this error should never happen", phase))
			}
		}
	}
}

/************************
 HELPERS
 */
func replaceApplicationNameInString(s string, manifestWrapper *manifestWrapper, key string, addErrorOnlyIfStringStartsWith string) string {
	r := strings.Replace(s, manifestWrapper.AppName, "[appName]", -1)

	if !strings.Contains(r, "[appName]") && strings.HasPrefix(s, addErrorOnlyIfStringStartsWith) {
		manifestWrapper.Errors = append(manifestWrapper.Errors, fmt.Sprintf("%v: did not find application name '%v' inside string: %v.", key, manifestWrapper.AppName, s))
	}

	return r
}
func extractDbName(dbUrl string) string {
	return dbUrl[strings.LastIndex(dbUrl, "/") + 1:]
}
func parseConfig(applicationConfig string) map[string]string {
	parsed := make(map[string]string)

	for _, line := range strings.Split(applicationConfig, "\n") {
		split := strings.SplitN(line, ":", 2)
		parsed[strings.TrimSpace(split[0])] = strings.TrimSpace(split[1])
	}
	return parsed
}<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cumulativetodeltaprocessor

import (
	"context"
	"math"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/model/pdata"
	"go.uber.org/zap"

	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor/internal/tracking"
)

type cumulativeToDeltaProcessor struct {
	metrics         map[string]struct{}
	logger          *zap.Logger
	deltaCalculator tracking.MetricTracker
	cancelFunc      context.CancelFunc
}

func newCumulativeToDeltaProcessor(config *Config, logger *zap.Logger) *cumulativeToDeltaProcessor {
	ctx, cancel := context.WithCancel(context.Background())
	p := &cumulativeToDeltaProcessor{
		logger:          logger,
		deltaCalculator: tracking.NewMetricTracker(ctx, logger, config.MaxStaleness),
		cancelFunc:      cancel,
	}
	if len(config.Metrics) > 0 {
		p.metrics = make(map[string]struct{}, len(config.Metrics))
		for _, m := range config.Metrics {
			p.metrics[m] = struct{}{}
		}
	}
	return p
}

// Start is invoked during service startup.
func (ctdp *cumulativeToDeltaProcessor) Start(context.Context, component.Host) error {
	return nil
}

// processMetrics implements the ProcessMetricsFunc type.
func (ctdp *cumulativeToDeltaProcessor) processMetrics(_ context.Context, md pdata.Metrics) (pdata.Metrics, error) {
	resourceMetricsSlice := md.ResourceMetrics()
	resourceMetricsSlice.RemoveIf(func(rm pdata.ResourceMetrics) bool {
		ilms := rm.InstrumentationLibraryMetrics()
		ilms.RemoveIf(func(ilm pdata.InstrumentationLibraryMetrics) bool {
			ms := ilm.Metrics()
			ms.RemoveIf(func(m pdata.Metric) bool {
				if _, ok := ctdp.metrics[m.Name()]; !ok {
					return false
				}
				baseIdentity := tracking.MetricIdentity{
					Resource:               rm.Resource(),
					InstrumentationLibrary: ilm.InstrumentationLibrary(),
					MetricDataType:         m.DataType(),
					MetricName:             m.Name(),
					MetricUnit:             m.Unit(),
				}
				switch m.DataType() {
				case pdata.MetricDataTypeSum:
					ms := m.Sum()
					if ms.AggregationTemporality() != pdata.MetricAggregationTemporalityCumulative {
						return false
					}

					// Ignore any metrics that aren't monotonic
					if !ms.IsMonotonic() {
						return false
					}

					baseIdentity.MetricIsMonotonic = ms.IsMonotonic()
					ctdp.convertDataPoints(ms.DataPoints(), baseIdentity)
					ms.SetAggregationTemporality(pdata.MetricAggregationTemporalityDelta)
					return ms.DataPoints().Len() == 0
				default:
					return false
				}
			})
			return ilm.Metrics().Len() == 0
		})
		return rm.InstrumentationLibraryMetrics().Len() == 0
	})
	return md, nil
}

// Shutdown is invoked during service shutdown.
func (ctdp *cumulativeToDeltaProcessor) Shutdown(context.Context) error {
	ctdp.cancelFunc()
	return nil
}

func (ctdp *cumulativeToDeltaProcessor) convertDataPoints(in interface{}, baseIdentity tracking.MetricIdentity) {
	switch dps := in.(type) {
	case pdata.NumberDataPointSlice:
		dps.RemoveIf(func(dp pdata.NumberDataPoint) bool {
			id := baseIdentity
			id.StartTimestamp = dp.StartTimestamp()
			id.Attributes = dp.Attributes()
			id.MetricValueType = dp.Type()
			point := tracking.ValuePoint{
				ObservedTimestamp: dp.Timestamp(),
			}
			if id.IsFloatVal() {
				// Do not attempt to transform NaN values
				if math.IsNaN(dp.DoubleVal()) {
					return false
				}
				point.FloatValue = dp.DoubleVal()
			} else {
				point.IntValue = dp.IntVal()
			}
			trackingPoint := tracking.MetricPoint{
				Identity: id,
				Value:    point,
			}
			delta, valid := ctdp.deltaCalculator.Convert(trackingPoint)

			// When converting non-monotonic cumulative counters,
			// the first data point is omitted since the initial
			// reference is not assumed to be zero
			if !valid {
				return true
			}
			dp.SetStartTimestamp(delta.StartTimestamp)
			if id.IsFloatVal() {
				dp.SetDoubleVal(delta.FloatValue)
			} else {
				dp.SetIntVal(delta.IntValue)
			}
			return false
		})
	}
}
<commit_msg>[cumulativetodeltaprocessor]: small nit, avoid creating object until is necessary (#5872)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cumulativetodeltaprocessor

import (
	"context"
	"math"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/model/pdata"
	"go.uber.org/zap"

	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor/internal/tracking"
)

type cumulativeToDeltaProcessor struct {
	metrics         map[string]struct{}
	logger          *zap.Logger
	deltaCalculator tracking.MetricTracker
	cancelFunc      context.CancelFunc
}

func newCumulativeToDeltaProcessor(config *Config, logger *zap.Logger) *cumulativeToDeltaProcessor {
	ctx, cancel := context.WithCancel(context.Background())
	p := &cumulativeToDeltaProcessor{
		logger:          logger,
		deltaCalculator: tracking.NewMetricTracker(ctx, logger, config.MaxStaleness),
		cancelFunc:      cancel,
	}
	if len(config.Metrics) > 0 {
		p.metrics = make(map[string]struct{}, len(config.Metrics))
		for _, m := range config.Metrics {
			p.metrics[m] = struct{}{}
		}
	}
	return p
}

// Start is invoked during service startup.
func (ctdp *cumulativeToDeltaProcessor) Start(context.Context, component.Host) error {
	return nil
}

// processMetrics implements the ProcessMetricsFunc type.
func (ctdp *cumulativeToDeltaProcessor) processMetrics(_ context.Context, md pdata.Metrics) (pdata.Metrics, error) {
	resourceMetricsSlice := md.ResourceMetrics()
	resourceMetricsSlice.RemoveIf(func(rm pdata.ResourceMetrics) bool {
		ilms := rm.InstrumentationLibraryMetrics()
		ilms.RemoveIf(func(ilm pdata.InstrumentationLibraryMetrics) bool {
			ms := ilm.Metrics()
			ms.RemoveIf(func(m pdata.Metric) bool {
				if _, ok := ctdp.metrics[m.Name()]; !ok {
					return false
				}
				switch m.DataType() {
				case pdata.MetricDataTypeSum:
					ms := m.Sum()
					if ms.AggregationTemporality() != pdata.MetricAggregationTemporalityCumulative {
						return false
					}

					// Ignore any metrics that aren't monotonic
					if !ms.IsMonotonic() {
						return false
					}

					baseIdentity := tracking.MetricIdentity{
						Resource:               rm.Resource(),
						InstrumentationLibrary: ilm.InstrumentationLibrary(),
						MetricDataType:         m.DataType(),
						MetricName:             m.Name(),
						MetricUnit:             m.Unit(),
						MetricIsMonotonic:      ms.IsMonotonic(),
					}
					ctdp.convertDataPoints(ms.DataPoints(), baseIdentity)
					ms.SetAggregationTemporality(pdata.MetricAggregationTemporalityDelta)
					return ms.DataPoints().Len() == 0
				default:
					return false
				}
			})
			return ilm.Metrics().Len() == 0
		})
		return rm.InstrumentationLibraryMetrics().Len() == 0
	})
	return md, nil
}

// Shutdown is invoked during service shutdown.
func (ctdp *cumulativeToDeltaProcessor) Shutdown(context.Context) error {
	ctdp.cancelFunc()
	return nil
}

func (ctdp *cumulativeToDeltaProcessor) convertDataPoints(in interface{}, baseIdentity tracking.MetricIdentity) {
	switch dps := in.(type) {
	case pdata.NumberDataPointSlice:
		dps.RemoveIf(func(dp pdata.NumberDataPoint) bool {
			id := baseIdentity
			id.StartTimestamp = dp.StartTimestamp()
			id.Attributes = dp.Attributes()
			id.MetricValueType = dp.Type()
			point := tracking.ValuePoint{
				ObservedTimestamp: dp.Timestamp(),
			}
			if id.IsFloatVal() {
				// Do not attempt to transform NaN values
				if math.IsNaN(dp.DoubleVal()) {
					return false
				}
				point.FloatValue = dp.DoubleVal()
			} else {
				point.IntValue = dp.IntVal()
			}
			trackingPoint := tracking.MetricPoint{
				Identity: id,
				Value:    point,
			}
			delta, valid := ctdp.deltaCalculator.Convert(trackingPoint)

			// When converting non-monotonic cumulative counters,
			// the first data point is omitted since the initial
			// reference is not assumed to be zero
			if !valid {
				return true
			}
			dp.SetStartTimestamp(delta.StartTimestamp)
			if id.IsFloatVal() {
				dp.SetDoubleVal(delta.FloatValue)
			} else {
				dp.SetIntVal(delta.IntValue)
			}
			return false
		})
	}
}
<|endoftext|>


<commit_before>package igc

import (
	"bufio"
	"fmt"
	"github.com/twpayne/gogeom/geom"
	"io"
	"strings"
	"time"
)

type Errors map[int]error

func (es Errors) Error() string {
	ss := make([]string, len(es))
	for i, e := range es {
		ss[i] = e.Error()
	}
	return strings.Join(ss, "\n")
}

func parseDec(s string, start, stop int) (int, error) {
	result := 0
	for i := start; i < stop; i++ {
		if c := s[i]; '0' <= c && c <= '9' {
			result = 10*result + int(c) - '0'
		} else {
			return 0, fmt.Errorf("invalid")
		}
	}
	return result, nil
}

type parser struct {
	pointMs           []geom.PointM
	year, month, day  int
	ladStart, ladStop int
	lodStart, lodStop int
	tdsStart, tdsStop int
}

func newParser() *parser {
	return new(parser)
}

func (p *parser) parseB(line string) error {
	var err error
	var hour, minute, second int
	if hour, err = parseDec(line, 1, 3); err != nil {
		return err
	}
	if minute, err = parseDec(line, 3, 5); err != nil {
		return err
	}
	if second, err = parseDec(line, 5, 7); err != nil {
		return err
	}
	var latDeg, latMin int
	if latDeg, err = parseDec(line, 7, 9); err != nil {
		return err
	}
	if latMin, err = parseDec(line, 9, 14); err != nil {
		return err
	}
	lat := float64(latDeg) + float64(latMin)/60000.
	switch c := line[14]; c {
	case 'N':
	case 'S':
		lat = -lat
	default:
		return fmt.Errorf("unexpected character %v", c)
	}
	var lngDeg, lngMin int
	lngDeg, err = parseDec(line, 15, 18)
	if err != nil {
		return err
	}
	lngMin, err = parseDec(line, 18, 23)
	if err != nil {
		return err
	}
	lng := float64(lngDeg) + float64(lngMin)/60000.
	switch c := line[23]; c {
	case 'E':
	case 'W':
		lng = -lng
	default:
		return fmt.Errorf("unexpected character %v", c)
	}
	date := time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, 0, time.UTC)
	pointM := geom.PointM{lng, lat, float64(date.UnixNano()) / 1e9}
	p.pointMs = append(p.pointMs, pointM)
	return nil
}

func (p *parser) parseH(line string) error {
	switch {
	case strings.HasPrefix(line, "HFDTE"):
		return p.parseHFDTE(line)
	default:
		return nil
	}
}

func (p *parser) parseHFDTE(line string) error {
	var err error
	var day, month, year int
	if day, err = parseDec(line, 5, 7); err != nil {
		return err
	}
	if month, err = parseDec(line, 7, 9); err != nil {
		return err
	}
	if year, err = parseDec(line, 9, 11); err != nil {
		return err
	}
	// FIXME check for invalid dates
	p.day = day
	p.month = month
	if year < 70 {
		p.year = 2000 + year
	} else {
		p.year = 1970 + year
	}
	return nil
}

func (p *parser) parseI(line string) error {
	var err error
	var n int
	if len(line) < 3 {
		return fmt.Errorf("line too short")
	}
	if n, err = parseDec(line, 1, 3); err != nil {
		return err
	}
	if len(line) < 7*n+3 {
		return fmt.Errorf("line too short")
	}
	for i := 0; i < n; i++ {
		var start, stop int
		if start, err = parseDec(line, 7*i+3, 7*i+5); err != nil {
			return err
		}
		if stop, err = parseDec(line, 7*i+5, 7*i+7); err != nil {
			return err
		}
		switch line[7*i+7 : 7*i+10] {
		case "LAD":
			p.ladStart, p.ladStop = start-1, stop
		case "LOD":
			p.lodStart, p.lodStop = start-1, stop
		case "TDS":
			p.tdsStart, p.tdsStop = start-1, stop
		}
	}
	return nil
}

func (p *parser) parseLine(line string) error {
	switch line[0] {
	case 'B':
		return p.parseB(line)
	case 'H':
		return p.parseH(line)
	case 'I':
		return p.parseI(line)
	default:
		return nil
	}
}

func doParse(r io.Reader) (*parser, Errors) {
	errors := make(Errors)
	p := newParser()
	s := bufio.NewScanner(r)
	line := 0
	for s.Scan() {
		line++
		if err := p.parseLine(s.Text()); err != nil {
			errors[line] = err
		}
	}
	return p, errors
}

func Read(r io.Reader) ([]geom.PointM, error) {
	p, errors := doParse(r)
	if len(errors) == 0 {
		return p.pointMs, nil
	} else {
		return p.pointMs, errors
	}
}
<commit_msg>Rename Min to MMin (milliminutes)<commit_after>package igc

import (
	"bufio"
	"fmt"
	"github.com/twpayne/gogeom/geom"
	"io"
	"strings"
	"time"
)

type Errors map[int]error

func (es Errors) Error() string {
	ss := make([]string, len(es))
	for i, e := range es {
		ss[i] = e.Error()
	}
	return strings.Join(ss, "\n")
}

func parseDec(s string, start, stop int) (int, error) {
	result := 0
	for i := start; i < stop; i++ {
		if c := s[i]; '0' <= c && c <= '9' {
			result = 10*result + int(c) - '0'
		} else {
			return 0, fmt.Errorf("invalid")
		}
	}
	return result, nil
}

type parser struct {
	pointMs           []geom.PointM
	year, month, day  int
	ladStart, ladStop int
	lodStart, lodStop int
	tdsStart, tdsStop int
}

func newParser() *parser {
	return new(parser)
}

func (p *parser) parseB(line string) error {
	var err error
	var hour, minute, second int
	if hour, err = parseDec(line, 1, 3); err != nil {
		return err
	}
	if minute, err = parseDec(line, 3, 5); err != nil {
		return err
	}
	if second, err = parseDec(line, 5, 7); err != nil {
		return err
	}
	var latDeg, latMMin int
	if latDeg, err = parseDec(line, 7, 9); err != nil {
		return err
	}
	if latMMin, err = parseDec(line, 9, 14); err != nil {
		return err
	}
	lat := float64(latDeg) + float64(latMMin)/60000.
	switch c := line[14]; c {
	case 'N':
	case 'S':
		lat = -lat
	default:
		return fmt.Errorf("unexpected character %v", c)
	}
	var lngDeg, lngMMin int
	lngDeg, err = parseDec(line, 15, 18)
	if err != nil {
		return err
	}
	lngMMin, err = parseDec(line, 18, 23)
	if err != nil {
		return err
	}
	lng := float64(lngDeg) + float64(lngMMin)/60000.
	switch c := line[23]; c {
	case 'E':
	case 'W':
		lng = -lng
	default:
		return fmt.Errorf("unexpected character %v", c)
	}
	date := time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, 0, time.UTC)
	pointM := geom.PointM{lng, lat, float64(date.UnixNano()) / 1e9}
	p.pointMs = append(p.pointMs, pointM)
	return nil
}

func (p *parser) parseH(line string) error {
	switch {
	case strings.HasPrefix(line, "HFDTE"):
		return p.parseHFDTE(line)
	default:
		return nil
	}
}

func (p *parser) parseHFDTE(line string) error {
	var err error
	var day, month, year int
	if day, err = parseDec(line, 5, 7); err != nil {
		return err
	}
	if month, err = parseDec(line, 7, 9); err != nil {
		return err
	}
	if year, err = parseDec(line, 9, 11); err != nil {
		return err
	}
	// FIXME check for invalid dates
	p.day = day
	p.month = month
	if year < 70 {
		p.year = 2000 + year
	} else {
		p.year = 1970 + year
	}
	return nil
}

func (p *parser) parseI(line string) error {
	var err error
	var n int
	if len(line) < 3 {
		return fmt.Errorf("line too short")
	}
	if n, err = parseDec(line, 1, 3); err != nil {
		return err
	}
	if len(line) < 7*n+3 {
		return fmt.Errorf("line too short")
	}
	for i := 0; i < n; i++ {
		var start, stop int
		if start, err = parseDec(line, 7*i+3, 7*i+5); err != nil {
			return err
		}
		if stop, err = parseDec(line, 7*i+5, 7*i+7); err != nil {
			return err
		}
		switch line[7*i+7 : 7*i+10] {
		case "LAD":
			p.ladStart, p.ladStop = start-1, stop
		case "LOD":
			p.lodStart, p.lodStop = start-1, stop
		case "TDS":
			p.tdsStart, p.tdsStop = start-1, stop
		}
	}
	return nil
}

func (p *parser) parseLine(line string) error {
	switch line[0] {
	case 'B':
		return p.parseB(line)
	case 'H':
		return p.parseH(line)
	case 'I':
		return p.parseI(line)
	default:
		return nil
	}
}

func doParse(r io.Reader) (*parser, Errors) {
	errors := make(Errors)
	p := newParser()
	s := bufio.NewScanner(r)
	line := 0
	for s.Scan() {
		line++
		if err := p.parseLine(s.Text()); err != nil {
			errors[line] = err
		}
	}
	return p, errors
}

func Read(r io.Reader) ([]geom.PointM, error) {
	p, errors := doParse(r)
	if len(errors) == 0 {
		return p.pointMs, nil
	} else {
		return p.pointMs, errors
	}
}
<|endoftext|>


<commit_before>package jit

/*
#cgo CFLAGS: -I./include
#cgo LDFLAGS: -L./lib -lm -ljit

#include <stdio.h>
#include <jit/jit.h>
#include <jit/jit-dump.h>

extern int on_demand_compile(jit_function_t);

static void SetOnDemandCompileFunction(jit_function_t f) {
    jit_function_set_on_demand_compiler(f, on_demand_compile);
}

static FILE *getStdout(void) { return stdout; }
*/
import "C"
import "unsafe"
import "reflect"

const CDECL = C.jit_abi_cdecl

type Context struct {
	C C.jit_context_t
}

type Type struct {
	C C.jit_type_t
}

type Signature struct {
	C C.jit_type_t
}

type Function struct {
	C         C.jit_function_t
	retType   Type
	paramType []Type
}

type Value struct {
	C C.jit_value_t
}

type Label struct {
	C C.jit_label_t
}

func Int() Type {
	return Type{C.jit_type_int}
}

func UInt() Type {
	return Type{C.jit_type_uint}
}

func Void() Type {
	return Type{C.jit_type_void}
}

func VoidPtr() Type {
	return Type{C.jit_type_void_ptr}
}

func NewContext() *Context {
	return &Context{C.jit_context_create()}
}

func NewSignature(ret Type, params []Type) *Signature {
	if len(params) == 0 {
		signature := C.jit_type_create_signature(C.jit_abi_cdecl,
			ret.C,
			(*C.jit_type_t)(nil),
			C.uint(0), 1)
		return &Signature{signature}
	} else {
		signature := C.jit_type_create_signature(C.jit_abi_cdecl,
			ret.C,
			(*C.jit_type_t)(unsafe.Pointer(&params[0])),
			C.uint(len(params)), 1)
		return &Signature{signature}
	}
}

func NewLabel() *Label {
	return &Label{C.jit_label_undefined}
}

// ========== Context =============

func (c *Context) NewFunction(ret Type, params []Type) *Function {
	signature := NewSignature(ret, params)
	function := C.jit_function_create(c.C, signature.C)
	C.jit_type_free(signature.C)
	return &Function{C: function, retType: ret, paramType: params}
}

func (c *Context) BuildStart() {
	C.jit_context_build_start(c.C)
}

func (c *Context) BuildEnd() {
	C.jit_context_build_end(c.C)
}

func (c *Context) Destroy() {
	C.jit_context_destroy(c.C)
}

// ========== Context =============

// ========== Function =============

func (f *Function) Param(i int) *Value {
	return &Value{C.jit_value_get_param(f.C, C.uint(i))}
}

func (f *Function) Param2() (*Value, *Value) {
	return &Value{C.jit_value_get_param(f.C, C.uint(0))},
		&Value{C.jit_value_get_param(f.C, C.uint(1))}
}

func (f *Function) Param3() (*Value, *Value, *Value) {
	return &Value{C.jit_value_get_param(f.C, C.uint(0))},
		&Value{C.jit_value_get_param(f.C, C.uint(1))},
		&Value{C.jit_value_get_param(f.C, C.uint(2))}
}

func (f *Function) Mul(a, b *Value) *Value {
	return &Value{C.jit_insn_mul(f.C, a.C, b.C)}
}

func (f *Function) Add(a, b *Value) *Value {
	return &Value{C.jit_insn_add(f.C, a.C, b.C)}
}

func (f *Function) Sub(a, b *Value) *Value {
	return &Value{C.jit_insn_sub(f.C, a.C, b.C)}
}

func (f *Function) Return(ret *Value) {
	C.jit_insn_return(f.C, ret.C)
}

func (f *Function) Store(x, y *Value) {
	C.jit_insn_store(f.C, x.C, y.C)
}

func (f *Function) StoreRelative(x *Value, offset int, y *Value) {
	C.jit_insn_store_relative(f.C, x.C, C.jit_nint(offset), y.C)
}

func (f *Function) Eq(a, b *Value) *Value {
	return &Value{C.jit_insn_eq(f.C, a.C, b.C)}
}

func (f *Function) BranchIfNot(v *Value, label *Label) {
	C.jit_insn_branch_if_not(f.C, v.C, (*C.jit_label_t)(unsafe.Pointer(label)))
}

func (f *Function) Label(label *Label) {
	C.jit_insn_label(f.C, (*C.jit_label_t)(unsafe.Pointer(label)))
}

func (f *Function) LessThan(a, b *Value) *Value {
	return &Value{C.jit_insn_lt(f.C, a.C, b.C)}
}

func (f *Function) TailCall(target *Function, values ...*Value) *Value {
	args := make([]C.jit_value_t, len(values))
	for i := range values {
		args[i] = values[i].C
	}
	return &Value{C.jit_insn_call(f.C,
		C.CString("noname"),
		target.C, nil, (*C.jit_value_t)(&args[0]), C.uint(len(args)), C.JIT_CALL_TAIL)}
}

func (f *Function) Call(target *Function, values []*Value) *Value {
	args := make([]C.jit_value_t, len(values))
	for i := range values {
		args[i] = values[i].C
	}
	return &Value{C.jit_insn_call(f.C,
		C.CString("noname"),
		target.C, nil, (*C.jit_value_t)(&args[0]), C.uint(len(args)), C.int(0))}
}

func (f *Function) Call0(name string, target *Function, values ...*Value) *Value {
	args := make([]C.jit_value_t, len(values))
	for i := range values {
		args[i] = values[i].C
	}
	return &Value{C.jit_insn_call(f.C,
		C.CString(name),
		target.C, nil, (*C.jit_value_t)(&args[0]), C.uint(len(args)), C.int(0))}
}

func (f *Function) CallNative(name string, target unsafe.Pointer, sig *Signature, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call_native(f.C,
			C.CString(name),
			target,
			sig.C, (*C.jit_value_t)(nil),
			C.uint(0), C.JIT_CALL_NOTHROW)}
	} else {
		args := make([]C.jit_value_t, len(values))
		for i := range values {
			args[i] = values[i].C
		}
		return &Value{C.jit_insn_call_native(f.C,
			C.CString(name),
			target,
			sig.C, (*C.jit_value_t)(&args[0]),
			C.uint(len(args)), C.JIT_CALL_NOTHROW)}
	}
}

func (f *Function) Compile() {
	C.jit_function_compile(f.C)
}

func (f *Function) Run(values ...interface{}) interface{} {
	args := make([]unsafe.Pointer, len(values))
	for i := range values {
		switch f.paramType[i].C {
		case C.jit_type_int:
			x := C.jit_int(values[i].(int))
			args[i] = (unsafe.Pointer)(&x)

		case C.jit_type_uint:
			x := C.jit_uint(values[i].(uint))
			args[i] = (unsafe.Pointer)(&x)

		case C.jit_type_void_ptr:
			x := (unsafe.Pointer)(reflect.ValueOf(values[i]).Pointer())
			args[i] = (unsafe.Pointer)(&x)
		}
	}

	switch f.retType.C {
	case C.jit_type_int:
		result := C.jit_int(0)
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(&result))
		return int(result)

	case C.jit_type_uint:
		result := C.jit_uint(0)
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(&result))
		return uint(result)

	case C.jit_type_void:
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(nil))
		return nil

	}

	return nil
}

func (f *Function) Dump(name string) {
	C.jit_dump_function(C.getStdout(), f.C, C.CString(name))
}

func (f *Function) SetRecompilable() {
	C.jit_function_set_recompilable(f.C)
}

type compileFunction struct {
	F           *Function
	compileFunc func(*Function) bool
}

var registry = make(map[C.jit_function_t]*compileFunction)

func (f *Function) SetOnDemandCompiler(function func(f *Function) bool) {
	registry[f.C] = &compileFunction{f, function}
	C.SetOnDemandCompileFunction(f.C)
}

func (f *Function) GetOnDemandCompiler() func(*Function) bool {
	return registry[f.C].compileFunc
}

// ========== Function =============

//export on_demand_compile
func on_demand_compile(f C.jit_function_t) C.int {
	cf := registry[f]
	result := cf.compileFunc(cf.F)
	if result {
		return C.int(1)
	}
	return C.int(0)
}
<commit_msg>add platform selection to CGO<commit_after>package jit

/*
#cgo CFLAGS: -I./include
#cgo linux,amd64 LDFLAGS: -L./lib -lm -ljit

#include <stdio.h>
#include <jit/jit.h>
#include <jit/jit-dump.h>

extern int on_demand_compile(jit_function_t);

static void SetOnDemandCompileFunction(jit_function_t f) {
    jit_function_set_on_demand_compiler(f, on_demand_compile);
}

static FILE *getStdout(void) { return stdout; }
*/
import "C"
import "unsafe"
import "reflect"

const CDECL = C.jit_abi_cdecl

type Context struct {
	C C.jit_context_t
}

type Type struct {
	C C.jit_type_t
}

type Signature struct {
	C C.jit_type_t
}

type Function struct {
	C         C.jit_function_t
	retType   Type
	paramType []Type
}

type Value struct {
	C C.jit_value_t
}

type Label struct {
	C C.jit_label_t
}

func Int() Type {
	return Type{C.jit_type_int}
}

func UInt() Type {
	return Type{C.jit_type_uint}
}

func Void() Type {
	return Type{C.jit_type_void}
}

func VoidPtr() Type {
	return Type{C.jit_type_void_ptr}
}

func NewContext() *Context {
	return &Context{C.jit_context_create()}
}

func NewSignature(ret Type, params []Type) *Signature {
	if len(params) == 0 {
		signature := C.jit_type_create_signature(C.jit_abi_cdecl,
			ret.C,
			(*C.jit_type_t)(nil),
			C.uint(0), 1)
		return &Signature{signature}
	} else {
		signature := C.jit_type_create_signature(C.jit_abi_cdecl,
			ret.C,
			(*C.jit_type_t)(unsafe.Pointer(&params[0])),
			C.uint(len(params)), 1)
		return &Signature{signature}
	}
}

func NewLabel() *Label {
	return &Label{C.jit_label_undefined}
}

// ========== Context =============

func (c *Context) NewFunction(ret Type, params []Type) *Function {
	signature := NewSignature(ret, params)
	function := C.jit_function_create(c.C, signature.C)
	C.jit_type_free(signature.C)
	return &Function{C: function, retType: ret, paramType: params}
}

func (c *Context) BuildStart() {
	C.jit_context_build_start(c.C)
}

func (c *Context) BuildEnd() {
	C.jit_context_build_end(c.C)
}

func (c *Context) Destroy() {
	C.jit_context_destroy(c.C)
}

// ========== Context =============

// ========== Function =============

func (f *Function) Param(i int) *Value {
	return &Value{C.jit_value_get_param(f.C, C.uint(i))}
}

func (f *Function) Param2() (*Value, *Value) {
	return &Value{C.jit_value_get_param(f.C, C.uint(0))},
		&Value{C.jit_value_get_param(f.C, C.uint(1))}
}

func (f *Function) Param3() (*Value, *Value, *Value) {
	return &Value{C.jit_value_get_param(f.C, C.uint(0))},
		&Value{C.jit_value_get_param(f.C, C.uint(1))},
		&Value{C.jit_value_get_param(f.C, C.uint(2))}
}

func (f *Function) Mul(a, b *Value) *Value {
	return &Value{C.jit_insn_mul(f.C, a.C, b.C)}
}

func (f *Function) Add(a, b *Value) *Value {
	return &Value{C.jit_insn_add(f.C, a.C, b.C)}
}

func (f *Function) Sub(a, b *Value) *Value {
	return &Value{C.jit_insn_sub(f.C, a.C, b.C)}
}

func (f *Function) Return(ret *Value) {
	C.jit_insn_return(f.C, ret.C)
}

func (f *Function) Store(x, y *Value) {
	C.jit_insn_store(f.C, x.C, y.C)
}

func (f *Function) StoreRelative(x *Value, offset int, y *Value) {
	C.jit_insn_store_relative(f.C, x.C, C.jit_nint(offset), y.C)
}

func (f *Function) Eq(a, b *Value) *Value {
	return &Value{C.jit_insn_eq(f.C, a.C, b.C)}
}

func (f *Function) BranchIfNot(v *Value, label *Label) {
	C.jit_insn_branch_if_not(f.C, v.C, (*C.jit_label_t)(unsafe.Pointer(label)))
}

func (f *Function) Label(label *Label) {
	C.jit_insn_label(f.C, (*C.jit_label_t)(unsafe.Pointer(label)))
}

func (f *Function) LessThan(a, b *Value) *Value {
	return &Value{C.jit_insn_lt(f.C, a.C, b.C)}
}

func (f *Function) TailCall(target *Function, values ...*Value) *Value {
	args := make([]C.jit_value_t, len(values))
	for i := range values {
		args[i] = values[i].C
	}
	return &Value{C.jit_insn_call(f.C,
		C.CString("noname"),
		target.C, nil, (*C.jit_value_t)(&args[0]), C.uint(len(args)), C.JIT_CALL_TAIL)}
}

func (f *Function) Call(target *Function, values []*Value) *Value {
	args := make([]C.jit_value_t, len(values))
	for i := range values {
		args[i] = values[i].C
	}
	return &Value{C.jit_insn_call(f.C,
		C.CString("noname"),
		target.C, nil, (*C.jit_value_t)(&args[0]), C.uint(len(args)), C.int(0))}
}

func (f *Function) Call0(name string, target *Function, values ...*Value) *Value {
	args := make([]C.jit_value_t, len(values))
	for i := range values {
		args[i] = values[i].C
	}
	return &Value{C.jit_insn_call(f.C,
		C.CString(name),
		target.C, nil, (*C.jit_value_t)(&args[0]), C.uint(len(args)), C.int(0))}
}

func (f *Function) CallNative(name string, target unsafe.Pointer, sig *Signature, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call_native(f.C,
			C.CString(name),
			target,
			sig.C, (*C.jit_value_t)(nil),
			C.uint(0), C.JIT_CALL_NOTHROW)}
	} else {
		args := make([]C.jit_value_t, len(values))
		for i := range values {
			args[i] = values[i].C
		}
		return &Value{C.jit_insn_call_native(f.C,
			C.CString(name),
			target,
			sig.C, (*C.jit_value_t)(&args[0]),
			C.uint(len(args)), C.JIT_CALL_NOTHROW)}
	}
}

func (f *Function) Compile() {
	C.jit_function_compile(f.C)
}

func (f *Function) Run(values ...interface{}) interface{} {
	args := make([]unsafe.Pointer, len(values))
	for i := range values {
		switch f.paramType[i].C {
		case C.jit_type_int:
			x := C.jit_int(values[i].(int))
			args[i] = (unsafe.Pointer)(&x)

		case C.jit_type_uint:
			x := C.jit_uint(values[i].(uint))
			args[i] = (unsafe.Pointer)(&x)

		case C.jit_type_void_ptr:
			x := (unsafe.Pointer)(reflect.ValueOf(values[i]).Pointer())
			args[i] = (unsafe.Pointer)(&x)
		}
	}

	switch f.retType.C {
	case C.jit_type_int:
		result := C.jit_int(0)
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(&result))
		return int(result)

	case C.jit_type_uint:
		result := C.jit_uint(0)
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(&result))
		return uint(result)

	case C.jit_type_void:
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(nil))
		return nil

	}

	return nil
}

func (f *Function) Dump(name string) {
	C.jit_dump_function(C.getStdout(), f.C, C.CString(name))
}

func (f *Function) SetRecompilable() {
	C.jit_function_set_recompilable(f.C)
}

type compileFunction struct {
	F           *Function
	compileFunc func(*Function) bool
}

var registry = make(map[C.jit_function_t]*compileFunction)

func (f *Function) SetOnDemandCompiler(function func(f *Function) bool) {
	registry[f.C] = &compileFunction{f, function}
	C.SetOnDemandCompileFunction(f.C)
}

func (f *Function) GetOnDemandCompiler() func(*Function) bool {
	return registry[f.C].compileFunc
}

// ========== Function =============

//export on_demand_compile
func on_demand_compile(f C.jit_function_t) C.int {
	cf := registry[f]
	result := cf.compileFunc(cf.F)
	if result {
		return C.int(1)
	}
	return C.int(0)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/groovenauts/blocks-variable"

	log "github.com/Sirupsen/logrus"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *JobMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}

		cmd *exec.Cmd
	}
)

func (job *Job) run() error {
	err := job.runWithoutErrorHandling()
	switch err.(type) {
	case RetryableError:
		var f func() error
		e := err.(RetryableError)
		if e.Retryable() {
			f = job.withNotify(NACKSENDING, job.message.Nack)
		} else {
			f = job.withNotify(CANCELLING, job.message.Ack)
		}
		err := f()
		if err != nil {
			return err
		}
	}
	return err
}

func (job *Job) runWithoutErrorHandling() error {
	defer job.withNotify(CLEANUP, job.clearWorkspace)() // Call clearWorkspace even if setupWorkspace retuns error

	err := job.withNotify(INITIALIZING, job.prepare)()
	if err != nil {
		return err
	}

	go job.message.sendMADPeriodically(job.notification)
	defer job.message.Done()

	err = job.withNotify(DOWNLOADING, job.downloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(EXECUTING, job.execute)()
	if err != nil {
		return err
	}

	err = job.withNotify(UPLOADING, job.uploadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(ACKSENDING, job.message.Ack)()
	if err != nil {
		return err
	}

	return err
}

func (job *Job) withNotify(step JobStep, f func() error) func() error {
	return job.notification.wrap(job.message.MessageId(), step, f)
}

func (job *Job) prepare() error {
	logAttrs := log.Fields{"job_message_id": job.message.MessageId()}
	err := job.message.Validate()
	if err != nil {
		logAttrs["message"] = job.message.raw.Message
		logAttrs["error"] = err
		log.WithFields(logAttrs).Errorf("Invalid Message")
		return err
	}

	err = job.setupWorkspace()
	if err != nil {
		return err
	}

	err = job.useDataAsAttributesIfPossible()
	if err != nil {
		return err
	}

	err = job.setupDownloadFiles()
	if err != nil {
		return err
	}

	err = job.build()
	if err != nil {
		logAttrs["template"] = job.config.Template
		logAttrs["message"] = job.message
		logAttrs["error"] = err
		log.WithFields(logAttrs).Errorf("Failed to build command")
		return err
	}
	return nil
}

func (job *Job) setupWorkspace() error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return nil
}

func (job *Job) clearWorkspace() error {
	if job.workspace != "" {
		return os.RemoveAll(job.workspace)
	}
	return nil
}

const UseDataAsAttributesKey = "use-data-as-attributes"

var UseDataAsAttributesRegexp = regexp.MustCompile("(?i)true|yes|on|1")

func (job *Job) useDataAsAttributesIfPossible() error {
	attrs := job.message.raw.Message.Attributes
	flg := attrs[UseDataAsAttributesKey]
	if !UseDataAsAttributesRegexp.MatchString(flg) {
		return nil
	}
	decoded, err := base64.StdEncoding.DecodeString(job.message.raw.Message.Data)
	if err != nil {
		logAttrs := log.Fields{"error": err, "data": job.message.raw.Message.Data}
		log.WithFields(logAttrs).Errorf("Failed to decode by base64")
		return err
	}
	var parsed map[string]interface{}
	err = json.Unmarshal([]byte(decoded), &parsed)
	if err != nil {
		logAttrs := log.Fields{"error": err, "data": job.message.raw.Message.Data}
		log.WithFields(logAttrs).Errorf("Failed to json.Unmarshal")
		return err
	}
	for key, obj := range parsed {
		var value string
		switch obj.(type) {
		case string:
			value = obj.(string)
		default:
			b, err := json.Marshal(obj)
			if err != nil {
				logAttrs := log.Fields{"error": err, "obj": obj}
				log.WithFields(logAttrs).Errorf("Failed to json.Marshal")
				return err
			}
			value = string(b)
		}
		attrs[key] = value
	}
	return nil
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.message.DownloadFiles()
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.WithFields(log.Fields{"url": obj}).Errorf("Invalid download file URL: %T\n", obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.WithFields(log.Fields{"url": remote_url}).Errorln("Invalid download file URL")
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *bvariable.Variable {
	return &bvariable.Variable{
		Data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.raw.Message.Attributes,
			"attributes":            job.message.raw.Message.Attributes,
			"data":                  job.message.raw.Message.Data,
		},
	}
}

func (job *Job) build() error {
	logAttrs := log.Fields{}
	v := job.buildVariable()
	values, err := job.extract(v, job.config.Template)
	if len(job.config.Options) > 0 {
		logAttrs["options_key_template"] = job.config.Template
		logAttrs["options_key_base"] = values
		log.WithFields(logAttrs).Debugln("extracting key of options")
		if err != nil {
			logAttrs["error"] = err
			switch err.(type) {
			case NestableError:
				ne := err.(NestableError)
				if ne.CausedBy((*bvariable.InvalidExpression)(nil)) {
					log.WithFields(logAttrs).Warnln("Invalid Expression to extract")
					values = []string{}
				} else {
					log.WithFields(logAttrs).Errorln("extract error")
					return err
				}
			default:
				log.WithFields(logAttrs).Errorln("extract error")
				return err
			}
		}
		key := strings.Join(values, " ")
		if key == "" {
			key = "default"
		}
		t := job.config.Options[key]
		logAttrs["options_key"] = key
		logAttrs["command_template"] = t
		log.WithFields(logAttrs).Debugln("extracting command of options")
		if t == nil {
			msg := fmt.Sprintf("Invalid command options key: %q", key)
			log.WithFields(logAttrs).Errorln(msg)
			return &InvalidJobError{msg: msg}
		}
		values, err = job.extract(v, t)
		if err != nil {
			logAttrs["error"] = err
			log.WithFields(logAttrs).Errorln("extract error")
			return err
		}
	} else {
		logAttrs["command_template"] = job.config.Template
		if err != nil {
			logAttrs["error"] = err
			log.WithFields(logAttrs).Errorln("extract error")
			return err
		}
	}
	logAttrs["command"] = values
	log.WithFields(logAttrs).Debugln("")
	job.cmd = exec.Command(values[0], values[1:]...)
	return nil
}

func (job *Job) extract(v *bvariable.Variable, values []string) ([]string, error) {
	result := []string{}
	errors := []error{}
	for _, src := range values {
		extracted, err := v.Expand(src)
		err = job.convertError(err)
		if err != nil {
			errors = append(errors, &InvalidJobError{cause: err})
			continue
		}
		vals := strings.Split(extracted, v.Separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	if len(errors) > 0 {
		return nil, &CompositeError{errors}
	}
	return result, nil
}

func (job *Job) convertError(src error) error {
	switch src.(type) {
	case *bvariable.Errors:
		err := src.(*bvariable.Errors)
		return &CompositeError{[]error(*err)}
	default:
		return src
	}
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.WithFields(log.Fields{"url": remoteURL, "error": err}).Errorln("Invalid URL")
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	var out bytes.Buffer
	job.cmd.Stdout = &out
	job.cmd.Stderr = &out
	log.WithFields(log.Fields{"cmd": job.cmd}).Debugln("EXECUTING")
	err := job.cmd.Run()
	if err != nil {
		log.WithFields(log.Fields{"cmd": job.cmd, "error": err, "outputs": out.String()}).Errorln("Command returned error")
		return err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	targets := make(chan *Target)
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.WithFields(log.Fields{"error": err, "localPath": localPath, "uploads_dir": job.uploads_dir}).Errorln("Failed to get relative path")
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		t := &Target{
			Bucket:    parts[0],
			Object:    strings.Join(parts[1:], sep),
			LocalPath: localPath,
		}
		targets <- t
	}

	uploaders := TargetWorkers{}
	for i := 0; i < 3; i++ {
		uploader := &TargetWorker{
			name:    "upload",
			targets: targets,
			impl:    job.storage.Upload,
		}
		uploaders = append(uploaders, uploader)
	}

	err = uploaders.runAndWait()
	return err
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.WithFields(log.Fields{"error": err}).Errorln("Error to list upload files")
		return nil, err
	}
	return result, nil
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<commit_msg>:+1: Add Uploaders to CommandConfig<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/groovenauts/blocks-variable"

	log "github.com/Sirupsen/logrus"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
		Uploaders int                `json:"uploaders,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *JobMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}

		cmd *exec.Cmd
	}
)

func (job *Job) run() error {
	err := job.runWithoutErrorHandling()
	switch err.(type) {
	case RetryableError:
		var f func() error
		e := err.(RetryableError)
		if e.Retryable() {
			f = job.withNotify(NACKSENDING, job.message.Nack)
		} else {
			f = job.withNotify(CANCELLING, job.message.Ack)
		}
		err := f()
		if err != nil {
			return err
		}
	}
	return err
}

func (job *Job) runWithoutErrorHandling() error {
	defer job.withNotify(CLEANUP, job.clearWorkspace)() // Call clearWorkspace even if setupWorkspace retuns error

	err := job.withNotify(INITIALIZING, job.prepare)()
	if err != nil {
		return err
	}

	go job.message.sendMADPeriodically(job.notification)
	defer job.message.Done()

	err = job.withNotify(DOWNLOADING, job.downloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(EXECUTING, job.execute)()
	if err != nil {
		return err
	}

	err = job.withNotify(UPLOADING, job.uploadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(ACKSENDING, job.message.Ack)()
	if err != nil {
		return err
	}

	return err
}

func (job *Job) withNotify(step JobStep, f func() error) func() error {
	return job.notification.wrap(job.message.MessageId(), step, f)
}

func (job *Job) prepare() error {
	logAttrs := log.Fields{"job_message_id": job.message.MessageId()}
	err := job.message.Validate()
	if err != nil {
		logAttrs["message"] = job.message.raw.Message
		logAttrs["error"] = err
		log.WithFields(logAttrs).Errorf("Invalid Message")
		return err
	}

	err = job.setupWorkspace()
	if err != nil {
		return err
	}

	err = job.useDataAsAttributesIfPossible()
	if err != nil {
		return err
	}

	err = job.setupDownloadFiles()
	if err != nil {
		return err
	}

	err = job.build()
	if err != nil {
		logAttrs["template"] = job.config.Template
		logAttrs["message"] = job.message
		logAttrs["error"] = err
		log.WithFields(logAttrs).Errorf("Failed to build command")
		return err
	}
	return nil
}

func (job *Job) setupWorkspace() error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return nil
}

func (job *Job) clearWorkspace() error {
	if job.workspace != "" {
		return os.RemoveAll(job.workspace)
	}
	return nil
}

const UseDataAsAttributesKey = "use-data-as-attributes"

var UseDataAsAttributesRegexp = regexp.MustCompile("(?i)true|yes|on|1")

func (job *Job) useDataAsAttributesIfPossible() error {
	attrs := job.message.raw.Message.Attributes
	flg := attrs[UseDataAsAttributesKey]
	if !UseDataAsAttributesRegexp.MatchString(flg) {
		return nil
	}
	decoded, err := base64.StdEncoding.DecodeString(job.message.raw.Message.Data)
	if err != nil {
		logAttrs := log.Fields{"error": err, "data": job.message.raw.Message.Data}
		log.WithFields(logAttrs).Errorf("Failed to decode by base64")
		return err
	}
	var parsed map[string]interface{}
	err = json.Unmarshal([]byte(decoded), &parsed)
	if err != nil {
		logAttrs := log.Fields{"error": err, "data": job.message.raw.Message.Data}
		log.WithFields(logAttrs).Errorf("Failed to json.Unmarshal")
		return err
	}
	for key, obj := range parsed {
		var value string
		switch obj.(type) {
		case string:
			value = obj.(string)
		default:
			b, err := json.Marshal(obj)
			if err != nil {
				logAttrs := log.Fields{"error": err, "obj": obj}
				log.WithFields(logAttrs).Errorf("Failed to json.Marshal")
				return err
			}
			value = string(b)
		}
		attrs[key] = value
	}
	return nil
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.message.DownloadFiles()
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.WithFields(log.Fields{"url": obj}).Errorf("Invalid download file URL: %T\n", obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.WithFields(log.Fields{"url": remote_url}).Errorln("Invalid download file URL")
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *bvariable.Variable {
	return &bvariable.Variable{
		Data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.raw.Message.Attributes,
			"attributes":            job.message.raw.Message.Attributes,
			"data":                  job.message.raw.Message.Data,
		},
	}
}

func (job *Job) build() error {
	logAttrs := log.Fields{}
	v := job.buildVariable()
	values, err := job.extract(v, job.config.Template)
	if len(job.config.Options) > 0 {
		logAttrs["options_key_template"] = job.config.Template
		logAttrs["options_key_base"] = values
		log.WithFields(logAttrs).Debugln("extracting key of options")
		if err != nil {
			logAttrs["error"] = err
			switch err.(type) {
			case NestableError:
				ne := err.(NestableError)
				if ne.CausedBy((*bvariable.InvalidExpression)(nil)) {
					log.WithFields(logAttrs).Warnln("Invalid Expression to extract")
					values = []string{}
				} else {
					log.WithFields(logAttrs).Errorln("extract error")
					return err
				}
			default:
				log.WithFields(logAttrs).Errorln("extract error")
				return err
			}
		}
		key := strings.Join(values, " ")
		if key == "" {
			key = "default"
		}
		t := job.config.Options[key]
		logAttrs["options_key"] = key
		logAttrs["command_template"] = t
		log.WithFields(logAttrs).Debugln("extracting command of options")
		if t == nil {
			msg := fmt.Sprintf("Invalid command options key: %q", key)
			log.WithFields(logAttrs).Errorln(msg)
			return &InvalidJobError{msg: msg}
		}
		values, err = job.extract(v, t)
		if err != nil {
			logAttrs["error"] = err
			log.WithFields(logAttrs).Errorln("extract error")
			return err
		}
	} else {
		logAttrs["command_template"] = job.config.Template
		if err != nil {
			logAttrs["error"] = err
			log.WithFields(logAttrs).Errorln("extract error")
			return err
		}
	}
	logAttrs["command"] = values
	log.WithFields(logAttrs).Debugln("")
	job.cmd = exec.Command(values[0], values[1:]...)
	return nil
}

func (job *Job) extract(v *bvariable.Variable, values []string) ([]string, error) {
	result := []string{}
	errors := []error{}
	for _, src := range values {
		extracted, err := v.Expand(src)
		err = job.convertError(err)
		if err != nil {
			errors = append(errors, &InvalidJobError{cause: err})
			continue
		}
		vals := strings.Split(extracted, v.Separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	if len(errors) > 0 {
		return nil, &CompositeError{errors}
	}
	return result, nil
}

func (job *Job) convertError(src error) error {
	switch src.(type) {
	case *bvariable.Errors:
		err := src.(*bvariable.Errors)
		return &CompositeError{[]error(*err)}
	default:
		return src
	}
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.WithFields(log.Fields{"url": remoteURL, "error": err}).Errorln("Invalid URL")
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	var out bytes.Buffer
	job.cmd.Stdout = &out
	job.cmd.Stderr = &out
	log.WithFields(log.Fields{"cmd": job.cmd}).Debugln("EXECUTING")
	err := job.cmd.Run()
	if err != nil {
		log.WithFields(log.Fields{"cmd": job.cmd, "error": err, "outputs": out.String()}).Errorln("Command returned error")
		return err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	targets := make(chan *Target)
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.WithFields(log.Fields{"error": err, "localPath": localPath, "uploads_dir": job.uploads_dir}).Errorln("Failed to get relative path")
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		t := &Target{
			Bucket:    parts[0],
			Object:    strings.Join(parts[1:], sep),
			LocalPath: localPath,
		}
		targets <- t
	}

	if job.config.Uploaders < 1 {
		job.config.Uploaders = 1
	}
	uploaders := TargetWorkers{}
	for i := 0; i < job.config.Uploaders; i++ {
		uploader := &TargetWorker{
			name:    "upload",
			targets: targets,
			impl:    job.storage.Upload,
		}
		uploaders = append(uploaders, uploader)
	}

	err = uploaders.runAndWait()
	return err
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.WithFields(log.Fields{"error": err}).Errorln("Error to list upload files")
		return nil, err
	}
	return result, nil
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<|endoftext|>


<commit_before>package handling

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/gorilla/mux"
	"github.com/victorspringer/trapAdvisor/database"
	ta "github.com/victorspringer/trapAdvisor/touristattraction"
	"github.com/victorspringer/trapAdvisor/trip"
)

func (s *service) Health(w http.ResponseWriter, r *http.Request) {
	_, err := s.travRepo.Find(0)
	if err != nil {
		if err.Error() == "Error 1046: No database selected" {
			database.DB.Close()
			database.DB, err = database.Init(os.Getenv("ENV"))
			if err != nil {
				fmt.Fprintf(w, "Database connection error")
				return
			}
		} else if err.Error() != "sql: no rows in result set" {
			fmt.Fprintf(w, "Database connection error")
			return
		}
	}

	fmt.Fprintf(w, "OK")
}

func (s *service) StoreTrip(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	var t trip.Trip

	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if err = r.Body.Close(); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if err = json.Unmarshal(body, &t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusUnprocessableEntity)
		return
	}

	if err = s.tripRepo.Store(&t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusCreated)
}

func (s *service) StoreTouristAttraction(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	var t ta.TouristAttraction

	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if err = r.Body.Close(); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if err = json.Unmarshal(body, &t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusUnprocessableEntity)
		return
	}

	if err = s.taRepo.Store(&t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusCreated)
}

func (s *service) FindTraveller(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.travRepo.Find(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindFriendshipByTravellerID(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.fRepo.FindByTravellerID(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTrip(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.tripRepo.Find(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTripByTravellerID(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.tripRepo.FindByTravellerID(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTouristAttraction(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.taRepo.Find(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTouristAttractionByTripID(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.taRepo.FindByTripID(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTouristAttractionByNamePart(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	namePart := mux.Vars(r)["namePart"]

	t, err := s.taRepo.FindByNamePart(namePart)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindMostVisitedTouristAttractions(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	t, err := s.taRepo.FindMostVisited()
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindBestRatedTouristAttractions(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	t, err := s.taRepo.FindBestRated()
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}
<commit_msg>returns stored trip and ta ids<commit_after>package handling

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/gorilla/mux"
	"github.com/victorspringer/trapAdvisor/database"
	ta "github.com/victorspringer/trapAdvisor/touristattraction"
	"github.com/victorspringer/trapAdvisor/trip"
)

func (s *service) Health(w http.ResponseWriter, r *http.Request) {
	_, err := s.travRepo.Find(0)
	if err != nil {
		if err.Error() == "Error 1046: No database selected" {
			database.DB.Close()
			database.DB, err = database.Init(os.Getenv("ENV"))
			if err != nil {
				fmt.Fprintf(w, "Database connection error")
				return
			}
		} else if err.Error() != "sql: no rows in result set" {
			fmt.Fprintf(w, "Database connection error")
			return
		}
	}

	fmt.Fprintf(w, "OK")
}

func (s *service) StoreTrip(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	var t trip.Trip

	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if err = r.Body.Close(); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if err = json.Unmarshal(body, &t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusUnprocessableEntity)
		return
	}

	if err = s.tripRepo.Store(&t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	var tripID string
	if t.ID == 0 {
		trips, err := s.tripRepo.FindByTravellerID(t.TravellerID)
		if err != nil {
			log.Println(err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		tripID = strconv.Itoa(trips[len(trips)-1].ID)
	} else {
		tripID = strconv.Itoa(t.ID)
	}

	w.WriteHeader(http.StatusCreated)
	w.Write([]byte(tripID))
}

func (s *service) StoreTouristAttraction(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	var t ta.TouristAttraction

	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if err = r.Body.Close(); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if err = json.Unmarshal(body, &t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusUnprocessableEntity)
		return
	}

	if err = s.taRepo.Store(&t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	tas, err := s.taRepo.FindByTripID(t.TripID)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	taID := strconv.Itoa(tas[len(tas)-1].ID)

	w.WriteHeader(http.StatusCreated)
	w.Write([]byte(taID))
}

func (s *service) FindTraveller(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.travRepo.Find(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindFriendshipByTravellerID(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.fRepo.FindByTravellerID(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTrip(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.tripRepo.Find(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTripByTravellerID(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.tripRepo.FindByTravellerID(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTouristAttraction(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.taRepo.Find(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTouristAttractionByTripID(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	id, err := strconv.Atoi(mux.Vars(r)["id"])
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	t, err := s.taRepo.FindByTripID(id)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindTouristAttractionByNamePart(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	namePart := mux.Vars(r)["namePart"]

	t, err := s.taRepo.FindByNamePart(namePart)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindMostVisitedTouristAttractions(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	t, err := s.taRepo.FindMostVisited()
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (s *service) FindBestRatedTouristAttractions(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	t, err := s.taRepo.FindBestRated()
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusOK)
}
<|endoftext|>