<commit_before>/**
 * Copyright 2017 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package mounter

import (
	"fmt"
	"os"

	"github.com/IBM/ubiquity/remote/mounter/block_device_mounter_utils"
	"github.com/IBM/ubiquity/remote/mounter/block_device_utils"
	"github.com/IBM/ubiquity/resources"
	"github.com/IBM/ubiquity/utils"
	"github.com/IBM/ubiquity/utils/logs"
)

type scbeMounter struct {
	logger                  logs.Logger
	blockDeviceMounterUtils block_device_mounter_utils.BlockDeviceMounterUtils
	exec                    utils.Executor
}

func newScbMounter(blockDeviceMounterUtils block_device_mounter_utils.BlockDeviceMounterUtils, executer utils.Executor) resources.Mounter {
	return &scbeMounter{
		logger:                  logs.GetLogger(),
		blockDeviceMounterUtils: blockDeviceMounterUtils,
		exec:                    executer,
	}
}

func NewScbeMounter() resources.Mounter {
	blockDeviceMounterUtils := block_device_mounter_utils.NewBlockDeviceMounterUtils()
	return newScbMounter(blockDeviceMounterUtils, utils.NewExecutor())
}

func NewScbeMounterWithExecuter(blockDeviceMounterUtils block_device_mounter_utils.BlockDeviceMounterUtils, executer utils.Executor) resources.Mounter {
	return newScbMounter(blockDeviceMounterUtils, executer)
}

func (s *scbeMounter) prepareVolumeMountProperties(vcGetter resources.VolumeConfigGetter) *resources.VolumeMountProperties {
	volumeConfig := vcGetter.GetVolumeConfig()
	volumeWWN := volumeConfig["Wwn"].(string)
	volumeLunNumber := -1
	if volumeLunNumberInterface, exists := volumeConfig[resources.ScbeKeyVolAttachLunNumToHost]; exists {
		volumeLunNumber = volumeLunNumberInterface.(int)
	}
	return &resources.VolumeMountProperties{WWN: volumeWWN, LunNumber: volumeLunNumber}
}

func (s *scbeMounter) Mount(mountRequest resources.MountRequest) (string, error) {
	defer s.logger.Trace(logs.DEBUG)()
	volumeMountProperties := s.prepareVolumeMountProperties(mountRequest)

	// Rescan OS
	if err := s.blockDeviceMounterUtils.RescanAll(volumeMountProperties); err != nil {
		return "", s.logger.ErrorRet(err, "RescanAll failed")
	}

	// Discover device
	devicePath, err := s.blockDeviceMounterUtils.Discover(volumeMountProperties.WWN, true)
	if err != nil {
		return "", s.logger.ErrorRet(err, "Discover failed", logs.Args{{"volumeWWN", volumeMountProperties.WWN}})
	}

	// Create mount point if needed   // TODO consider to move it inside the util
	if _, err := s.exec.Stat(mountRequest.Mountpoint); err != nil {
		s.logger.Info("Create mountpoint directory " + mountRequest.Mountpoint)
		if err := s.exec.MkdirAll(mountRequest.Mountpoint, 0700); err != nil {
			return "", s.logger.ErrorRet(err, "MkdirAll failed", logs.Args{{"mountpoint", mountRequest.Mountpoint}})
		}
	} else {
		s.logger.Warning("Idempotent issue : mount point directory already exists", logs.Args{{"mountpoint", mountRequest.Mountpoint}})
	}

	// Mount device and mkfs if needed
	var fstype string
	fstypeInterface, ok := mountRequest.VolumeConfig[resources.OptionNameForVolumeFsType]
	if !ok {
		// the backend should do this default, but this is just for safe
		fstype = resources.DefaultForScbeConfigParamDefaultFilesystem
	} else {
		fstype = fstypeInterface.(string)
	}

	if err := s.blockDeviceMounterUtils.MountDeviceFlow(devicePath, fstype, mountRequest.Mountpoint); err != nil {
		return "", s.logger.ErrorRet(err, "MountDeviceFlow failed", logs.Args{{"devicePath", devicePath}})
	}

	return mountRequest.Mountpoint, nil
}

func (s *scbeMounter) Unmount(unmountRequest resources.UnmountRequest) error {
	defer s.logger.Trace(logs.DEBUG)()

	volumeWWN := unmountRequest.VolumeConfig["Wwn"].(string)
	mountpoint := fmt.Sprintf(resources.PathToMountUbiquityBlockDevices, volumeWWN) // TODO instead of build the mountpoint it should come from unmountRequest.
	skipUnmountFlow := false
	devicePath, err := s.blockDeviceMounterUtils.Discover(volumeWWN, true)
	if err != nil {
		switch err.(type) {
		case *block_device_utils.VolumeNotFoundError:
			s.logger.Warning("Idempotent issue encountered: volume not found. skipping UnmountDeviceFlow ", logs.Args{{"volumeWWN", volumeWWN}})
			skipUnmountFlow = true
		case *block_device_utils.FaultyDeviceError:
			s.logger.Warning("Idempotent issue encountered: mpath device is faulty. continuing with UnmountDeviceFlow", logs.Args{{"device", devicePath}, {"volumeWWN", volumeWWN}})
		default:
			return s.logger.ErrorRet(err, "Discover failed", logs.Args{{"volumeWWN", volumeWWN}})
		}
	}

	if !skipUnmountFlow {
		if err := s.blockDeviceMounterUtils.UnmountDeviceFlow(devicePath, volumeWWN); err != nil {
			return s.logger.ErrorRet(err, "UnmountDeviceFlow failed", logs.Args{{"devicePath", devicePath}})
		}
	}

	s.logger.Info("Delete mountpoint directory if exist", logs.Args{{"mountpoint", mountpoint}})
	// TODO move this part to the util
	if _, err := s.exec.Stat(mountpoint); err == nil {
		s.logger.Debug("Checking if mountpoint is empty and can be deleted", logs.Args{{"mountpoint", mountpoint}})
		emptyDir, err := s.exec.IsDirEmpty(mountpoint)
		if err != nil {
			return s.logger.ErrorRet(err, "Getting number of files failed.", logs.Args{{"mountpoint", mountpoint}})
		}
		if emptyDir != true {
			return s.logger.ErrorRet(&DirecotryIsNotEmptyError{mountpoint}, "Directory is not empty and cannot be removed.", logs.Args{{"mountpoint", mountpoint}})
		}

		if err := s.exec.RemoveAll(mountpoint); err != nil { // TODO its enough to do Remove without All.
			return s.logger.ErrorRet(err, "RemoveAll failed", logs.Args{{"mountpoint", mountpoint}})
		}
	} else {
		if os.IsNotExist(err) {
			s.logger.Warning("Idempotent issue encountered: mountpoint directory does not exist.", logs.Args{{"mountpoint", mountpoint}})
		}
	}

	return nil

}

func (s *scbeMounter) ActionAfterDetach(request resources.AfterDetachRequest) error {
	defer s.logger.Trace(logs.DEBUG)()
	volumeMountProperties := s.prepareVolumeMountProperties(request)

	// Disconnect volume
	if err := s.blockDeviceMounterUtils.DisconnectAll(volumeMountProperties); err != nil {
		return s.logger.ErrorRet(err, "DisconnectAll failed")
	}
	return nil
}
<commit_msg>fixed a defect<commit_after>/**
 * Copyright 2017 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package mounter

import (
	"fmt"
	"os"

	"github.com/IBM/ubiquity/remote/mounter/block_device_mounter_utils"
	"github.com/IBM/ubiquity/remote/mounter/block_device_utils"
	"github.com/IBM/ubiquity/resources"
	"github.com/IBM/ubiquity/utils"
	"github.com/IBM/ubiquity/utils/logs"
)

type scbeMounter struct {
	logger                  logs.Logger
	blockDeviceMounterUtils block_device_mounter_utils.BlockDeviceMounterUtils
	exec                    utils.Executor
}

func newScbMounter(blockDeviceMounterUtils block_device_mounter_utils.BlockDeviceMounterUtils, executer utils.Executor) resources.Mounter {
	return &scbeMounter{
		logger:                  logs.GetLogger(),
		blockDeviceMounterUtils: blockDeviceMounterUtils,
		exec:                    executer,
	}
}

func NewScbeMounter() resources.Mounter {
	blockDeviceMounterUtils := block_device_mounter_utils.NewBlockDeviceMounterUtils()
	return newScbMounter(blockDeviceMounterUtils, utils.NewExecutor())
}

func NewScbeMounterWithExecuter(blockDeviceMounterUtils block_device_mounter_utils.BlockDeviceMounterUtils, executer utils.Executor) resources.Mounter {
	return newScbMounter(blockDeviceMounterUtils, executer)
}

func (s *scbeMounter) prepareVolumeMountProperties(vcGetter resources.VolumeConfigGetter) *resources.VolumeMountProperties {
	volumeConfig := vcGetter.GetVolumeConfig()
	volumeWWN := volumeConfig["Wwn"].(string)
	volumeLunNumber := -1
	if volumeLunNumberInterface, exists := volumeConfig[resources.ScbeKeyVolAttachLunNumToHost]; exists {
		volumeLunNumber = volumeLunNumberInterface.(int)
	}
	return &resources.VolumeMountProperties{WWN: volumeWWN, LunNumber: volumeLunNumber}
}

func (s *scbeMounter) Mount(mountRequest resources.MountRequest) (string, error) {
	defer s.logger.Trace(logs.DEBUG)()
	volumeMountProperties := s.prepareVolumeMountProperties(&mountRequest)

	// Rescan OS
	if err := s.blockDeviceMounterUtils.RescanAll(volumeMountProperties); err != nil {
		return "", s.logger.ErrorRet(err, "RescanAll failed")
	}

	// Discover device
	devicePath, err := s.blockDeviceMounterUtils.Discover(volumeMountProperties.WWN, true)
	if err != nil {
		return "", s.logger.ErrorRet(err, "Discover failed", logs.Args{{"volumeWWN", volumeMountProperties.WWN}})
	}

	// Create mount point if needed   // TODO consider to move it inside the util
	if _, err := s.exec.Stat(mountRequest.Mountpoint); err != nil {
		s.logger.Info("Create mountpoint directory " + mountRequest.Mountpoint)
		if err := s.exec.MkdirAll(mountRequest.Mountpoint, 0700); err != nil {
			return "", s.logger.ErrorRet(err, "MkdirAll failed", logs.Args{{"mountpoint", mountRequest.Mountpoint}})
		}
	} else {
		s.logger.Warning("Idempotent issue : mount point directory already exists", logs.Args{{"mountpoint", mountRequest.Mountpoint}})
	}

	// Mount device and mkfs if needed
	var fstype string
	fstypeInterface, ok := mountRequest.VolumeConfig[resources.OptionNameForVolumeFsType]
	if !ok {
		// the backend should do this default, but this is just for safe
		fstype = resources.DefaultForScbeConfigParamDefaultFilesystem
	} else {
		fstype = fstypeInterface.(string)
	}

	if err := s.blockDeviceMounterUtils.MountDeviceFlow(devicePath, fstype, mountRequest.Mountpoint); err != nil {
		return "", s.logger.ErrorRet(err, "MountDeviceFlow failed", logs.Args{{"devicePath", devicePath}})
	}

	return mountRequest.Mountpoint, nil
}

func (s *scbeMounter) Unmount(unmountRequest resources.UnmountRequest) error {
	defer s.logger.Trace(logs.DEBUG)()

	volumeWWN := unmountRequest.VolumeConfig["Wwn"].(string)
	mountpoint := fmt.Sprintf(resources.PathToMountUbiquityBlockDevices, volumeWWN) // TODO instead of build the mountpoint it should come from unmountRequest.
	skipUnmountFlow := false
	devicePath, err := s.blockDeviceMounterUtils.Discover(volumeWWN, true)
	if err != nil {
		switch err.(type) {
		case *block_device_utils.VolumeNotFoundError:
			s.logger.Warning("Idempotent issue encountered: volume not found. skipping UnmountDeviceFlow ", logs.Args{{"volumeWWN", volumeWWN}})
			skipUnmountFlow = true
		case *block_device_utils.FaultyDeviceError:
			s.logger.Warning("Idempotent issue encountered: mpath device is faulty. continuing with UnmountDeviceFlow", logs.Args{{"device", devicePath}, {"volumeWWN", volumeWWN}})
		default:
			return s.logger.ErrorRet(err, "Discover failed", logs.Args{{"volumeWWN", volumeWWN}})
		}
	}

	if !skipUnmountFlow {
		if err := s.blockDeviceMounterUtils.UnmountDeviceFlow(devicePath, volumeWWN); err != nil {
			return s.logger.ErrorRet(err, "UnmountDeviceFlow failed", logs.Args{{"devicePath", devicePath}})
		}
	}

	s.logger.Info("Delete mountpoint directory if exist", logs.Args{{"mountpoint", mountpoint}})
	// TODO move this part to the util
	if _, err := s.exec.Stat(mountpoint); err == nil {
		s.logger.Debug("Checking if mountpoint is empty and can be deleted", logs.Args{{"mountpoint", mountpoint}})
		emptyDir, err := s.exec.IsDirEmpty(mountpoint)
		if err != nil {
			return s.logger.ErrorRet(err, "Getting number of files failed.", logs.Args{{"mountpoint", mountpoint}})
		}
		if emptyDir != true {
			return s.logger.ErrorRet(&DirecotryIsNotEmptyError{mountpoint}, "Directory is not empty and cannot be removed.", logs.Args{{"mountpoint", mountpoint}})
		}

		if err := s.exec.RemoveAll(mountpoint); err != nil { // TODO its enough to do Remove without All.
			return s.logger.ErrorRet(err, "RemoveAll failed", logs.Args{{"mountpoint", mountpoint}})
		}
	} else {
		if os.IsNotExist(err) {
			s.logger.Warning("Idempotent issue encountered: mountpoint directory does not exist.", logs.Args{{"mountpoint", mountpoint}})
		}
	}

	return nil

}

func (s *scbeMounter) ActionAfterDetach(request resources.AfterDetachRequest) error {
	defer s.logger.Trace(logs.DEBUG)()
	volumeMountProperties := s.prepareVolumeMountProperties(&request)

	// Disconnect volume
	if err := s.blockDeviceMounterUtils.DisconnectAll(volumeMountProperties); err != nil {
		return s.logger.ErrorRet(err, "DisconnectAll failed")
	}
	return nil
}
<|endoftext|>


<commit_before>package spreadsheet

import (
	"appengine"
	"appengine/urlfetch"
	"encoding/csv"
	"fmt"
	"time"

	"lib/constants"
)

type Request struct {
	Time time.Time
	Data []map[string]string
}

var cache = make(map[string]Request)

// Get one Spreadheet’s sheet data
func Get(c appengine.Context, spreadsheetId string, sheetId string, headerRow int) (data []map[string]string) {
	cachedRequest, exists := cache[spreadsheetId]

	if exists {
		cacheExpire := cachedRequest.Time.Add(time.Second * constants.CachingTime)
		useCache := time.Now().Before(cacheExpire)

		if useCache {
			data = cachedRequest.Data
			return
		}
	}

	url := fmt.Sprintf(constants.SheetUrl, spreadsheetId, sheetId)
	rows := fetch(c, url)
	data = parse(c, rows, headerRow)
	cache[spreadsheetId] = Request{time.Now(), data}

	return
}

// Fetch the Spreadsheet
func fetch(c appengine.Context, url string) (rows [][]string) {
	client := urlfetch.Client(c)

	response, getErr := client.Get(url)
	defer response.Body.Close()
	if getErr != nil {
		c.Errorf("spreadsheet.Get: %v", getErr.Error())
		return
	}

	reader := csv.NewReader(response.Body)
	reader.Comma = '\t'

	rows, readErr := reader.ReadAll()
	if readErr != nil {
		c.Errorf("spreadsheet.ReadCsv: %v", readErr.Error())
		return
	}

	return
}

// Parse the spreadsheet and assume first row as header
func parse(c appengine.Context, rows [][]string, headerRow int) (data []map[string]string) {
	for rowKey, row := range rows {
		if rowKey <= headerRow {
			continue
		}

		rowData := make(map[string]string)

		for key, value := range row {
			rowData[rows[headerRow][key]] = value
		}

		data = append(data, rowData)
	}

	return
}
<commit_msg>feat(spreadsheet): trim keys and values<commit_after>package spreadsheet

import (
	"appengine"
	"appengine/urlfetch"
	"encoding/csv"
	"fmt"
	"strings"
	"time"

	"lib/constants"
)

type Request struct {
	Time time.Time
	Data []map[string]string
}

var cache = make(map[string]Request)

// Get one Spreadheet’s sheet data
func Get(c appengine.Context, spreadsheetId string, sheetId string, headerRow int) (data []map[string]string) {
	cachedRequest, exists := cache[spreadsheetId]

	if exists {
		cacheExpire := cachedRequest.Time.Add(time.Second * constants.CachingTime)
		useCache := time.Now().Before(cacheExpire)

		if useCache {
			data = cachedRequest.Data
			return
		}
	}

	url := fmt.Sprintf(constants.SheetUrl, spreadsheetId, sheetId)
	rows := fetch(c, url)
	data = parse(c, rows, headerRow)
	cache[spreadsheetId] = Request{time.Now(), data}

	return
}

// Fetch the Spreadsheet
func fetch(c appengine.Context, url string) (rows [][]string) {
	client := urlfetch.Client(c)

	response, getErr := client.Get(url)
	defer response.Body.Close()
	if getErr != nil {
		c.Errorf("spreadsheet.Get: %v", getErr.Error())
		return
	}

	reader := csv.NewReader(response.Body)
	reader.Comma = '\t'

	rows, readErr := reader.ReadAll()
	if readErr != nil {
		c.Errorf("spreadsheet.ReadCsv: %v", readErr.Error())
		return
	}

	return
}

// Parse the spreadsheet and assume first row as header
func parse(c appengine.Context, rows [][]string, headerRow int) (data []map[string]string) {
	for rowKey, row := range rows {
		if rowKey <= headerRow {
			continue
		}

		rowData := make(map[string]string)

		for key, value := range row {
			rowData[strings.TrimSpace(rows[headerRow][key])] = strings.TrimSpace(value)
		}

		data = append(data, rowData)
	}

	return
}
<|endoftext|>


<commit_before>package specs

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 1
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 0
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 0

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = "-rc6"
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<commit_msg>version: master back to -dev<commit_after>package specs

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 1
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 0
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 0

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = "-rc6-dev"
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/square/spincycle/proto"
	"github.com/square/spincycle/spinc/app"
)

type Status struct {
	ctx   app.Context
	reqId string
}

func NewStatus(ctx app.Context) *Status {
	return &Status{
		ctx: ctx,
	}
}

func (c *Status) Prepare() error {
	if len(c.ctx.Command.Args) == 0 {
		return fmt.Errorf("Usage: spinc status <id>\n")
	}
	c.reqId = c.ctx.Command.Args[0]
	return nil
}

func (c *Status) Run() error {
	status, err := c.ctx.RMClient.RequestStatus(c.reqId)
	if err != nil {
		return err
	}
	if c.ctx.Options.Debug {
		app.Debug("status: %#v", status)
	}

	if c.ctx.Hooks.CommandRunResult != nil {
		c.ctx.Hooks.CommandRunResult(status, err)
		return nil
	}

	running := []string{}
	statuses := []string{}
	js := status.JobChainStatus.JobStatuses
	for _, job := range js {
		if job.State == proto.STATE_RUNNING {
			running = append(running, job.Name)
			statuses = append(statuses, job.Status)
		}
	}

	fmt.Printf("state:      %s\n", proto.StateName[status.State])
	fmt.Printf("running jobs: \n")
	for _, j := range running {
		fmt.Printf("\t%s\n", j)
	}
	fmt.Printf("jobs done:  %d\n", status.FinishedJobs)
	fmt.Printf("jobs total: %d\n", status.TotalJobs)
	fmt.Printf("created:    %s\n", status.CreatedAt)
	fmt.Printf("started:    %s\n", status.StartedAt)
	fmt.Printf("finished:   %s\n", status.FinishedAt)
	fmt.Printf("user:       %s\n", status.User)
	fmt.Printf("type:       %s\n", status.Type)
	fmt.Printf("id:         %s\n", status.Id)

	if c.ctx.Options.Verbose {
		fmt.Printf("live status:\n")
		for i, jobstatus := range statuses {
			fmt.Printf("            %s: %s\n", running[i], jobstatus)
		}
	}

	return nil
}
<commit_msg>map instead of 2 string slices<commit_after>package cmd

import (
	"fmt"

	"github.com/square/spincycle/proto"
	"github.com/square/spincycle/spinc/app"
)

type Status struct {
	ctx   app.Context
	reqId string
}

func NewStatus(ctx app.Context) *Status {
	return &Status{
		ctx: ctx,
	}
}

func (c *Status) Prepare() error {
	if len(c.ctx.Command.Args) == 0 {
		return fmt.Errorf("Usage: spinc status <id>\n")
	}
	c.reqId = c.ctx.Command.Args[0]
	return nil
}

func (c *Status) Run() error {
	status, err := c.ctx.RMClient.RequestStatus(c.reqId)
	if err != nil {
		return err
	}
	if c.ctx.Options.Debug {
		app.Debug("status: %#v", status)
	}

	if c.ctx.Hooks.CommandRunResult != nil {
		c.ctx.Hooks.CommandRunResult(status, err)
		return nil
	}

	running := map[string]string{}
	js := status.JobChainStatus.JobStatuses
	for _, job := range js {
		if job.State == proto.STATE_RUNNING {
			running[job.Name] = job.Status
		}
	}

	fmt.Printf("state:      %s\n", proto.StateName[status.State])
	fmt.Printf("running jobs: \n")
	for j, _ := range running {
		fmt.Printf("\t%s\n", j)
	}
	fmt.Printf("jobs done:  %d\n", status.FinishedJobs)
	fmt.Printf("jobs total: %d\n", status.TotalJobs)
	fmt.Printf("created:    %s\n", status.CreatedAt)
	fmt.Printf("started:    %s\n", status.StartedAt)
	fmt.Printf("finished:   %s\n", status.FinishedAt)
	fmt.Printf("user:       %s\n", status.User)
	fmt.Printf("type:       %s\n", status.Type)
	fmt.Printf("id:         %s\n", status.Id)

	if c.ctx.Options.Verbose {
		fmt.Printf("live status:\n")
		for j, s := range running {
			fmt.Printf("            %s: %s\n", j, s)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2017
// Cavium
// Mainflux
//
// SPDX-License-Identifier: Apache-2.0
//

package distro

import (
	"strconv"

	"github.com/drasko/edgex-export"
	MQTT "github.com/eclipse/paho.mqtt.golang"
	"go.uber.org/zap"
)

type mqttSender struct {
	client MQTT.Client
	topic  string
}

const clientID = "edgex"
const topic = "EdgeX"

// NewMqttSender - create new mqtt sender
func NewMqttSender(addr export.Addressable) Sender {
	opts := MQTT.NewClientOptions()
	// CHN: Should be added protocol from Addressable instead of include it the address param.
	// CHN: We will maintain this behaviour for compatibility with Java
	broker := addr.Address + ":" + strconv.Itoa(addr.Port)
	opts.AddBroker(broker)
	opts.SetClientID(clientID)
	opts.SetUsername(addr.User)
	opts.SetPassword(addr.Password)
	opts.SetAutoReconnect(false)

	sender := mqttSender{
		client: MQTT.NewClient(opts),
		topic:  addr.Topic,
	}

	return sender
}

func (sender mqttSender) Send(data []byte) {
	if !sender.client.IsConnected() {
		logger.Info("Connecting to mqtt server")
		if token := sender.client.Connect(); token.Wait() && token.Error() != nil {
			logger.Warn("Could not connect to mqtt server, drop event")
			return
		}
	}

	token := sender.client.Publish(sender.topic, 0, false, data)
	// FIXME: could be removed? set of tokens?
	token.Wait()
	if token.Error() != nil {
		logger.Warn("mqtt error: ", zap.Error(token.Error()))
	} else {
		logger.Debug("Sent data: ", zap.ByteString("data", data))
	}
}
<commit_msg>Use a different mqtt client id for each connection<commit_after>//
// Copyright (c) 2017
// Cavium
// Mainflux
//
// SPDX-License-Identifier: Apache-2.0
//

package distro

import (
	"math/rand"
	"strconv"

	"github.com/drasko/edgex-export"
	MQTT "github.com/eclipse/paho.mqtt.golang"
	"go.uber.org/zap"
)

type mqttSender struct {
	client MQTT.Client
	topic  string
}

const clientID = "edgex"
const topic = "EdgeX"

// NewMqttSender - create new mqtt sender
func NewMqttSender(addr export.Addressable) Sender {
	clientStr := clientID + "_" + strconv.FormatInt(rand.Int63(), 16)

	opts := MQTT.NewClientOptions()
	// CHN: Should be added protocol from Addressable instead of include it the address param.
	// CHN: We will maintain this behaviour for compatibility with Java
	broker := addr.Address + ":" + strconv.Itoa(addr.Port)
	opts.AddBroker(broker)
	opts.SetClientID(clientStr)
	opts.SetUsername(addr.User)
	opts.SetPassword(addr.Password)
	opts.SetAutoReconnect(false)

	sender := &mqttSender{
		client: MQTT.NewClient(opts),
		topic:  addr.Topic,
	}

	return sender
}

func (sender *mqttSender) Send(data []byte) {
	if !sender.client.IsConnected() {
		logger.Info("Connecting to mqtt server")
		if token := sender.client.Connect(); token.Wait() && token.Error() != nil {
			logger.Warn("Could not connect to mqtt server, drop event")
			return
		}
	}

	token := sender.client.Publish(sender.topic, 0, false, data)
	// FIXME: could be removed? set of tokens?
	token.Wait()
	if token.Error() != nil {
		logger.Warn("mqtt error: ", zap.Error(token.Error()))
	} else {
		logger.Debug("Sent data: ", zap.ByteString("data", data))
	}
}
<|endoftext|>


<commit_before>// Package meetup implements the OAuth2 protocol for authenticating users through meetup.com .
// This package can be used as a reference implementation of an OAuth2 provider for Goth.
package meetup

import (
	"net/http"
	"golang.org/x/oauth2"
	"github.com/markbates/goth"
	"io/ioutil"
	"encoding/json"
	"bytes"
	"io"
	"strconv"
)

const (
	authURL         string = "https://secure.meetup.com/oauth2/authorize"
	tokenURL        string = "https://secure.meetup.com/oauth2/access"
	endpointProfile string = "https://api.meetup.com/2/member/self"
)

// New creates a new Meetup provider, and sets up important connection details.
// You should always call `meetup.New` to get a new Provider. Never try to create
// one manually.
func New(clientKey, secret, callbackURL string, scopes ...string) *Provider {
	p := &Provider{
		ClientKey:           clientKey,
		Secret:              secret,
		CallbackURL:         callbackURL,
		providerName:        "meetup",
	}
	// register this meetup.com provider as broken for oauth2 RetrieveToken
	oauth2.RegisterBrokenAuthHeaderProvider(tokenURL)
	p.config = newConfig(p, scopes)
	return p
}

// Provider is the implementation of `goth.Provider` for accessing meetup.com .
type Provider struct {
	ClientKey    string
	Secret       string
	CallbackURL  string
	HTTPClient   *http.Client
	config       *oauth2.Config
	providerName string
}

// Name is the name used to retrieve this provider later.
func (p *Provider) Name() string {
	return p.providerName
}

// SetName is to update the name of the provider (needed in case of multiple providers of 1 type)
func (p *Provider) SetName(name string) {
	p.providerName = name
}

func (p *Provider) Client() *http.Client {
	return goth.HTTPClientWithFallBack(p.HTTPClient)
}

// Debug is a no-op for the meetup package.
func (p *Provider) Debug(debug bool) {}

// BeginAuth asks meetup.com for an authentication end-point.
func (p *Provider) BeginAuth(state string) (goth.Session, error) {
	url := p.config.AuthCodeURL(state)
	session := &Session{
		AuthURL: url,
	}
	return session, nil
}

// FetchUser will go to meetup.com and access basic information about the user.
func (p *Provider) FetchUser(session goth.Session) (goth.User, error) {
	sess := session.(*Session)
	user := goth.User{
		AccessToken:  sess.AccessToken,
		Provider:     p.Name(),
		RefreshToken: sess.RefreshToken,
		ExpiresAt:    sess.ExpiresAt,
	}

	request, err := http.NewRequest("GET", endpointProfile, nil)
	if err != nil {
		return user, err
	}

	request.Header.Set("Authorization", "Bearer " + sess.AccessToken)
	response, err := p.Client().Do(request)
	if err != nil {
		return user, err
	}
	defer response.Body.Close()

	bits, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return user, err
	}

	err = json.NewDecoder(bytes.NewReader(bits)).Decode(&user.RawData)
	if err != nil {
		return user, err
	}

	err = userFromReader(bytes.NewReader(bits), &user)
	return user, err
}

func newConfig(provider *Provider, scopes []string) *oauth2.Config {
	c := &oauth2.Config{
		ClientID:     provider.ClientKey,
		ClientSecret: provider.Secret,
		RedirectURL:  provider.CallbackURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  authURL,
			TokenURL: tokenURL,
		},
		Scopes: []string{},
	}

	if len(scopes) > 0 {
		for _, scope := range scopes {
			c.Scopes = append(c.Scopes, scope)
		}
	}

	return c
}

func userFromReader(r io.Reader, user *goth.User) error {
	u := struct {
		ID      uint64 `json:"id"`
		Name    string `json:"name"`
		Picture string `json:"photo_url"`
		Country string `json:"country"`
		City    string `json:"city"`
		State   string `json:"state"`
	}{}
	err := json.NewDecoder(r).Decode(&u)
	if err != nil {
		return err
	}
	user.UserID = strconv.FormatUint(u.ID, 10)
	user.Name = u.Name
	user.NickName = u.Name

	var location string
	if len(u.City) > 0 {
		location = u.City
	}
	if len(u.State) > 0 {
		if len(location) > 0 {
			location = location + ", " + u.State
		} else {
			location = u.State
		}
	}
	if len(u.Country) > 0 {
		if len(location) > 0 {
			location = location + ", " + u.Country
		} else {
			location = u.Country
		}
	}

	user.Location = location
	user.AvatarURL = u.Picture
	return nil
}

//RefreshTokenAvailable refresh token is provided by auth provider or not
func (p *Provider) RefreshTokenAvailable() bool {
	return true
}

//RefreshToken get new access token based on the refresh token
func (p *Provider) RefreshToken(refreshToken string) (*oauth2.Token, error) {
	token := &oauth2.Token{RefreshToken: refreshToken}
	ts := p.config.TokenSource(goth.ContextForClient(p.Client()), token)
	newToken, err := ts.Token()
	if err != nil {
		return nil, err
	}
	return newToken, err
}
<commit_msg>update meetup provider with additional validations in FetchUser<commit_after>// Package meetup implements the OAuth2 protocol for authenticating users through meetup.com .
// This package can be used as a reference implementation of an OAuth2 provider for Goth.
package meetup

import (
	"net/http"
	"golang.org/x/oauth2"
	"github.com/markbates/goth"
	"io/ioutil"
	"encoding/json"
	"bytes"
	"io"
	"strconv"
	"fmt"
)

const (
	authURL         string = "https://secure.meetup.com/oauth2/authorize"
	tokenURL        string = "https://secure.meetup.com/oauth2/access"
	endpointProfile string = "https://api.meetup.com/2/member/self"
)

// New creates a new Meetup provider, and sets up important connection details.
// You should always call `meetup.New` to get a new Provider. Never try to create
// one manually.
func New(clientKey, secret, callbackURL string, scopes ...string) *Provider {
	p := &Provider{
		ClientKey:           clientKey,
		Secret:              secret,
		CallbackURL:         callbackURL,
		providerName:        "meetup",
	}
	// register this meetup.com provider as broken for oauth2 RetrieveToken
	oauth2.RegisterBrokenAuthHeaderProvider(tokenURL)
	p.config = newConfig(p, scopes)
	return p
}

// Provider is the implementation of `goth.Provider` for accessing meetup.com .
type Provider struct {
	ClientKey    string
	Secret       string
	CallbackURL  string
	HTTPClient   *http.Client
	config       *oauth2.Config
	providerName string
}

// Name is the name used to retrieve this provider later.
func (p *Provider) Name() string {
	return p.providerName
}

// SetName is to update the name of the provider (needed in case of multiple providers of 1 type)
func (p *Provider) SetName(name string) {
	p.providerName = name
}

func (p *Provider) Client() *http.Client {
	return goth.HTTPClientWithFallBack(p.HTTPClient)
}

// Debug is a no-op for the meetup package.
func (p *Provider) Debug(debug bool) {}

// BeginAuth asks meetup.com for an authentication end-point.
func (p *Provider) BeginAuth(state string) (goth.Session, error) {
	url := p.config.AuthCodeURL(state)
	session := &Session{
		AuthURL: url,
	}
	return session, nil
}

// FetchUser will go to meetup.com and access basic information about the user.
func (p *Provider) FetchUser(session goth.Session) (goth.User, error) {
	sess := session.(*Session)
	user := goth.User{
		AccessToken:  sess.AccessToken,
		Provider:     p.Name(),
		RefreshToken: sess.RefreshToken,
		ExpiresAt:    sess.ExpiresAt,
	}

	if user.AccessToken == "" {
		// data is not yet retrieved since accessToken is still empty
		return user, fmt.Errorf("%s cannot get user information without accessToken", p.providerName)
	}

	request, err := http.NewRequest("GET", endpointProfile, nil)
	if err != nil {
		return user, err
	}

	request.Header.Set("Authorization", "Bearer " + sess.AccessToken)
	response, err := p.Client().Do(request)
	if err != nil {
		return user, err
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		return user, fmt.Errorf("%s responded with a %d trying to fetch user information", p.providerName, response.StatusCode)
	}

	bits, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return user, err
	}

	err = json.NewDecoder(bytes.NewReader(bits)).Decode(&user.RawData)
	if err != nil {
		return user, err
	}

	err = userFromReader(bytes.NewReader(bits), &user)
	return user, err
}

func newConfig(provider *Provider, scopes []string) *oauth2.Config {
	c := &oauth2.Config{
		ClientID:     provider.ClientKey,
		ClientSecret: provider.Secret,
		RedirectURL:  provider.CallbackURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  authURL,
			TokenURL: tokenURL,
		},
		Scopes: []string{},
	}

	if len(scopes) > 0 {
		for _, scope := range scopes {
			c.Scopes = append(c.Scopes, scope)
		}
	}

	return c
}

func userFromReader(r io.Reader, user *goth.User) error {
	u := struct {
		ID      uint64 `json:"id"`
		Name    string `json:"name"`
		Picture string `json:"photo_url"`
		Country string `json:"country"`
		City    string `json:"city"`
		State   string `json:"state"`
	}{}
	err := json.NewDecoder(r).Decode(&u)
	if err != nil {
		return err
	}
	user.UserID = strconv.FormatUint(u.ID, 10)
	user.Name = u.Name
	user.NickName = u.Name

	var location string
	if len(u.City) > 0 {
		location = u.City
	}
	if len(u.State) > 0 {
		if len(location) > 0 {
			location = location + ", " + u.State
		} else {
			location = u.State
		}
	}
	if len(u.Country) > 0 {
		if len(location) > 0 {
			location = location + ", " + u.Country
		} else {
			location = u.Country
		}
	}

	user.Location = location
	user.AvatarURL = u.Picture
	return nil
}

//RefreshTokenAvailable refresh token is provided by auth provider or not
func (p *Provider) RefreshTokenAvailable() bool {
	return true
}

//RefreshToken get new access token based on the refresh token
func (p *Provider) RefreshToken(refreshToken string) (*oauth2.Token, error) {
	token := &oauth2.Token{RefreshToken: refreshToken}
	ts := p.config.TokenSource(goth.ContextForClient(p.Client()), token)
	newToken, err := ts.Token()
	if err != nil {
		return nil, err
	}
	return newToken, err
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/dotcloud/docker"
	dclient "github.com/fsouza/go-dockerclient"
	"github.com/globocom/config"
	"github.com/globocom/docker-cluster/cluster"
	"github.com/globocom/tsuru/action"
	"github.com/globocom/tsuru/fs"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"io"
	"labix.org/v2/mgo/bson"
	"strings"
	"sync"
)

var (
	dCluster *cluster.Cluster
	cmutext  sync.Mutex
	fsystem  fs.Fs
)

func dockerCluster() *cluster.Cluster {
	cmutext.Lock()
	defer cmutext.Unlock()
	if dCluster == nil {
		servers, _ := config.GetList("docker:servers")
		nodes := []cluster.Node{}
		for index, server := range servers {
			node := cluster.Node{
				ID:      fmt.Sprintf("server%d", index),
				Address: server,
			}
			nodes = append(nodes, node)
		}
		dCluster, _ = cluster.New(nodes...)
	}
	return dCluster
}

func filesystem() fs.Fs {
	if fsystem == nil {
		fsystem = fs.OsFs{}
	}
	return fsystem
}

// runCmd executes commands and log the given stdout and stderror.
func runCmd(cmd string, args ...string) (string, error) {
	out := bytes.Buffer{}
	err := executor().Execute(cmd, args, nil, &out, &out)
	log.Printf("running the cmd: %s with the args: %s", cmd, args)
	if err != nil {
		return "", &cmdError{cmd: cmd, args: args, err: err, out: out.String()}
	}
	return out.String(), nil
}

func getPort() (string, error) {
	return config.GetString("docker:run-cmd:port")
}

type container struct {
	ID       string `bson:"_id"`
	AppName  string
	Type     string
	IP       string
	Port     string
	HostPort string
	Status   string
	Version  string
	Image    string
}

func (c *container) getAddress() string {
	hostAddr, err := config.Get("docker:host-address")
	if err != nil {
		log.Printf("Failed to obtain container address: %s", err.Error())
		return ""
	}
	return fmt.Sprintf("http://%s:%s", hostAddr, c.HostPort)
}

// newContainer creates a new container in Docker and stores it in the database.
func newContainer(app provision.App, imageId string, cmds []string) (container, error) {
	cont := container{
		AppName: app.GetName(),
		Type:    app.GetPlatform(),
	}
	port, err := getPort()
	if err != nil {
		log.Printf("error on getting port for container %s - %s", cont.AppName, port)
		return container{}, err
	}
	// user, err := config.GetString("docker:ssh:user")
	// if err != nil {
	// 	log.Printf("error on getting user for container %s - %s", cont.AppName, user)
	// 	return container{}, err
	// }
	config := docker.Config{
		Image:     imageId,
		Cmd:       cmds,
		PortSpecs: []string{port},
		/* User:         user, */
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
	}
	_, c, err := dockerCluster().CreateContainer(&config)
	if err != nil {
		log.Printf("error on creating container in docker %s - %s", cont.AppName, err.Error())
		return container{}, err
	}
	cont.ID = c.ID
	cont.Port = port
	return cont, nil
}

// hostPort returns the host port mapped for the container.
func (c *container) hostPort() (string, error) {
	if c.Port == "" {
		return "", errors.New("Container does not contain any mapped port")
	}
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		return "", err
	}
	if dockerContainer.NetworkSettings != nil {
		mappedPorts := dockerContainer.NetworkSettings.PortMapping
		if port, ok := mappedPorts[c.Port]; ok {
			return port, nil
		}
	}
	return "", fmt.Errorf("Container port %s is not mapped to any host port", c.Port)
}

// ip returns the ip for the container.
func (c *container) ip() (string, error) {
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		return "", err
	}
	if dockerContainer.NetworkSettings == nil {
		msg := "Error when getting container information. NetworkSettings is missing."
		log.Print(msg)
		return "", errors.New(msg)
	}
	instanceIP := dockerContainer.NetworkSettings.IPAddress
	if instanceIP == "" {
		msg := "error: Can't get ipaddress..."
		log.Print(msg)
		return "", errors.New(msg)
	}
	log.Printf("Instance IpAddress: %s", instanceIP)
	return instanceIP, nil
}

func (c *container) setStatus(status string) error {
	c.Status = status
	coll := collection()
	defer coll.Database.Session.Close()
	return coll.UpdateId(c.ID, c)
}

func (c *container) setImage(imageId string) error {
	c.Image = imageId
	coll := collection()
	defer coll.Database.Session.Close()
	return coll.UpdateId(c.ID, c)
}

func deploy(app provision.App, version string, w io.Writer) (string, error) {
	commands, err := deployCmds(app, version)
	if err != nil {
		return "", err
	}
	imageId := getImage(app)
	actions := []*action.Action{&createContainer, &startContainer, &insertContainer}
	pipeline := action.NewPipeline(actions...)
	err = pipeline.Execute(app, imageId, commands)
	if err != nil {
		log.Printf("error on execute deploy pipeline for app %s - %s", app.GetName(), err.Error())
		return "", err
	}
	c := pipeline.Result().(container)
	for {
		result, err := c.stopped()
		if err != nil {
			log.Printf("error on stopped for container %s - %s", c.ID, err.Error())
			return "", err
		}
		if result {
			break
		}
	}
	err = c.logs(w)
	if err != nil {
		log.Printf("error on get logs for container %s - %s", c.ID, err.Error())
		return "", err
	}
	imageId, err = c.commit()
	if err != nil {
		log.Printf("error on commit container %s - %s", c.ID, err.Error())
		return "", err
	}
	c.remove()
	// if err != nil {
	// 	return "", err
	// }
	return imageId, nil
}

func start(app provision.App, imageId string, w io.Writer) (*container, error) {
	commands, err := runCmds()
	if err != nil {
		return nil, err
	}
	actions := []*action.Action{&createContainer, &startContainer, &setIp, &setHostPort, &insertContainer, &addRoute}
	pipeline := action.NewPipeline(actions...)
	err = pipeline.Execute(app, imageId, commands)
	if err != nil {
		return nil, err
	}
	c := pipeline.Result().(container)
	err = c.setImage(imageId)
	if err != nil {
		return nil, err
	}
	err = c.setStatus("running")
	if err != nil {
		return nil, err
	}
	return &c, nil
}

// remove removes a docker container.
func (c *container) remove() error {
	address := c.getAddress()
	log.Printf("Removing container %s from docker", c.ID)
	err := dockerCluster().RemoveContainer(c.ID)
	if err != nil {
		log.Printf("Failed to remove container from docker: %s", err)
		return err
	}
	runCmd("ssh-keygen", "-R", c.IP)
	log.Printf("Removing container %s from database", c.ID)
	coll := collection()
	defer coll.Database.Session.Close()
	if err := coll.RemoveId(c.ID); err != nil {
		log.Printf("Failed to remove container from database: %s", err.Error())
		return err
	}
	r, err := getRouter()
	if err != nil {
		log.Printf("Failed to obtain router: %s", err.Error())
		return err
	}
	if err := r.RemoveRoute(c.AppName, address); err != nil {
		log.Printf("Failed to remove route: %s", err.Error())
		return err
	}
	return nil
}

func (c *container) ssh(stdout, stderr io.Writer, cmd string, args ...string) error {
	stderr = &filter{w: stderr, content: []byte("unable to resolve host")}
	user, err := config.GetString("docker:ssh:user")
	if err != nil {
		return err
	}
	sshArgs := []string{c.IP, "-l", user, "-o", "StrictHostKeyChecking no"}
	if keyFile, err := config.GetString("docker:ssh:private-key"); err == nil {
		sshArgs = append(sshArgs, "-i", keyFile)
	}
	sshArgs = append(sshArgs, "--", cmd)
	sshArgs = append(sshArgs, args...)
	return executor().Execute("ssh", sshArgs, nil, stdout, stderr)
}

// commit commits an image in docker based in the container
func (c *container) commit() (string, error) {
	log.Printf("commiting container %s", c.ID)
	repoNamespace, _ := config.GetString("docker:repository-namespace")
	opts := dclient.CommitContainerOptions{Container: c.ID, Repository: repoNamespace + "/" + c.AppName}
	image, err := dockerCluster().CommitContainer(opts)
	if err != nil {
		log.Printf("Could not commit docker image: %s", err.Error())
		return "", err
	}
	log.Printf("image %s gerenated from container %s", image.ID, c.ID)
	return image.ID, nil
}

// stopped returns true if the container is stopped.
func (c *container) stopped() (bool, error) {
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		log.Printf("error on get log for container %s", c.ID)
		return false, err
	}
	running := dockerContainer.State.Running
	return !running, nil
}

// logs returns logs for the container.
func (c *container) logs(w io.Writer) error {
	opts := dclient.AttachToContainerOptions{
		Container:    c.ID,
		Logs:         true,
		OutputStream: w,
	}
	return dockerCluster().AttachToContainer(opts)
}

func getContainer(id string) (*container, error) {
	var c container
	coll := collection()
	defer coll.Database.Session.Close()
	err := coll.Find(bson.M{"_id": id}).One(&c)
	if err != nil {
		return nil, err
	}
	return &c, nil
}

func listAppContainers(appName string) ([]container, error) {
	var containers []container
	err := collection().Find(bson.M{"appname": appName}).All(&containers)
	return containers, err
}

// getImage returns the image name or id from an app.
func getImage(app provision.App) string {
	var c container
	collection().Find(bson.M{"appname": app.GetName()}).One(&c)
	if c.Image != "" {
		return c.Image
	}
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		return ""
	}
	return fmt.Sprintf("%s/%s", repoNamespace, app.GetPlatform())
}

// removeImage removes an image from docker registry
func removeImage(imageId string) error {
	return dockerCluster().RemoveImage(imageId)
}

type cmdError struct {
	cmd  string
	args []string
	err  error
	out  string
}

func (e *cmdError) Error() string {
	command := e.cmd + " " + strings.Join(e.args, " ")
	return fmt.Sprintf("Failed to run command %q (%s): %s.", command, e.err, e.out)
}
<commit_msg>provision/docker: simplify code in stopped<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/dotcloud/docker"
	dclient "github.com/fsouza/go-dockerclient"
	"github.com/globocom/config"
	"github.com/globocom/docker-cluster/cluster"
	"github.com/globocom/tsuru/action"
	"github.com/globocom/tsuru/fs"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"io"
	"labix.org/v2/mgo/bson"
	"strings"
	"sync"
)

var (
	dCluster *cluster.Cluster
	cmutext  sync.Mutex
	fsystem  fs.Fs
)

func dockerCluster() *cluster.Cluster {
	cmutext.Lock()
	defer cmutext.Unlock()
	if dCluster == nil {
		servers, _ := config.GetList("docker:servers")
		nodes := []cluster.Node{}
		for index, server := range servers {
			node := cluster.Node{
				ID:      fmt.Sprintf("server%d", index),
				Address: server,
			}
			nodes = append(nodes, node)
		}
		dCluster, _ = cluster.New(nodes...)
	}
	return dCluster
}

func filesystem() fs.Fs {
	if fsystem == nil {
		fsystem = fs.OsFs{}
	}
	return fsystem
}

// runCmd executes commands and log the given stdout and stderror.
func runCmd(cmd string, args ...string) (string, error) {
	out := bytes.Buffer{}
	err := executor().Execute(cmd, args, nil, &out, &out)
	log.Printf("running the cmd: %s with the args: %s", cmd, args)
	if err != nil {
		return "", &cmdError{cmd: cmd, args: args, err: err, out: out.String()}
	}
	return out.String(), nil
}

func getPort() (string, error) {
	return config.GetString("docker:run-cmd:port")
}

type container struct {
	ID       string `bson:"_id"`
	AppName  string
	Type     string
	IP       string
	Port     string
	HostPort string
	Status   string
	Version  string
	Image    string
}

func (c *container) getAddress() string {
	hostAddr, err := config.Get("docker:host-address")
	if err != nil {
		log.Printf("Failed to obtain container address: %s", err.Error())
		return ""
	}
	return fmt.Sprintf("http://%s:%s", hostAddr, c.HostPort)
}

// newContainer creates a new container in Docker and stores it in the database.
func newContainer(app provision.App, imageId string, cmds []string) (container, error) {
	cont := container{
		AppName: app.GetName(),
		Type:    app.GetPlatform(),
	}
	port, err := getPort()
	if err != nil {
		log.Printf("error on getting port for container %s - %s", cont.AppName, port)
		return container{}, err
	}
	// user, err := config.GetString("docker:ssh:user")
	// if err != nil {
	// 	log.Printf("error on getting user for container %s - %s", cont.AppName, user)
	// 	return container{}, err
	// }
	config := docker.Config{
		Image:     imageId,
		Cmd:       cmds,
		PortSpecs: []string{port},
		/* User:         user, */
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
	}
	_, c, err := dockerCluster().CreateContainer(&config)
	if err != nil {
		log.Printf("error on creating container in docker %s - %s", cont.AppName, err.Error())
		return container{}, err
	}
	cont.ID = c.ID
	cont.Port = port
	return cont, nil
}

// hostPort returns the host port mapped for the container.
func (c *container) hostPort() (string, error) {
	if c.Port == "" {
		return "", errors.New("Container does not contain any mapped port")
	}
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		return "", err
	}
	if dockerContainer.NetworkSettings != nil {
		mappedPorts := dockerContainer.NetworkSettings.PortMapping
		if port, ok := mappedPorts[c.Port]; ok {
			return port, nil
		}
	}
	return "", fmt.Errorf("Container port %s is not mapped to any host port", c.Port)
}

// ip returns the ip for the container.
func (c *container) ip() (string, error) {
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		return "", err
	}
	if dockerContainer.NetworkSettings == nil {
		msg := "Error when getting container information. NetworkSettings is missing."
		log.Print(msg)
		return "", errors.New(msg)
	}
	instanceIP := dockerContainer.NetworkSettings.IPAddress
	if instanceIP == "" {
		msg := "error: Can't get ipaddress..."
		log.Print(msg)
		return "", errors.New(msg)
	}
	log.Printf("Instance IpAddress: %s", instanceIP)
	return instanceIP, nil
}

func (c *container) setStatus(status string) error {
	c.Status = status
	coll := collection()
	defer coll.Database.Session.Close()
	return coll.UpdateId(c.ID, c)
}

func (c *container) setImage(imageId string) error {
	c.Image = imageId
	coll := collection()
	defer coll.Database.Session.Close()
	return coll.UpdateId(c.ID, c)
}

func deploy(app provision.App, version string, w io.Writer) (string, error) {
	commands, err := deployCmds(app, version)
	if err != nil {
		return "", err
	}
	imageId := getImage(app)
	actions := []*action.Action{&createContainer, &startContainer, &insertContainer}
	pipeline := action.NewPipeline(actions...)
	err = pipeline.Execute(app, imageId, commands)
	if err != nil {
		log.Printf("error on execute deploy pipeline for app %s - %s", app.GetName(), err.Error())
		return "", err
	}
	c := pipeline.Result().(container)
	for {
		result, err := c.stopped()
		if err != nil {
			log.Printf("error on stopped for container %s - %s", c.ID, err.Error())
			return "", err
		}
		if result {
			break
		}
	}
	err = c.logs(w)
	if err != nil {
		log.Printf("error on get logs for container %s - %s", c.ID, err.Error())
		return "", err
	}
	imageId, err = c.commit()
	if err != nil {
		log.Printf("error on commit container %s - %s", c.ID, err.Error())
		return "", err
	}
	c.remove()
	// if err != nil {
	// 	return "", err
	// }
	return imageId, nil
}

func start(app provision.App, imageId string, w io.Writer) (*container, error) {
	commands, err := runCmds()
	if err != nil {
		return nil, err
	}
	actions := []*action.Action{&createContainer, &startContainer, &setIp, &setHostPort, &insertContainer, &addRoute}
	pipeline := action.NewPipeline(actions...)
	err = pipeline.Execute(app, imageId, commands)
	if err != nil {
		return nil, err
	}
	c := pipeline.Result().(container)
	err = c.setImage(imageId)
	if err != nil {
		return nil, err
	}
	err = c.setStatus("running")
	if err != nil {
		return nil, err
	}
	return &c, nil
}

// remove removes a docker container.
func (c *container) remove() error {
	address := c.getAddress()
	log.Printf("Removing container %s from docker", c.ID)
	err := dockerCluster().RemoveContainer(c.ID)
	if err != nil {
		log.Printf("Failed to remove container from docker: %s", err)
		return err
	}
	runCmd("ssh-keygen", "-R", c.IP)
	log.Printf("Removing container %s from database", c.ID)
	coll := collection()
	defer coll.Database.Session.Close()
	if err := coll.RemoveId(c.ID); err != nil {
		log.Printf("Failed to remove container from database: %s", err.Error())
		return err
	}
	r, err := getRouter()
	if err != nil {
		log.Printf("Failed to obtain router: %s", err.Error())
		return err
	}
	if err := r.RemoveRoute(c.AppName, address); err != nil {
		log.Printf("Failed to remove route: %s", err.Error())
		return err
	}
	return nil
}

func (c *container) ssh(stdout, stderr io.Writer, cmd string, args ...string) error {
	stderr = &filter{w: stderr, content: []byte("unable to resolve host")}
	user, err := config.GetString("docker:ssh:user")
	if err != nil {
		return err
	}
	sshArgs := []string{c.IP, "-l", user, "-o", "StrictHostKeyChecking no"}
	if keyFile, err := config.GetString("docker:ssh:private-key"); err == nil {
		sshArgs = append(sshArgs, "-i", keyFile)
	}
	sshArgs = append(sshArgs, "--", cmd)
	sshArgs = append(sshArgs, args...)
	return executor().Execute("ssh", sshArgs, nil, stdout, stderr)
}

// commit commits an image in docker based in the container
func (c *container) commit() (string, error) {
	log.Printf("commiting container %s", c.ID)
	repoNamespace, _ := config.GetString("docker:repository-namespace")
	opts := dclient.CommitContainerOptions{Container: c.ID, Repository: repoNamespace + "/" + c.AppName}
	image, err := dockerCluster().CommitContainer(opts)
	if err != nil {
		log.Printf("Could not commit docker image: %s", err.Error())
		return "", err
	}
	log.Printf("image %s gerenated from container %s", image.ID, c.ID)
	return image.ID, nil
}

// stopped returns true if the container is stopped.
func (c *container) stopped() (bool, error) {
	dockerContainer, err := dockerCluster().InspectContainer(c.ID)
	if err != nil {
		log.Printf("error on get log for container %s", c.ID)
		return false, err
	}
	return !dockerContainer.State.Running, nil
}

// logs returns logs for the container.
func (c *container) logs(w io.Writer) error {
	opts := dclient.AttachToContainerOptions{
		Container:    c.ID,
		Logs:         true,
		OutputStream: w,
	}
	return dockerCluster().AttachToContainer(opts)
}

func getContainer(id string) (*container, error) {
	var c container
	coll := collection()
	defer coll.Database.Session.Close()
	err := coll.Find(bson.M{"_id": id}).One(&c)
	if err != nil {
		return nil, err
	}
	return &c, nil
}

func listAppContainers(appName string) ([]container, error) {
	var containers []container
	err := collection().Find(bson.M{"appname": appName}).All(&containers)
	return containers, err
}

// getImage returns the image name or id from an app.
func getImage(app provision.App) string {
	var c container
	collection().Find(bson.M{"appname": app.GetName()}).One(&c)
	if c.Image != "" {
		return c.Image
	}
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		return ""
	}
	return fmt.Sprintf("%s/%s", repoNamespace, app.GetPlatform())
}

// removeImage removes an image from docker registry
func removeImage(imageId string) error {
	return dockerCluster().RemoveImage(imageId)
}

type cmdError struct {
	cmd  string
	args []string
	err  error
	out  string
}

func (e *cmdError) Error() string {
	command := e.cmd + " " + strings.Join(e.args, " ")
	return fmt.Sprintf("Failed to run command %q (%s): %s.", command, e.err, e.out)
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd

package cgo

import _ "unsafe"

// Supply environ and __progname, because we don't
// link against the standard FreeBSD crt0.o and the
// libc dynamic library needs them.

//go:linkname _environ environ
//go:linkname _progname __progname

var _environ uintptr
var _progname uintptr
<commit_msg>[dev.cc] runtime/cgo: fix freebsd build?<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd

package cgo

import _ "unsafe"

// Supply environ and __progname, because we don't
// link against the standard FreeBSD crt0.o and the
// libc dynamic library needs them.

//go:linkname _environ environ
//go:linkname _progname __progname

//go:cgo_export_static environ
//go:cgo_export_static __progname

var _environ uintptr
var _progname uintptr
<|endoftext|>


<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"code.google.com/p/gorest"
	"encoding/json"
	"errors"
	"github.com/prometheus/prometheus/rules"
	"github.com/prometheus/prometheus/rules/ast"
	"log"
	"net/http"
	"sort"
	"time"
)

func (serv MetricsService) Query(Expr string, Json string) (result string) {
	exprNode, err := rules.LoadExprFromString(Expr)
	if err != nil {
		return ast.ErrorToJSON(err)
	}

	timestamp := serv.time.Now()

	rb := serv.ResponseBuilder()
	var format ast.OutputFormat
	if Json != "" {
		format = ast.JSON
		rb.SetContentType(gorest.Application_Json)
	} else {
		format = ast.TEXT
		rb.SetContentType(gorest.Text_Plain)
	}

	return ast.EvalToString(exprNode, &timestamp, format)
}

func (serv MetricsService) QueryRange(Expr string, End int64, Range int64, Step int64) string {
	exprNode, err := rules.LoadExprFromString(Expr)
	if err != nil {
		return ast.ErrorToJSON(err)
	}
	if exprNode.Type() != ast.VECTOR {
		return ast.ErrorToJSON(errors.New("Expression does not evaluate to vector type"))
	}
	rb := serv.ResponseBuilder()
	rb.SetContentType(gorest.Application_Json)

	if End == 0 {
		End = serv.time.Now().Unix()
	}

	if Step < 1 {
		Step = 1
	}

	if End-Range < 0 {
		Range = End
	}

	// Align the start to step "tick" boundary.
	End -= End % Step

	matrix := ast.EvalVectorRange(
		exprNode.(ast.VectorNode),
		time.Unix(End-Range, 0),
		time.Unix(End, 0),
		time.Duration(Step)*time.Second)

	sort.Sort(matrix)
	return ast.TypedValueToJSON(matrix, "matrix")
}

func (serv MetricsService) Metrics() string {
	metricNames, err := serv.appState.Persistence.GetAllMetricNames()
	rb := serv.ResponseBuilder()
	rb.SetContentType(gorest.Application_Json)
	if err != nil {
		log.Printf("Error loading metric names: %v", err)
		rb.SetResponseCode(http.StatusInternalServerError)
		return err.Error()
	}
	sort.Strings(metricNames)
	resultBytes, err := json.Marshal(metricNames)
	if err != nil {
		log.Printf("Error marshalling metric names: %v", err)
		rb.SetResponseCode(http.StatusInternalServerError)
		return err.Error()
	}
	return string(resultBytes)
}
<commit_msg>Lower-case web API method arguments.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"code.google.com/p/gorest"
	"encoding/json"
	"errors"
	"github.com/prometheus/prometheus/rules"
	"github.com/prometheus/prometheus/rules/ast"
	"log"
	"net/http"
	"sort"
	"time"
)

func (serv MetricsService) Query(expr string, formatJson string) (result string) {
	exprNode, err := rules.LoadExprFromString(expr)
	if err != nil {
		return ast.ErrorToJSON(err)
	}

	timestamp := serv.time.Now()

	rb := serv.ResponseBuilder()
	var format ast.OutputFormat
	if formatJson != "" {
		format = ast.JSON
		rb.SetContentType(gorest.Application_Json)
	} else {
		format = ast.TEXT
		rb.SetContentType(gorest.Text_Plain)
	}

	return ast.EvalToString(exprNode, &timestamp, format)
}

func (serv MetricsService) QueryRange(expr string, end int64, duration int64, step int64) string {
	exprNode, err := rules.LoadExprFromString(expr)
	if err != nil {
		return ast.ErrorToJSON(err)
	}
	if exprNode.Type() != ast.VECTOR {
		return ast.ErrorToJSON(errors.New("Expression does not evaluate to vector type"))
	}
	rb := serv.ResponseBuilder()
	rb.SetContentType(gorest.Application_Json)

	if end == 0 {
		end = serv.time.Now().Unix()
	}

	if step < 1 {
		step = 1
	}

	if end-duration < 0 {
		duration = end
	}

	// Align the start to step "tick" boundary.
	end -= end % step

	matrix := ast.EvalVectorRange(
		exprNode.(ast.VectorNode),
		time.Unix(end-duration, 0),
		time.Unix(end, 0),
		time.Duration(step)*time.Second)

	sort.Sort(matrix)
	return ast.TypedValueToJSON(matrix, "matrix")
}

func (serv MetricsService) Metrics() string {
	metricNames, err := serv.appState.Persistence.GetAllMetricNames()
	rb := serv.ResponseBuilder()
	rb.SetContentType(gorest.Application_Json)
	if err != nil {
		log.Printf("Error loading metric names: %v", err)
		rb.SetResponseCode(http.StatusInternalServerError)
		return err.Error()
	}
	sort.Strings(metricNames)
	resultBytes, err := json.Marshal(metricNames)
	if err != nil {
		log.Printf("Error marshalling metric names: %v", err)
		rb.SetResponseCode(http.StatusInternalServerError)
		return err.Error()
	}
	return string(resultBytes)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"os"
	"os/exec"
	"syscall"
	"github.com/docker/docker/pkg/reexec"
	"path/filepath"
	"github.com/getsentry/raven-go"
	"../../models"
	"../../config"
	"flag"
	"strings"
	"bytes"
	"time"
	"strconv"
)

func init() {
	raven.SetDSN(config.SENTRY_DSN)
	reexec.Register("justiceInit", justiceInit)
	if reexec.Init() {
		os.Exit(models.CODE_OK)
	}
}

func pivotRoot(newRoot string) error {
	putOld := filepath.Join(newRoot, "/.pivot_root")

	// bind mount newRoot to itself - this is a slight hack needed to satisfy requirement (2)
	//
	// The following restrictions apply to new_root and put_old:
	// 1.  They must be directories.
	// 2.  new_root and put_old must not be on the same filesystem as the current root.
	// 3.  put_old must be underneath new_root, that is, adding a nonzero
	//     number of /.. to the string pointed to by put_old must yield the same directory as new_root.
	// 4.  No other filesystem may be mounted on put_old.
	if err := syscall.Mount(newRoot, newRoot, "", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
		return err
	}

	// create put_old directory
	if err := os.MkdirAll(putOld, 0700); err != nil {
		return err
	}

	// call pivotRoot
	if err := syscall.PivotRoot(newRoot, putOld); err != nil {
		return err
	}

	// Note that this also applies to the calling process: pivotRoot() may
	// or may not affect its current working directory.  It is therefore
	// recommended to call chdir("/") immediately after pivotRoot().
	if err := os.Chdir("/"); err != nil {
		return err
	}

	// umount putOld, which now lives at /.pivot_root
	putOld = "/.pivot_root"
	if err := syscall.Unmount(putOld, syscall.MNT_DETACH); err != nil {
		return err
	}

	// remove putOld
	if err := os.RemoveAll(putOld); err != nil {
		return err
	}

	return nil
}

func mountProc(newRoot string) error {
	target := filepath.Join(newRoot, "/proc")
	os.MkdirAll(target, 0755)
	return syscall.Mount("proc", target, "proc", uintptr(0), "")
}

func justiceInit() {
	newRootPath := os.Args[1]
	input := os.Args[2]
	expected := os.Args[3]
	timeout, _ := strconv.ParseInt(os.Args[4], 10, 32)

	if err := mountProc(newRootPath); err != nil {
		raven.CaptureErrorAndWait(err, map[string]string{"error": "InitContainerFailed"})
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		os.Exit(models.CODE_INIT_CONTAINER_FAILED)
	}

	if err := pivotRoot(newRootPath); err != nil {
		raven.CaptureErrorAndWait(err, map[string]string{"error": "InitContainerFailed"})
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		os.Exit(models.CODE_INIT_CONTAINER_FAILED)
	}

	if err := syscall.Sethostname([]byte("justice")); err != nil {
		raven.CaptureErrorAndWait(err, map[string]string{"error": "InitContainerFailed"})
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		os.Exit(models.CODE_INIT_CONTAINER_FAILED)
	}

	justiceRun(input, expected, int32(timeout))
}

func justiceRun(input, expected string, timeout int32) {
	// for c programs, compiled binary with name [Main] will be located in "/"
	var o, e bytes.Buffer
	cmd := exec.Command("/Main")
	cmd.Stdin = strings.NewReader(input)
	cmd.Stdout = &o
	cmd.Stderr = &e
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
	}
	cmd.Env = []string{"PS1=[justice] # "}

	time.AfterFunc(time.Duration(timeout)*time.Millisecond, func() {
		syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
	})

	// ms
	startTime := time.Now().UnixNano() / int64(time.Millisecond)
	if err := cmd.Run(); err != nil {
		raven.CaptureErrorAndWait(err, map[string]string{"error": "ContainerRunTimeError"})
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		return
	}
	endTime := time.Now().UnixNano() / int64(time.Millisecond)

	if e.Len() > 0 {
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		return
	}

	// MB
	memory := cmd.ProcessState.SysUsage().(*syscall.Rusage).Maxrss / 1024

	output := o.String()
	if output == expected {
		result, _ := json.Marshal(models.GetAccepptedTaskResult(endTime - startTime, memory))
		os.Stdout.Write(result)
	} else {
		result, _ := json.Marshal(models.GetWrongAnswerTaskResult(input, output, expected))
		os.Stdout.Write(result)
	}
}

func main() {
	basedir := flag.String("basedir", "/tmp", "basedir of tmp C binary")
	input := flag.String("input", "", "test case input")
	expected := flag.String("expected", "", "test case expected")
	timeout := flag.String("timeout", "10000", "timeout in milliseconds")
	flag.Parse()

	cmd := reexec.Command("justiceInit", *basedir, *input, *expected, *timeout)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: syscall.CLONE_NEWNS |
			syscall.CLONE_NEWUTS |
			syscall.CLONE_NEWIPC |
			syscall.CLONE_NEWPID |
			syscall.CLONE_NEWNET |
			syscall.CLONE_NEWUSER,
		UidMappings: []syscall.SysProcIDMap{
			{
				ContainerID: 0,
				HostID:      os.Getuid(),
				Size:        1,
			},
		},
		GidMappings: []syscall.SysProcIDMap{
			{
				ContainerID: 0,
				HostID:      os.Getgid(),
				Size:        1,
			},
		},
	}

	if err := cmd.Start(); err != nil {
		os.Exit(models.CODE_CONTAINER_RUNTIME_ERROR)
	}

	if err := cmd.Wait(); err != nil {
		os.Exit(models.CODE_CONTAINER_RUNTIME_ERROR)
	}

	os.Exit(models.CODE_OK)
}
<commit_msg>add trim string logic<commit_after>package main

import (
	"encoding/json"
	"os"
	"os/exec"
	"syscall"
	"github.com/docker/docker/pkg/reexec"
	"path/filepath"
	"github.com/getsentry/raven-go"
	"../../models"
	"../../config"
	"flag"
	"strings"
	"bytes"
	"time"
	"strconv"
)

func init() {
	raven.SetDSN(config.SENTRY_DSN)
	reexec.Register("justiceInit", justiceInit)
	if reexec.Init() {
		os.Exit(models.CODE_OK)
	}
}

func pivotRoot(newRoot string) error {
	putOld := filepath.Join(newRoot, "/.pivot_root")

	// bind mount newRoot to itself - this is a slight hack needed to satisfy requirement (2)
	//
	// The following restrictions apply to new_root and put_old:
	// 1.  They must be directories.
	// 2.  new_root and put_old must not be on the same filesystem as the current root.
	// 3.  put_old must be underneath new_root, that is, adding a nonzero
	//     number of /.. to the string pointed to by put_old must yield the same directory as new_root.
	// 4.  No other filesystem may be mounted on put_old.
	if err := syscall.Mount(newRoot, newRoot, "", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
		return err
	}

	// create put_old directory
	if err := os.MkdirAll(putOld, 0700); err != nil {
		return err
	}

	// call pivotRoot
	if err := syscall.PivotRoot(newRoot, putOld); err != nil {
		return err
	}

	// Note that this also applies to the calling process: pivotRoot() may
	// or may not affect its current working directory.  It is therefore
	// recommended to call chdir("/") immediately after pivotRoot().
	if err := os.Chdir("/"); err != nil {
		return err
	}

	// umount putOld, which now lives at /.pivot_root
	putOld = "/.pivot_root"
	if err := syscall.Unmount(putOld, syscall.MNT_DETACH); err != nil {
		return err
	}

	// remove putOld
	if err := os.RemoveAll(putOld); err != nil {
		return err
	}

	return nil
}

func mountProc(newRoot string) error {
	target := filepath.Join(newRoot, "/proc")
	os.MkdirAll(target, 0755)
	return syscall.Mount("proc", target, "proc", uintptr(0), "")
}

func justiceInit() {
	newRootPath := os.Args[1]
	input := os.Args[2]
	expected := os.Args[3]
	timeout, _ := strconv.ParseInt(os.Args[4], 10, 32)

	if err := mountProc(newRootPath); err != nil {
		raven.CaptureErrorAndWait(err, map[string]string{"error": "InitContainerFailed"})
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		os.Exit(models.CODE_INIT_CONTAINER_FAILED)
	}

	if err := pivotRoot(newRootPath); err != nil {
		raven.CaptureErrorAndWait(err, map[string]string{"error": "InitContainerFailed"})
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		os.Exit(models.CODE_INIT_CONTAINER_FAILED)
	}

	if err := syscall.Sethostname([]byte("justice")); err != nil {
		raven.CaptureErrorAndWait(err, map[string]string{"error": "InitContainerFailed"})
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		os.Exit(models.CODE_INIT_CONTAINER_FAILED)
	}

	justiceRun(input, expected, int32(timeout))
}

func justiceRun(input, expected string, timeout int32) {
	// for c programs, compiled binary with name [Main] will be located in "/"
	var o, e bytes.Buffer
	cmd := exec.Command("/Main")
	cmd.Stdin = strings.NewReader(input)
	cmd.Stdout = &o
	cmd.Stderr = &e
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
	}
	cmd.Env = []string{"PS1=[justice] # "}

	time.AfterFunc(time.Duration(timeout)*time.Millisecond, func() {
		syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
	})

	// ms
	startTime := time.Now().UnixNano() / int64(time.Millisecond)
	if err := cmd.Run(); err != nil {
		raven.CaptureErrorAndWait(err, map[string]string{"error": "ContainerRunTimeError"})
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		return
	}
	endTime := time.Now().UnixNano() / int64(time.Millisecond)

	if e.Len() > 0 {
		result, _ := json.Marshal(models.GetRuntimeErrorTaskResult())
		os.Stdout.Write(result)
		return
	}

	// MB
	memory := cmd.ProcessState.SysUsage().(*syscall.Rusage).Maxrss / 1024

	output := strings.TrimSpace(o.String())
	if output == expected {
		result, _ := json.Marshal(models.GetAccepptedTaskResult(endTime - startTime, memory))
		os.Stdout.Write(result)
	} else {
		result, _ := json.Marshal(models.GetWrongAnswerTaskResult(input, output, expected))
		os.Stdout.Write(result)
	}
}

func main() {
	basedir := flag.String("basedir", "/tmp", "basedir of tmp C binary")
	input := flag.String("input", "", "test case input")
	expected := flag.String("expected", "", "test case expected")
	timeout := flag.String("timeout", "10000", "timeout in milliseconds")
	flag.Parse()

	cmd := reexec.Command("justiceInit", *basedir, *input, *expected, *timeout)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Cloneflags: syscall.CLONE_NEWNS |
			syscall.CLONE_NEWUTS |
			syscall.CLONE_NEWIPC |
			syscall.CLONE_NEWPID |
			syscall.CLONE_NEWNET |
			syscall.CLONE_NEWUSER,
		UidMappings: []syscall.SysProcIDMap{
			{
				ContainerID: 0,
				HostID:      os.Getuid(),
				Size:        1,
			},
		},
		GidMappings: []syscall.SysProcIDMap{
			{
				ContainerID: 0,
				HostID:      os.Getgid(),
				Size:        1,
			},
		},
	}

	if err := cmd.Start(); err != nil {
		os.Exit(models.CODE_CONTAINER_RUNTIME_ERROR)
	}

	if err := cmd.Wait(); err != nil {
		os.Exit(models.CODE_CONTAINER_RUNTIME_ERROR)
	}

	os.Exit(models.CODE_OK)
}
<|endoftext|>


<commit_before>// +build windows

package native

import (
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/itchio/pelican"
	"github.com/itchio/wharf/eos"

	"github.com/itchio/butler/butlerd"
	"github.com/itchio/butler/cmd/elevate"
	"github.com/itchio/butler/configurator"
	"github.com/itchio/butler/endpoints/launch"
	"github.com/itchio/butler/installer"
	"github.com/itchio/butler/runner/winutil"
	"github.com/pkg/errors"
)

type UE4Marker struct {
	SHA256 string
}

func getMarkerPath(params *launch.LauncherParams) string {
	return filepath.Join(params.InstallFolder, ".itch", "ue4-prereqs-marker.txt")
}

func readUE4Marker(params *launch.LauncherParams) (*UE4Marker, error) {
	markerPath := getMarkerPath(params)
	markerBytes, err := ioutil.ReadFile(markerPath)
	if err != nil {
		if os.IsNotExist(err) {
			// ok
			return nil, nil
		}
		return nil, errors.WithMessage(err, "while checking for ue4 prereq marker")
	}

	marker := &UE4Marker{}
	err = json.Unmarshal(markerBytes, marker)
	if err != nil {
		return nil, errors.WithMessage(err, "while decoding ue4 prereq marker")
	}

	return marker, nil
}

func writeUE4Marker(params *launch.LauncherParams, marker *UE4Marker) error {
	payload, err := json.Marshal(marker)
	if err != nil {
		return errors.WithStack(err)
	}

	markerPath := getMarkerPath(params)
	err = os.MkdirAll(filepath.Dir(markerPath), 0755)
	if err != nil {
		return errors.WithStack(err)
	}

	err = ioutil.WriteFile(markerPath, payload, 0644)
	if err != nil {
		return errors.WithStack(err)
	}

	return nil
}

func handleUE4Prereqs(params *launch.LauncherParams) error {
	marker, err := readUE4Marker(params)
	if err != nil {
		return errors.WithStack(err)
	}

	consumer := params.RequestContext.Consumer

	if params.PeInfo != nil {
		prettyPeInfoBytes, err := json.MarshalIndent(params.PeInfo, "", "  ")
		if err == nil {
			consumer.Infof("PE info: %s", string(prettyPeInfoBytes))
		}
	}

	if params.Candidate == nil {
		return nil
	}

	var needle string
	switch params.Candidate.Arch {
	case configurator.Arch386:
		needle = "UE4PrereqSetup_x86.exe"
	case configurator.ArchAmd64:
		needle = "UE4PrereqSetup_x64.exe"
	}

	if needle == "" {
		return nil
	}

	configureRes, err := configurator.Configure(params.InstallFolder, false)
	if err != nil {
		return errors.WithStack(err)
	}

	var prereqCandidate *configurator.Candidate

	for _, c := range configureRes.Candidates {
		base := filepath.Base(c.Path)
		if base == needle {
			prereqCandidate = c
			break
		}
	}

	if prereqCandidate == nil {
		return nil
	}

	consumer.Infof("Found UE4 prereq candidate:\n %s", prereqCandidate)

	prereqCandidatePath := filepath.Join(params.InstallFolder, prereqCandidate.Path)
	f, err := eos.Open(prereqCandidatePath)
	if err != nil {
		return errors.WithStack(err)
	}
	defer f.Close()

	peInfo, err := pelican.Probe(f, &pelican.ProbeParams{
		Consumer: consumer,
	})
	if err != nil {
		return errors.WithStack(err)
	}

	prettyPeInfoBytes, err := json.MarshalIndent(peInfo, "", "  ")
	if err == nil {
		consumer.Infof("Prereq PE info: %s", string(prettyPeInfoBytes))
	}

	var expectedFileDescription string
	switch params.Candidate.Arch {
	case configurator.Arch386:
		expectedFileDescription = "UE4 Prerequisites (x86)"
	case configurator.ArchAmd64:
		expectedFileDescription = "UE4 Prerequisites (x64)"
	}

	fileDescription := peInfo.VersionProperties["FileDescription"]
	if fileDescription != expectedFileDescription {
		consumer.Warnf("Ignoring (%s), was expecting (%s) but got (%s)", prereqCandidate.Path, expectedFileDescription, fileDescription)
		return nil
	}

	_, err = f.Seek(0, io.SeekStart)
	if err != nil {
		return errors.WithMessage(err, "seeking to start of UE4 prereq")
	}

	hash := sha256.New()
	_, err = io.Copy(hash, f)
	if err != nil {
		return errors.WithMessage(err, "computing sha256 of UE4 prereq")
	}

	sha256Bytes := hash.Sum(nil)
	sha256String := fmt.Sprintf("%x", sha256Bytes)

	if marker == nil {
		consumer.Infof("SHA256 of UE4 prereq: %s", sha256String)
		consumer.Infof("No marker")
	} else {
		consumer.Infof("SHA256 of UE4 prereq: %s", sha256String)
		consumer.Infof("SHA256 of UE4 marker: %s", marker.SHA256)
		if sha256String == marker.SHA256 {
			consumer.Infof("UE4 prereqs already installed!")
			if params.ForcePrereqs {
				consumer.Infof("Prereqs forced, installing anyway...")
			} else {
				return nil
			}
		}
	}

	err = winutil.VerifyTrust(prereqCandidatePath)
	if err != nil {
		return errors.WithMessage(err, "while verifying UE4 prereqs signature")
	}

	consumer.Infof("Authenticate signature verified.")
	args := []string{
		"elevate",
		"--",
		prereqCandidatePath,
		"/quiet",
		"/norestart",
	}

	consumer.Infof("Attempting elevated UE4 prereqs install")
	installRes, err := installer.RunSelf(&installer.RunSelfParams{
		Consumer: consumer,
		Args:     args,
	})
	if err != nil {
		return errors.WithStack(err)
	}

	if installRes.ExitCode != 0 {
		if installRes.ExitCode == elevate.ExitCodeAccessDenied {
			msg := "User or system did not grant elevation privileges"
			consumer.Errorf(msg)
			return errors.WithStack(butlerd.CodeOperationAborted)
		}

		consumer.Errorf("UE4 prereq install failed (code %d, 0x%x), we're out of options", installRes.ExitCode, installRes.ExitCode)
		return errors.New("UE4 prereq installation failed")
	}

	err = writeUE4Marker(params, &UE4Marker{
		SHA256: fmt.Sprintf("%x", sha256Bytes),
	})
	if err != nil {
		return errors.WithMessage(err, "while writing ue4 marker")
	}

	return nil
}
<commit_msg>authenticate => authenticode<commit_after>// +build windows

package native

import (
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/itchio/pelican"
	"github.com/itchio/wharf/eos"

	"github.com/itchio/butler/butlerd"
	"github.com/itchio/butler/cmd/elevate"
	"github.com/itchio/butler/configurator"
	"github.com/itchio/butler/endpoints/launch"
	"github.com/itchio/butler/installer"
	"github.com/itchio/butler/runner/winutil"
	"github.com/pkg/errors"
)

type UE4Marker struct {
	SHA256 string
}

func getMarkerPath(params *launch.LauncherParams) string {
	return filepath.Join(params.InstallFolder, ".itch", "ue4-prereqs-marker.txt")
}

func readUE4Marker(params *launch.LauncherParams) (*UE4Marker, error) {
	markerPath := getMarkerPath(params)
	markerBytes, err := ioutil.ReadFile(markerPath)
	if err != nil {
		if os.IsNotExist(err) {
			// ok
			return nil, nil
		}
		return nil, errors.WithMessage(err, "while checking for ue4 prereq marker")
	}

	marker := &UE4Marker{}
	err = json.Unmarshal(markerBytes, marker)
	if err != nil {
		return nil, errors.WithMessage(err, "while decoding ue4 prereq marker")
	}

	return marker, nil
}

func writeUE4Marker(params *launch.LauncherParams, marker *UE4Marker) error {
	payload, err := json.Marshal(marker)
	if err != nil {
		return errors.WithStack(err)
	}

	markerPath := getMarkerPath(params)
	err = os.MkdirAll(filepath.Dir(markerPath), 0755)
	if err != nil {
		return errors.WithStack(err)
	}

	err = ioutil.WriteFile(markerPath, payload, 0644)
	if err != nil {
		return errors.WithStack(err)
	}

	return nil
}

func handleUE4Prereqs(params *launch.LauncherParams) error {
	marker, err := readUE4Marker(params)
	if err != nil {
		return errors.WithStack(err)
	}

	consumer := params.RequestContext.Consumer

	if params.PeInfo != nil {
		prettyPeInfoBytes, err := json.MarshalIndent(params.PeInfo, "", "  ")
		if err == nil {
			consumer.Infof("PE info: %s", string(prettyPeInfoBytes))
		}
	}

	if params.Candidate == nil {
		return nil
	}

	var needle string
	switch params.Candidate.Arch {
	case configurator.Arch386:
		needle = "UE4PrereqSetup_x86.exe"
	case configurator.ArchAmd64:
		needle = "UE4PrereqSetup_x64.exe"
	}

	if needle == "" {
		return nil
	}

	configureRes, err := configurator.Configure(params.InstallFolder, false)
	if err != nil {
		return errors.WithStack(err)
	}

	var prereqCandidate *configurator.Candidate

	for _, c := range configureRes.Candidates {
		base := filepath.Base(c.Path)
		if base == needle {
			prereqCandidate = c
			break
		}
	}

	if prereqCandidate == nil {
		return nil
	}

	consumer.Infof("Found UE4 prereq candidate:\n %s", prereqCandidate)

	prereqCandidatePath := filepath.Join(params.InstallFolder, prereqCandidate.Path)
	f, err := eos.Open(prereqCandidatePath)
	if err != nil {
		return errors.WithStack(err)
	}
	defer f.Close()

	peInfo, err := pelican.Probe(f, &pelican.ProbeParams{
		Consumer: consumer,
	})
	if err != nil {
		return errors.WithStack(err)
	}

	prettyPeInfoBytes, err := json.MarshalIndent(peInfo, "", "  ")
	if err == nil {
		consumer.Infof("Prereq PE info: %s", string(prettyPeInfoBytes))
	}

	var expectedFileDescription string
	switch params.Candidate.Arch {
	case configurator.Arch386:
		expectedFileDescription = "UE4 Prerequisites (x86)"
	case configurator.ArchAmd64:
		expectedFileDescription = "UE4 Prerequisites (x64)"
	}

	fileDescription := peInfo.VersionProperties["FileDescription"]
	if fileDescription != expectedFileDescription {
		consumer.Warnf("Ignoring (%s), was expecting (%s) but got (%s)", prereqCandidate.Path, expectedFileDescription, fileDescription)
		return nil
	}

	_, err = f.Seek(0, io.SeekStart)
	if err != nil {
		return errors.WithMessage(err, "seeking to start of UE4 prereq")
	}

	hash := sha256.New()
	_, err = io.Copy(hash, f)
	if err != nil {
		return errors.WithMessage(err, "computing sha256 of UE4 prereq")
	}

	sha256Bytes := hash.Sum(nil)
	sha256String := fmt.Sprintf("%x", sha256Bytes)

	if marker == nil {
		consumer.Infof("SHA256 of UE4 prereq: %s", sha256String)
		consumer.Infof("No marker")
	} else {
		consumer.Infof("SHA256 of UE4 prereq: %s", sha256String)
		consumer.Infof("SHA256 of UE4 marker: %s", marker.SHA256)
		if sha256String == marker.SHA256 {
			consumer.Infof("UE4 prereqs already installed!")
			if params.ForcePrereqs {
				consumer.Infof("Prereqs forced, installing anyway...")
			} else {
				return nil
			}
		}
	}

	err = winutil.VerifyTrust(prereqCandidatePath)
	if err != nil {
		return errors.WithMessage(err, "while verifying UE4 prereqs signature")
	}

	consumer.Infof("Authenticode signature verified.")
	args := []string{
		"elevate",
		"--",
		prereqCandidatePath,
		"/quiet",
		"/norestart",
	}

	consumer.Infof("Attempting elevated UE4 prereqs install")
	installRes, err := installer.RunSelf(&installer.RunSelfParams{
		Consumer: consumer,
		Args:     args,
	})
	if err != nil {
		return errors.WithStack(err)
	}

	if installRes.ExitCode != 0 {
		if installRes.ExitCode == elevate.ExitCodeAccessDenied {
			msg := "User or system did not grant elevation privileges"
			consumer.Errorf(msg)
			return errors.WithStack(butlerd.CodeOperationAborted)
		}

		consumer.Errorf("UE4 prereq install failed (code %d, 0x%x), we're out of options", installRes.ExitCode, installRes.ExitCode)
		return errors.New("UE4 prereq installation failed")
	}

	err = writeUE4Marker(params, &UE4Marker{
		SHA256: fmt.Sprintf("%x", sha256Bytes),
	})
	if err != nil {
		return errors.WithMessage(err, "while writing ue4 marker")
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import(
	"fmt"
	aspell "github.com/hugbotme/go-aspell"
	s "strings"
	"bytes"
	"regexp"
	"io/ioutil"
)

type spellCheckFileProcessor struct {
	spellChecker aspell.Speller
	stopWords []string
	probableWords []string
}

func newSpellCheckFileProcessor(stopWordsFile string, probableWordsFile string) (spellCheckFileProcessor, error) {
	// Initialize the speller
	speller, err := aspell.NewSpeller(map[string]string{
		"lang": "en_US",
		//"personal": stopWordsFile,
	})
	// jvt: be sure to clean up, C lib being used here....
	defer speller.Delete()

	// jvt: read stop words file to array
	// jvt: @todo error handling?
	stopWordsContent, _ := ioutil.ReadFile(stopWordsFile)

	// jvt: read probable words words file to array
	// jvt: @todo error handling?
	probableWordsContent, _ := ioutil.ReadFile(probableWordsFile)

	return spellCheckFileProcessor{
		spellChecker: speller,
		stopWords: s.Split(string(stopWordsContent), "\n"),
		probableWords: s.Split(string(probableWordsContent), "\n"),
	}, err
}

/**
 * run a spell check on passed content
 * passes back original content if an error occurs
 */
func (spfp spellCheckFileProcessor) processContent (content []byte) string {
	var buffer bytes.Buffer
	var wordBuffer bytes.Buffer
	syntaxNestingLevel := 0
	contentLength := len(content)

	// jvt: start looping content bytes
	for index, b := range content {
		//fmt.Println(string(b))
		if spfp.isMarkdownSyntaxOpeningChar(b) {
			//fmt.Println("entering nesting level")
			syntaxNestingLevel ++
		} else if spfp.isMarkdownSyntaxClosingChar(b) {
			//fmt.Println("leaving nesting level")
			syntaxNestingLevel --

			// jvt: write byte to buffer
			buffer.WriteByte(b)

			// jvt: and continue to next byte
			continue
		}

		// jvt: @todo values under 0 most likely mean invalid markdown, ignoring for now
		if (syntaxNestingLevel > 0) {
			//fmt.Println("in nesting level")
			// jvt: we're ignoring content, just copy
			buffer.WriteByte(b)
		} else {
			var isWordEndingChar bool
			if index > 0 && index < contentLength-2 {
				isWordEndingChar = spfp.isWordEndingChar(b, content[index-1], content[index+1])
			} else {
				isWordEndingChar = spfp.isWordEndingChar(b)
			}

			// jvt: check for end of word
			if wordBuffer.Len() > 0 && isWordEndingChar {
				//fmt.Println("found word " + wordBuffer.String())
				// jvt: process word & write back to buffer
				buffer.WriteString(spfp.processWord(wordBuffer.String()))

				// jvt: reset word buffer
				wordBuffer.Reset()
			} else if !isWordEndingChar {
				//fmt.Println("in word")
				// jvt: we're in a word, copy current byte to word buffer
				wordBuffer.WriteByte(b)
			}

			if (isWordEndingChar) {
				//fmt.Println("word-ending char")
				// jvt: write word-ending byte to buffer
				buffer.WriteByte(b)
			}
		}
	}

	// jvt: and pass back finished string
	return buffer.String()
}

func (spfp spellCheckFileProcessor) isMarkdownSyntaxOpeningChar (b byte) bool {
	char := string(b)
	return char == "[" || char == "(" || char == "`"
}

func (spfp spellCheckFileProcessor) isMarkdownSyntaxClosingChar (b byte) bool {
	char := string(b)
	return char == "]" || char == ")" || char == "´"
}

func (spfp spellCheckFileProcessor) isWordEndingChar (chars ...byte) bool {
	// jvt: @todo huh? byte -> string -> byte array type case is fine, but byte to byte array type cast not? missing something stupid here....
	// jvt: we always get the first param
	char := string(chars[0])

	var matched bool
	if len(chars) > 1 && spfp.isLookForwardAndBackChar(char) {
		//fmt.Println("checking for contraction " + char + string(chars[1]) + string(chars[2]))
		// jvt: try to detect contraction
		matched = spfp.matchLetter(string(chars[1])) && spfp.matchLetter(string(chars[2]))
	} else {
		matched = spfp.matchLetter(char)
	}

	return !matched
}

func (spfp spellCheckFileProcessor) matchLetter (char string) bool {
	matched, _ := regexp.Match("[A-Za-z]", []byte(char))
	return matched
}

func (spfp spellCheckFileProcessor) isLookForwardAndBackChar (char string) bool {
	return char == "'" || char == "-"
}

func (spfp spellCheckFileProcessor) processWord (word string) string {
	// jvt: check for stop word
	if spfp.checkForStopword(word) {
		return word
	}

	spellingCorrect, suggestions := spfp.checkSpelling(word)
	if (spellingCorrect) {
		return word
	} else {
		//fmt.Printf("Incorrect word, suggestions: %s\n", s.Join(suggestions, ", "))

		// jvt: @todo jup....
		if len(suggestions) > 0 {
			//fmt.Printf("suggestions: %s\n", s.Join(suggestions, ", "))
			preferredWord := spfp.checkForPreferred(suggestions)
			fmt.Println("Replacing \"" + word + "\" with \"" + preferredWord + "\"")
			return preferredWord
		}

		return word
	}
}

func (spfp spellCheckFileProcessor) checkForStopword (word string) bool {
	for _, stopword := range spfp.stopWords {
		if (word == stopword) {
			return true
		}
	}

	return false
}

func (spfp spellCheckFileProcessor) checkForPreferred (suggestions []string) string {
	for _, suggestion := range suggestions {
		for _, preferred := range spfp.probableWords {
			if suggestion == preferred {
				fmt.Println("found preferred word: " + preferred)
				return preferred
			}
		}
	}

	// jvt: if we didn't find anything preferred, just return first suggestion
	return suggestions[0]
}

func (spfp spellCheckFileProcessor) checkSpelling (word string) (bool, []string) {
	if spfp.spellChecker.Check(word) {
		//fmt.Print("OK\n")
		return true, nil
	}

	suggestions := spfp.spellChecker.Suggest(word)
	//fmt.Printf("Spelling mistake:\"" + word + "\" suggestions: %s\n", s.Join(suggestions, ", "))
	return false, suggestions
}
<commit_msg>GOFMT VERDAMMT NOCHMAL!<commit_after>package main

import (
	"bytes"
	"fmt"
	aspell "github.com/hugbotme/go-aspell"
	"io/ioutil"
	"regexp"
	s "strings"
)

type spellCheckFileProcessor struct {
	spellChecker  aspell.Speller
	stopWords     []string
	probableWords []string
}

func newSpellCheckFileProcessor(stopWordsFile string, probableWordsFile string) (spellCheckFileProcessor, error) {
	// Initialize the speller
	speller, err := aspell.NewSpeller(map[string]string{
		"lang": "en_US",
		//"personal": stopWordsFile,
	})
	// jvt: be sure to clean up, C lib being used here....
	defer speller.Delete()

	// jvt: read stop words file to array
	// jvt: @todo error handling?
	stopWordsContent, _ := ioutil.ReadFile(stopWordsFile)

	// jvt: read probable words words file to array
	// jvt: @todo error handling?
	probableWordsContent, _ := ioutil.ReadFile(probableWordsFile)

	return spellCheckFileProcessor{
		spellChecker:  speller,
		stopWords:     s.Split(string(stopWordsContent), "\n"),
		probableWords: s.Split(string(probableWordsContent), "\n"),
	}, err
}

/**
 * run a spell check on passed content
 * passes back original content if an error occurs
 */
func (spfp spellCheckFileProcessor) processContent(content []byte) string {
	var buffer bytes.Buffer
	var wordBuffer bytes.Buffer
	syntaxNestingLevel := 0
	contentLength := len(content)

	// jvt: start looping content bytes
	for index, b := range content {
		//fmt.Println(string(b))
		if spfp.isMarkdownSyntaxOpeningChar(b) {
			//fmt.Println("entering nesting level")
			syntaxNestingLevel++
		} else if spfp.isMarkdownSyntaxClosingChar(b) {
			//fmt.Println("leaving nesting level")
			syntaxNestingLevel--

			// jvt: write byte to buffer
			buffer.WriteByte(b)

			// jvt: and continue to next byte
			continue
		}

		// jvt: @todo values under 0 most likely mean invalid markdown, ignoring for now
		if syntaxNestingLevel > 0 {
			//fmt.Println("in nesting level")
			// jvt: we're ignoring content, just copy
			buffer.WriteByte(b)
		} else {
			var isWordEndingChar bool
			if index > 0 && index < contentLength-2 {
				isWordEndingChar = spfp.isWordEndingChar(b, content[index-1], content[index+1])
			} else {
				isWordEndingChar = spfp.isWordEndingChar(b)
			}

			// jvt: check for end of word
			if wordBuffer.Len() > 0 && isWordEndingChar {
				//fmt.Println("found word " + wordBuffer.String())
				// jvt: process word & write back to buffer
				buffer.WriteString(spfp.processWord(wordBuffer.String()))

				// jvt: reset word buffer
				wordBuffer.Reset()
			} else if !isWordEndingChar {
				//fmt.Println("in word")
				// jvt: we're in a word, copy current byte to word buffer
				wordBuffer.WriteByte(b)
			}

			if isWordEndingChar {
				//fmt.Println("word-ending char")
				// jvt: write word-ending byte to buffer
				buffer.WriteByte(b)
			}
		}
	}

	// jvt: and pass back finished string
	return buffer.String()
}

func (spfp spellCheckFileProcessor) isMarkdownSyntaxOpeningChar(b byte) bool {
	char := string(b)
	return char == "[" || char == "(" || char == "`"
}

func (spfp spellCheckFileProcessor) isMarkdownSyntaxClosingChar(b byte) bool {
	char := string(b)
	return char == "]" || char == ")" || char == "´"
}

func (spfp spellCheckFileProcessor) isWordEndingChar(chars ...byte) bool {
	// jvt: @todo huh? byte -> string -> byte array type case is fine, but byte to byte array type cast not? missing something stupid here....
	// jvt: we always get the first param
	char := string(chars[0])

	var matched bool
	if len(chars) > 1 && spfp.isLookForwardAndBackChar(char) {
		//fmt.Println("checking for contraction " + char + string(chars[1]) + string(chars[2]))
		// jvt: try to detect contraction
		matched = spfp.matchLetter(string(chars[1])) && spfp.matchLetter(string(chars[2]))
	} else {
		matched = spfp.matchLetter(char)
	}

	return !matched
}

func (spfp spellCheckFileProcessor) matchLetter(char string) bool {
	matched, _ := regexp.Match("[A-Za-z]", []byte(char))
	return matched
}

func (spfp spellCheckFileProcessor) isLookForwardAndBackChar(char string) bool {
	return char == "'" || char == "-"
}

func (spfp spellCheckFileProcessor) processWord(word string) string {
	// jvt: check for stop word
	if spfp.checkForStopword(word) {
		return word
	}

	spellingCorrect, suggestions := spfp.checkSpelling(word)
	if spellingCorrect {
		return word
	} else {
		//fmt.Printf("Incorrect word, suggestions: %s\n", s.Join(suggestions, ", "))

		// jvt: @todo jup....
		if len(suggestions) > 0 {
			//fmt.Printf("suggestions: %s\n", s.Join(suggestions, ", "))
			preferredWord := spfp.checkForPreferred(suggestions)
			fmt.Println("Replacing \"" + word + "\" with \"" + preferredWord + "\"")
			return preferredWord
		}

		return word
	}
}

func (spfp spellCheckFileProcessor) checkForStopword(word string) bool {
	for _, stopword := range spfp.stopWords {
		if word == stopword {
			return true
		}
	}

	return false
}

func (spfp spellCheckFileProcessor) checkForPreferred(suggestions []string) string {
	for _, suggestion := range suggestions {
		for _, preferred := range spfp.probableWords {
			if suggestion == preferred {
				fmt.Println("found preferred word: " + preferred)
				return preferred
			}
		}
	}

	// jvt: if we didn't find anything preferred, just return first suggestion
	return suggestions[0]
}

func (spfp spellCheckFileProcessor) checkSpelling(word string) (bool, []string) {
	if spfp.spellChecker.Check(word) {
		//fmt.Print("OK\n")
		return true, nil
	}

	suggestions := spfp.spellChecker.Suggest(word)
	//fmt.Printf("Spelling mistake:\"" + word + "\" suggestions: %s\n", s.Join(suggestions, ", "))
	return false, suggestions
}
<|endoftext|>