<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/docker/docker/pkg/jsonmessage"
	"github.com/iron-io/iron_go3/config"
	lambdaImpl "github.com/iron-io/lambda/lambda"
	"github.com/urfave/cli"
)

var availableRuntimes = []string{"nodejs", "python2.7", "java8"}

type lambdaCmd struct {
	settings  config.Settings
	token     *string
	projectID *string
}

type lambdaCreateCmd struct {
	lambdaCmd

	functionName string
	runtime      string
	handler      string
	fileNames    []string
}

func (lcc *lambdaCreateCmd) Config() error {
	return nil
}

type DockerJsonWriter struct {
	under io.Writer
	w     io.Writer
}

func NewDockerJsonWriter(under io.Writer) *DockerJsonWriter {
	r, w := io.Pipe()
	go func() {
		err := jsonmessage.DisplayJSONMessagesStream(r, under, 1, true, nil)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}()
	return &DockerJsonWriter{under, w}
}

func (djw *DockerJsonWriter) Write(p []byte) (int, error) {
	return djw.w.Write(p)
}

func (lcc *lambdaCreateCmd) run(c *cli.Context) error {

	handler := c.String("handler")
	functionName := c.String("name")
	runtime := c.String("runtime")

	lcc.fileNames = c.Args()
	lcc.handler = handler
	lcc.functionName = functionName
	lcc.runtime = runtime

	files := make([]lambdaImpl.FileLike, 0, len(lcc.fileNames))
	opts := lambdaImpl.CreateImageOptions{
		Name:          lcc.functionName,
		Base:          fmt.Sprintf("iron/lambda-%s", lcc.runtime),
		Package:       "",
		Handler:       lcc.handler,
		OutputStream:  NewDockerJsonWriter(os.Stdout),
		RawJSONStream: true,
	}

	if lcc.handler == "" {
		return errors.New("No handler specified.")
	}

	// For Java we allow only 1 file and it MUST be a JAR.
	if lcc.runtime == "java8" {
		if len(lcc.fileNames) != 1 {
			return errors.New("Java Lambda functions can only include 1 file and it must be a JAR file.")
		}

		if filepath.Ext(lcc.fileNames[0]) != ".jar" {
			return errors.New("Java Lambda function package must be a JAR file.")
		}

		opts.Package = filepath.Base(lcc.fileNames[0])
	}

	for _, fileName := range lcc.fileNames {
		file, err := os.Open(fileName)
		defer file.Close()
		if err != nil {
			return err
		}
		files = append(files, file)
	}

	return lambdaImpl.CreateImage(opts, files...)
}

func (lcc *lambdaCreateCmd) getFlags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:        "function-name",
			Usage:       "Name of function. This is usually follows Docker image naming conventions.",
			Destination: &lcc.functionName,
		},
		cli.StringFlag{
			Name:        "runtime",
			Usage:       fmt.Sprintf("Runtime that your Lambda function depends on. Valid values are %s.", strings.Join(availableRuntimes, ", ")),
			Destination: &lcc.runtime,
		},
		cli.StringFlag{
			Name:        "handler",
			Usage:       "function/class that is the entrypoint for this function. Of the form <module name>.<function name> for nodejs/Python, <full class name>::<function name base> for Java.",
			Destination: &lcc.handler,
		},
	}
}

func lambda() cli.Command {
	lcc := lambdaCreateCmd{}
	var flags []cli.Flag

	flags = append(flags, lcc.getFlags()...)
	return cli.Command{
		Name:      "lambda",
		Usage:     "create and publish lambda functions",
		ArgsUsage: "fnclt lambda",
		Subcommands: []cli.Command{
			{
				Name:      "create-function",
				Usage:     `Create Docker image that can run your Lambda function. The files are the contents of the zip file to be uploaded to AWS Lambda.`,
				ArgsUsage: "--function-name NAME --runtime RUNTIME --handler HANDLER file [files...]",
				Action:    lcc.run,
				Flags:     flags,
			},
		},
	}
}
<commit_msg>Fnclt lambda test-function (#209)<commit_after>package main

import (
	"archive/zip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/docker/docker/pkg/jsonmessage"
	"github.com/iron-io/iron_go3/config"
	lambdaImpl "github.com/iron-io/lambda/lambda"
	"github.com/urfave/cli"

	"github.com/aws/aws-sdk-go/aws"
	aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
	aws_session "github.com/aws/aws-sdk-go/aws/session"
	aws_lambda "github.com/aws/aws-sdk-go/service/lambda"
)

var availableRuntimes = []string{"nodejs", "python2.7", "java8"}

type lambdaCmd struct {
	settings  config.Settings
	token     *string
	projectID *string
}

type lambdaCreateCmd struct {
	lambdaCmd

	functionName string
	runtime      string
	handler      string
	fileNames    []string
	payload      string
	clientConext string
	arn          string
	version      string
	downloadOnly bool
	awsProfile   string
	image        string
	awsRegion    string
}

func (lcc *lambdaCreateCmd) Config() error {
	return nil
}

type DockerJsonWriter struct {
	under io.Writer
	w     io.Writer
}

func NewDockerJsonWriter(under io.Writer) *DockerJsonWriter {
	r, w := io.Pipe()
	go func() {
		err := jsonmessage.DisplayJSONMessagesStream(r, under, 1, true, nil)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}()
	return &DockerJsonWriter{under, w}
}

func (djw *DockerJsonWriter) Write(p []byte) (int, error) {
	return djw.w.Write(p)
}

func (lcc *lambdaCreateCmd) getFlags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:        "function-name",
			Usage:       "Name of function. This is usually follows Docker image naming conventions.",
			Destination: &lcc.functionName,
		},
		cli.StringFlag{
			Name:        "runtime",
			Usage:       fmt.Sprintf("Runtime that your Lambda function depends on. Valid values are %s.", strings.Join(availableRuntimes, ", ")),
			Destination: &lcc.runtime,
		},
		cli.StringFlag{
			Name:        "handler",
			Usage:       "function/class that is the entrypoint for this function. Of the form <module name>.<function name> for nodejs/Python, <full class name>::<function name base> for Java.",
			Destination: &lcc.handler,
		},
		cli.StringFlag{
			Name:        "payload",
			Usage:       "Payload to pass to the Lambda function. This is usually a JSON object.",
			Destination: &lcc.payload,
			Value:       "{}",
		},
		cli.StringFlag{
			Name:        "client-context",
			Usage:       "",
			Destination: &lcc.clientConext,
		},

		cli.StringFlag{
			Name:        "image",
			Usage:       "By default the name of the Docker image is the name of the Lambda function. Use this to set a custom name.",
			Destination: &lcc.image,
		},

		cli.StringFlag{
			Name:        "version",
			Usage:       "Version of the function to import.",
			Destination: &lcc.version,
		},

		cli.BoolFlag{
			Name:        "download-only",
			Usage:       "Only download the function into a directory. Will not create a Docker image.",
			Destination: &lcc.downloadOnly,
		},

		cli.StringFlag{
			Name:        "profile",
			Usage:       "AWS Profile to load from credentials file.",
			Destination: &lcc.awsProfile,
		},

		cli.StringFlag{
			Name:        "region",
			Usage:       "AWS region to use.",
			Value:       "us-east-1",
			Destination: &lcc.awsRegion,
		},
	}
}

func (lcc *lambdaCreateCmd) downloadToFile(url string) (string, error) {
	downloadResp, err := http.Get(url)
	if err != nil {
		return "", err
	}
	defer downloadResp.Body.Close()

	// zip reader needs ReaderAt, hence the indirection.
	tmpFile, err := ioutil.TempFile("", "lambda-function-")
	if err != nil {
		return "", err
	}

	io.Copy(tmpFile, downloadResp.Body)
	tmpFile.Close()
	return tmpFile.Name(), nil
}

func (lcc *lambdaCreateCmd) unzipAndGetTopLevelFiles(dst, src string) (files []lambdaImpl.FileLike, topErr error) {
	files = make([]lambdaImpl.FileLike, 0)

	zipReader, err := zip.OpenReader(src)
	if err != nil {
		return files, err
	}
	defer zipReader.Close()

	var fd *os.File
	for _, f := range zipReader.File {
		path := filepath.Join(dst, f.Name)
		fmt.Printf("Extracting '%s' to '%s'\n", f.Name, path)
		if f.FileInfo().IsDir() {
			os.Mkdir(path, 0644)
			// Only top-level dirs go into the list since that is what CreateImage expects.
			if filepath.Dir(f.Name) == filepath.Base(f.Name) {
				fd, topErr = os.Open(path)
				if topErr != nil {
					break
				}
				files = append(files, fd)
			}
		} else {
			// We do not close fd here since we may want to use it to dockerize.
			fd, topErr = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
			if topErr != nil {
				break
			}

			var zipFd io.ReadCloser
			zipFd, topErr = f.Open()
			if topErr != nil {
				break
			}

			_, topErr = io.Copy(fd, zipFd)
			if topErr != nil {
				// OK to skip closing fd here.
				break
			}

			zipFd.Close()

			// Only top-level files go into the list since that is what CreateImage expects.
			if filepath.Dir(f.Name) == "." {
				_, topErr = fd.Seek(0, 0)
				if topErr != nil {
					break
				}

				files = append(files, fd)
			} else {
				fd.Close()
			}
		}
	}
	return
}

func (lcc *lambdaCreateCmd) getFunction() (*aws_lambda.GetFunctionOutput, error) {
	creds := aws_credentials.NewChainCredentials([]aws_credentials.Provider{
		&aws_credentials.EnvProvider{},
		&aws_credentials.SharedCredentialsProvider{
			Filename: "", // Look in default location.
			Profile:  lcc.awsProfile,
		},
	})

	conf := aws.NewConfig().WithCredentials(creds).WithCredentialsChainVerboseErrors(true).WithRegion(lcc.awsRegion)
	sess := aws_session.New(conf)
	conn := aws_lambda.New(sess)
	resp, err := conn.GetFunction(&aws_lambda.GetFunctionInput{
		FunctionName: aws.String(lcc.arn),
		Qualifier:    aws.String(lcc.version),
	})

	return resp, err
}

func (lcc *lambdaCreateCmd) init(c *cli.Context) {
	handler := c.String("handler")
	functionName := c.String("function-name")
	runtime := c.String("runtime")
	clientContext := c.String("client-context")
	payload := c.String("payload")
	version := c.String("version")
	downloadOnly := c.Bool("download-only")
	image := c.String("image")
	profile := c.String("profile")
	region := c.String("region")

	lcc.fileNames = c.Args()
	lcc.handler = handler
	lcc.functionName = functionName
	lcc.runtime = runtime
	lcc.clientConext = clientContext
	lcc.payload = payload
	lcc.version = version
	lcc.downloadOnly = downloadOnly
	lcc.awsProfile = profile
	lcc.image = image
	lcc.awsRegion = region
}

func (lcc *lambdaCreateCmd) create(c *cli.Context) error {
	lcc.init(c)

	files := make([]lambdaImpl.FileLike, 0, len(lcc.fileNames))
	opts := lambdaImpl.CreateImageOptions{
		Name:          lcc.functionName,
		Base:          fmt.Sprintf("iron/lambda-%s", lcc.runtime),
		Package:       "",
		Handler:       lcc.handler,
		OutputStream:  NewDockerJsonWriter(os.Stdout),
		RawJSONStream: true,
	}

	if lcc.handler == "" {
		return errors.New("No handler specified.")
	}

	// For Java we allow only 1 file and it MUST be a JAR.
	if lcc.runtime == "java8" {
		if len(lcc.fileNames) != 1 {
			return errors.New("Java Lambda functions can only include 1 file and it must be a JAR file.")
		}

		if filepath.Ext(lcc.fileNames[0]) != ".jar" {
			return errors.New("Java Lambda function package must be a JAR file.")
		}

		opts.Package = filepath.Base(lcc.fileNames[0])
	}

	for _, fileName := range lcc.fileNames {
		file, err := os.Open(fileName)
		defer file.Close()
		if err != nil {
			return err
		}
		files = append(files, file)
	}

	return lambdaImpl.CreateImage(opts, files...)
}

func (lcc *lambdaCreateCmd) runTest(c *cli.Context) error {
	lcc.init(c)
	exists, err := lambdaImpl.ImageExists(lcc.functionName)
	if err != nil {
		return err
	}
	if !exists {
		return fmt.Errorf("Function %s does not exist.", lcc.functionName)
	}

	// Redirect output to stdout.
	return lambdaImpl.RunImageWithPayload(lcc.functionName, lcc.payload)
}

func (lcc *lambdaCreateCmd) awsImport(c *cli.Context) error {
	lcc.init(c)
	function, err := lcc.getFunction()
	if err != nil {
		return err
	}
	functionName := *function.Configuration.FunctionName

	err = os.Mkdir(fmt.Sprintf("./%s", functionName), os.ModePerm)
	if err != nil {
		return err
	}

	tmpFileName, err := lcc.downloadToFile(*function.Code.Location)
	if err != nil {
		return err
	}
	defer os.Remove(tmpFileName)

	var files []lambdaImpl.FileLike

	if *function.Configuration.Runtime == "java8" {
		fmt.Println("Found Java Lambda function. Going to assume code is a single JAR file.")
		path := filepath.Join(functionName, "function.jar")
		os.Rename(tmpFileName, path)
		fd, err := os.Open(path)
		if err != nil {
			return err
		}

		files = append(files, fd)
	} else {
		files, err = lcc.unzipAndGetTopLevelFiles(functionName, tmpFileName)
		if err != nil {
			return err
		}
	}

	if lcc.downloadOnly {
		// Since we are a command line program that will quit soon, it is OK to
		// let the OS clean `files` up.
		return err
	}

	opts := lambdaImpl.CreateImageOptions{
		Name:          functionName,
		Base:          fmt.Sprintf("iron/lambda-%s", *function.Configuration.Runtime),
		Package:       "",
		Handler:       *function.Configuration.Handler,
		OutputStream:  NewDockerJsonWriter(os.Stdout),
		RawJSONStream: true,
	}

	if lcc.image != "" {
		opts.Name = lcc.image
	}

	if *function.Configuration.Runtime == "java8" {
		opts.Package = filepath.Base(files[0].(*os.File).Name())
	}

	err = lambdaImpl.CreateImage(opts, files...)
	if err != nil {
		return err
	}
	return nil
}

func lambda() cli.Command {
	lcc := lambdaCreateCmd{}
	var flags []cli.Flag

	flags = append(flags, lcc.getFlags()...)

	return cli.Command{
		Name:      "lambda",
		Usage:     "create and publish lambda functions",
		ArgsUsage: "fnclt lambda",
		Subcommands: []cli.Command{
			{
				Name:      "create-function",
				Usage:     `Create Docker image that can run your Lambda function. The files are the contents of the zip file to be uploaded to AWS Lambda.`,
				ArgsUsage: "--function-name NAME --runtime RUNTIME --handler HANDLER file [files...]",
				Action:    lcc.create,
				Flags:     flags,
			},
			{
				Name:      "test-function",
				Usage:     `Runs local Dockerized Lambda function and writes output to stdout.`,
				ArgsUsage: "--function-name NAME [--client-context <value>] [--payload <value>]",
				Action:    lcc.runTest,
				Flags:     flags,
			},
			{
				Name:      "aws-import",
				Usage:     `Converts an existing Lambda function to an image. The function code is downloaded to a directory in the current working directory that has the same name as the Lambda function..`,
				ArgsUsage: "[--region <region>] [--profile <aws profile>] [--version <version>] [--download-only] [--image <name>] ARN",
				Action:    lcc.awsImport,
				Flags:     flags,
			},
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"sync"
	"time"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/go-semver/semver"
	"github.com/coreos/pkg/capnslog"
)

type Capability string

const (
	AuthCapability  Capability = "auth"
	V3rpcCapability Capability = "v3rpc"
)

var (
	plog = capnslog.NewPackageLogger("github.com/coreos/etcd/etcdserver", "api")

	// capabilityMaps is a static map of version to capability map.
	// the base capabilities is the set of capability 2.0 supports.
	capabilityMaps = map[string]map[Capability]bool{
		"2.1.0": {AuthCapability: true},
		"2.2.0": {AuthCapability: true},
		"2.3.0": {AuthCapability: true},
		"3.0.0": {AuthCapability: true, V3rpcCapability: true},
	}

	// capLoopOnce ensures we only create one capability monitor goroutine
	capLoopOnce sync.Once

	enableMapMu sync.RWMutex
	// enabledMap points to a map in capabilityMaps
	enabledMap map[Capability]bool
)

func init() {
	enabledMap = make(map[Capability]bool)
}

// RunCapabilityLoop checks the cluster version every 500ms and updates
// the enabledMap when the cluster version increased.
func RunCapabilityLoop(s *etcdserver.EtcdServer) {
	go capLoopOnce.Do(func() { runCapabilityLoop(s) })
}

func runCapabilityLoop(s *etcdserver.EtcdServer) {
	stopped := s.StopNotify()

	var pv *semver.Version
	for {
		if v := s.ClusterVersion(); v != pv {
			if pv == nil || (v != nil && pv.LessThan(*v)) {
				pv = v
				enableMapMu.Lock()
				enabledMap = capabilityMaps[pv.String()]
				enableMapMu.Unlock()
				plog.Infof("enabled capabilities for version %s", pv)
			}
		}

		select {
		case <-stopped:
			return
		case <-time.After(500 * time.Millisecond):
		}
	}
}

func IsCapabilityEnabled(c Capability) bool {
	enableMapMu.RLock()
	defer enableMapMu.RUnlock()
	if enabledMap == nil {
		return false
	}
	return enabledMap[c]
}

func EnableCapability(c Capability) {
	enableMapMu.Lock()
	defer enableMapMu.Unlock()
	enabledMap[c] = true
}
<commit_msg>etcdserver/api: print only major.minor version API<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"sync"
	"time"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/version"
	"github.com/coreos/go-semver/semver"
	"github.com/coreos/pkg/capnslog"
)

type Capability string

const (
	AuthCapability  Capability = "auth"
	V3rpcCapability Capability = "v3rpc"
)

var (
	plog = capnslog.NewPackageLogger("github.com/coreos/etcd/etcdserver", "api")

	// capabilityMaps is a static map of version to capability map.
	// the base capabilities is the set of capability 2.0 supports.
	capabilityMaps = map[string]map[Capability]bool{
		"2.1.0": {AuthCapability: true},
		"2.2.0": {AuthCapability: true},
		"2.3.0": {AuthCapability: true},
		"3.0.0": {AuthCapability: true, V3rpcCapability: true},
	}

	// capLoopOnce ensures we only create one capability monitor goroutine
	capLoopOnce sync.Once

	enableMapMu sync.RWMutex
	// enabledMap points to a map in capabilityMaps
	enabledMap map[Capability]bool
)

func init() {
	enabledMap = make(map[Capability]bool)
}

// RunCapabilityLoop checks the cluster version every 500ms and updates
// the enabledMap when the cluster version increased.
func RunCapabilityLoop(s *etcdserver.EtcdServer) {
	go capLoopOnce.Do(func() { runCapabilityLoop(s) })
}

func runCapabilityLoop(s *etcdserver.EtcdServer) {
	stopped := s.StopNotify()

	var pv *semver.Version
	for {
		if v := s.ClusterVersion(); v != pv {
			if pv == nil || (v != nil && pv.LessThan(*v)) {
				pv = v
				enableMapMu.Lock()
				enabledMap = capabilityMaps[pv.String()]
				enableMapMu.Unlock()
				plog.Infof("enabled capabilities for version %s", version.Cluster(pv.String()))
			}
		}

		select {
		case <-stopped:
			return
		case <-time.After(500 * time.Millisecond):
		}
	}
}

func IsCapabilityEnabled(c Capability) bool {
	enableMapMu.RLock()
	defer enableMapMu.RUnlock()
	if enabledMap == nil {
		return false
	}
	return enabledMap[c]
}

func EnableCapability(c Capability) {
	enableMapMu.Lock()
	defer enableMapMu.Unlock()
	enabledMap[c] = true
}
<|endoftext|>


<commit_before>package hashmap

import (
	"reflect"
	"sync"
	"sync/atomic"
	"unsafe"
)

const intSizeBytes = 4 << (^uint(0) >> 63)

type (
	hashMapEntry struct {
		key1  uint64
		key2  uint64
		value interface{}
	}

	hashMapData struct {
		andMask uint64
		data    unsafe.Pointer
		size    uint64
		count   uint64
		slice   []*hashMapEntry
	}

	// HashMap implements a read optimized hash map
	HashMap struct {
		mapData unsafe.Pointer
		sync.Mutex
	}
)

// New returns a new HashMap.
func New() *HashMap {
	return NewSize(8)
}

// NewSize returns a new HashMap instance with a specific initialization size.
func NewSize(size uint64) *HashMap {
	hashmap := &HashMap{}
	hashmap.Resize(size)
	return hashmap
}

// Count returns the number of elements within the map.
func (m *HashMap) Count() uint64 {
	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	return atomic.LoadUint64(&mapData.count)
}

// Get retrieves an element from map under given key.
func (m *HashMap) Get(key1 uint64, key2 uint64) (interface{}, bool) {
	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	index := key1 & mapData.andMask
	sliceDataIndexPointer := (*unsafe.Pointer)(unsafe.Pointer(uintptr(mapData.data) + uintptr(index*intSizeBytes)))
	entry := (*hashMapEntry)(atomic.LoadPointer(sliceDataIndexPointer))

	if entry == nil || key1 != entry.key1 || key2 != entry.key2 {
		return nil, false
	}

	return entry.value, true
}

// Set sets the given value under the specified key.
func (m *HashMap) Set(key1 uint64, key2 uint64, value interface{}) {
	m.Lock()
	defer m.Unlock()

	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	index := key1 & mapData.andMask
	sliceDataIndexPointer := (*unsafe.Pointer)(unsafe.Pointer(uintptr(mapData.data) + uintptr(index*intSizeBytes)))
	entry := (*hashMapEntry)(atomic.LoadPointer(sliceDataIndexPointer))

	if entry != nil { // space in slice is used?
		if key1 == entry.key1 && key2 == entry.key2 { // slice entry keys match what we are looking for?
			if value == entry.value { // trying to set the same key and value?
				return
			}
		} else {
			m.Resize(mapData.size + 1) // collision found with shortened key, resize
		}

		for {
			existingEntry := (*hashMapEntry)(atomic.LoadPointer(sliceDataIndexPointer))
			if existingEntry == nil || existingEntry.key1 == key1 { // last resizing operation fixed the collision?
				break
			}
			m.Resize(mapData.size + 1)

			mapData = (*hashMapData)(atomic.LoadPointer(&m.mapData))                                                       // update pointer
			index = key1 & mapData.andMask                                                                                // update index key
			sliceDataIndexPointer = (*unsafe.Pointer)(unsafe.Pointer(uintptr(mapData.data) + uintptr(index*intSizeBytes))) // update index pointer
		}
	}

	entry = &hashMapEntry{ // create a new instance in the update case as well, updating value would not be thread-safe
		key1:  key1,
		key2:  key2,
		value: value,
	}

	atomic.StorePointer((*unsafe.Pointer)(sliceDataIndexPointer), unsafe.Pointer(entry))
	atomic.AddUint64(&mapData.count, 1)
}

// Remove removes an element from the map.
func (m *HashMap) Remove(key1 uint64, key2 uint64) {
	m.Lock()
	defer m.Unlock()

	_, exists := m.Get(key1, key2)
	if !exists {
		return
	}

	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	index := key1 & mapData.andMask

	sliceDataIndexPointer := unsafe.Pointer(uintptr(mapData.data) + uintptr(index*intSizeBytes))
	atomic.StorePointer((*unsafe.Pointer)(sliceDataIndexPointer), nil)
	atomic.AddUint64(&mapData.count, ^uint64(0))
}

// Resize resizes the hashmap to a new size, gets rounded up to next power of 2
// Locking of the hashmap needs to be done outside of this function
func (m *HashMap) Resize(newSize uint64) {
	newSize = roundUpPower2(newSize)
	newSlice := make([]*hashMapEntry, newSize)
	header := (*reflect.SliceHeader)(unsafe.Pointer(&newSlice))

	newMapData := &hashMapData{
		andMask: newSize - 1,
		data:    unsafe.Pointer(header.Data),
		size:    newSize,
		count:   0,
		slice:   newSlice,
	}

	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	if mapData != nil { // copy hashmap contents to new slice with longer key
		newMapData.count = mapData.count
		for _, entry := range mapData.slice {
			if entry == nil {
				continue
			}

			index := entry.key1 & mapData.andMask
			newSlice[index] = entry
		}
	}

	atomic.StorePointer(&m.mapData, unsafe.Pointer(newMapData))
}

// roundUpPower2 rounds a number to the next power of 2.
func roundUpPower2(i uint64) uint64 {
	i--
	i |= i >> 1
	i |= i >> 2
	i |= i >> 4
	i |= i >> 8
	i |= i >> 16
	i |= i >> 32
	i++
	return i
}
<commit_msg>Reuse strconv.IntSize to fix go vet warning<commit_after>package hashmap

import (
	"reflect"
	"strconv"
	"sync"
	"sync/atomic"
	"unsafe"
)

const intSizeBytes = strconv.IntSize >> 3

type (
	hashMapEntry struct {
		key1  uint64
		key2  uint64
		value interface{}
	}

	hashMapData struct {
		andMask uint64
		data    unsafe.Pointer
		size    uint64
		count   uint64
		slice   []*hashMapEntry
	}

	// HashMap implements a read optimized hash map
	HashMap struct {
		mapData unsafe.Pointer
		sync.Mutex
	}
)

// New returns a new HashMap.
func New() *HashMap {
	return NewSize(8)
}

// NewSize returns a new HashMap instance with a specific initialization size.
func NewSize(size uint64) *HashMap {
	hashmap := &HashMap{}
	hashmap.Resize(size)
	return hashmap
}

// Count returns the number of elements within the map.
func (m *HashMap) Count() uint64 {
	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	return atomic.LoadUint64(&mapData.count)
}

// Get retrieves an element from map under given key.
func (m *HashMap) Get(key1 uint64, key2 uint64) (interface{}, bool) {
	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	index := key1 & mapData.andMask
	sliceDataIndexPointer := (*unsafe.Pointer)(unsafe.Pointer(uintptr(mapData.data) + uintptr(index*intSizeBytes)))
	entry := (*hashMapEntry)(atomic.LoadPointer(sliceDataIndexPointer))

	if entry == nil || key1 != entry.key1 || key2 != entry.key2 {
		return nil, false
	}

	return entry.value, true
}

// Set sets the given value under the specified key.
func (m *HashMap) Set(key1 uint64, key2 uint64, value interface{}) {
	m.Lock()
	defer m.Unlock()

	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	index := key1 & mapData.andMask
	sliceDataIndexPointer := (*unsafe.Pointer)(unsafe.Pointer(uintptr(mapData.data) + uintptr(index*intSizeBytes)))
	entry := (*hashMapEntry)(atomic.LoadPointer(sliceDataIndexPointer))

	if entry != nil { // space in slice is used?
		if key1 == entry.key1 && key2 == entry.key2 { // slice entry keys match what we are looking for?
			if value == entry.value { // trying to set the same key and value?
				return
			}
		} else {
			m.Resize(mapData.size + 1) // collision found with shortened key, resize
		}

		for {
			existingEntry := (*hashMapEntry)(atomic.LoadPointer(sliceDataIndexPointer))
			if existingEntry == nil || existingEntry.key1 == key1 { // last resizing operation fixed the collision?
				break
			}
			m.Resize(mapData.size + 1)

			mapData = (*hashMapData)(atomic.LoadPointer(&m.mapData))                                                       // update pointer
			index = key1 & mapData.andMask                                                                                // update index key
			sliceDataIndexPointer = (*unsafe.Pointer)(unsafe.Pointer(uintptr(mapData.data) + uintptr(index*intSizeBytes))) // update index pointer
		}
	}

	entry = &hashMapEntry{ // create a new instance in the update case as well, updating value would not be thread-safe
		key1:  key1,
		key2:  key2,
		value: value,
	}

	atomic.StorePointer((*unsafe.Pointer)(sliceDataIndexPointer), unsafe.Pointer(entry))
	atomic.AddUint64(&mapData.count, 1)
}

// Remove removes an element from the map.
func (m *HashMap) Remove(key1 uint64, key2 uint64) {
	m.Lock()
	defer m.Unlock()

	_, exists := m.Get(key1, key2)
	if !exists {
		return
	}

	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	index := key1 & mapData.andMask

	sliceDataIndexPointer := unsafe.Pointer(uintptr(mapData.data) + uintptr(index*intSizeBytes))
	atomic.StorePointer((*unsafe.Pointer)(sliceDataIndexPointer), nil)
	atomic.AddUint64(&mapData.count, ^uint64(0))
}

// Resize resizes the hashmap to a new size, gets rounded up to next power of 2
// Locking of the hashmap needs to be done outside of this function
func (m *HashMap) Resize(newSize uint64) {
	newSize = roundUpPower2(newSize)
	newSlice := make([]*hashMapEntry, newSize)
	header := (*reflect.SliceHeader)(unsafe.Pointer(&newSlice))

	newMapData := &hashMapData{
		andMask: newSize - 1,
		data:    unsafe.Pointer(header.Data),
		size:    newSize,
		count:   0,
		slice:   newSlice,
	}

	mapData := (*hashMapData)(atomic.LoadPointer(&m.mapData))
	if mapData != nil { // copy hashmap contents to new slice with longer key
		newMapData.count = mapData.count
		for _, entry := range mapData.slice {
			if entry == nil {
				continue
			}

			index := entry.key1 & mapData.andMask
			newSlice[index] = entry
		}
	}

	atomic.StorePointer(&m.mapData, unsafe.Pointer(newMapData))
}

// roundUpPower2 rounds a number to the next power of 2.
func roundUpPower2(i uint64) uint64 {
	i--
	i |= i >> 1
	i |= i >> 2
	i |= i >> 4
	i |= i >> 8
	i |= i >> 16
	i |= i >> 32
	i++
	return i
}
<|endoftext|>


<commit_before>package frame

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/zetamatta/nyagos/commands"
	"github.com/zetamatta/nyagos/dos"
	"github.com/zetamatta/nyagos/shell"
	"github.com/zetamatta/nyagos/texts"
)

// OptionNorc is true, then rcfiles are not executed.
var OptionNorc = false

// OptionGoColorable is true,
// then escape sequences are interpreted by go-colorable library.
var OptionGoColorable = true

// OptionEnableVirtualTerminalProcessing is true,
// then Windows10's ENABLE_VIRTUAL_TERMINAL_PROCESSING is enabled.
var OptionEnableVirtualTerminalProcessing = false

type ScriptEngineForOption interface {
	SetArg([]string)
	RunFile(context.Context, string) ([]byte, error)
	RunString(context.Context, string) error
}

type optionArg struct {
	args []string
	sh   *shell.Shell
	e    ScriptEngineForOption
	ctx  context.Context // ctx is the Context object at parsing
}

type optionT struct {
	F func()
	V func(*optionArg) (func(context.Context) error, error)
	U string
}

var optionMap = map[string]optionT{
	"--lua-first": {
		U: "\"LUACODE\"\nExecute \"LUACODE\" before processing any rcfiles and continue shell",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("--lua-first: requires parameters")
			}
			if err := p.e.RunString(p.ctx, p.args[0]); err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
			}
			return nil, nil
		},
	},
	"--cmd-first": {
		U: "\"COMMAND\"\nExecute \"COMMAND\" before processing any rcfiles and continue shell",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("--cmd-first: requires parameters")
			}
			p.sh.Interpret(p.ctx, p.args[0])
			return nil, nil
		},
	},
	"-k": {
		U: "\"COMMAND\"\nExecute \"COMMAND\" and continue the command-line.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-k: requires parameters")
			}
			return func(ctx context.Context) error {
				p.sh.Interpret(ctx, p.args[0])
				return nil
			}, nil
		},
	},
	"-c": {
		U: "\"COMMAND\"\nExecute `COMMAND` and quit.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-c: requires parameters")
			}
			return func(ctx context.Context) error {
				p.sh.Interpret(ctx, p.args[0])
				return io.EOF
			}, nil
		},
	},
	"-b": {
		U: "\"BASE64edCOMMAND\"\nDecode and execute the command which is encoded with Base64.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-b: requires parameters")
			}
			data, err := base64.StdEncoding.DecodeString(p.args[0])
			if err != nil {
				return nil, err
			}
			text := string(data)
			return func(ctx context.Context) error {
				p.sh.Interpret(ctx, text)
				return io.EOF
			}, nil
		},
	},
	"-f": {
		U: "FILE ARG1 ARG2 ...\n" +
			"If FILE's suffix is .lua, execute Lua-code on it.\n" +
			"The script can refer arguments as `arg[]`.\n" +
			"Otherwise, read and execute commands on it.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-f: requires parameters")
			}
			if strings.HasSuffix(strings.ToLower(p.args[0]), ".lua") {
				// lua script
				return func(ctx context.Context) error {
					p.e.SetArg(p.args)
					_, err := p.e.RunFile(ctx, p.args[0])
					if err != nil {
						return err
					}
					return io.EOF
				}, nil
			}
			return func(ctx context.Context) error {
				// command script
				if err := p.sh.Source(ctx, p.args[0]); err != nil {
					return err
				}
				return io.EOF
			}, nil
		},
	},
	"-e": {
		U: "\"SCRIPTCODE\"\nExecute SCRIPTCODE with Lua interpreter and quit.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-e: requires parameters")
			}
			return func(ctx context.Context) error {
				p.e.SetArg(p.args)
				err := p.e.RunString(ctx, p.args[0])
				if err != nil {
					return err
				}
				return io.EOF
			}, nil
		},
	},
	"--lua-file": {
		U: "FILE ARG1 ARG2...\n" +
			"Execute FILE as Lua Script even if FILE's suffix is not .lua .\n" +
			"The script can refer arguments as `arg[]`.\n" +
			"Lines starting with `@` are ignored to embed into batchfile.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("--lua-file: requires parameters")
			}
			return func(ctx context.Context) error {
				p.e.SetArg(p.args)
				_, err := p.e.RunFile(ctx, p.args[0])
				if err != nil {
					return err
				}
				return io.EOF
			}, nil
		},
	},
	"--show-version-only": {
		U: "\nshow version only",
		V: func(p *optionArg) (func(context.Context) error, error) {
			OptionNorc = true
			return func(context.Context) error {
				fmt.Printf("%s-%s\n", Version, runtime.GOARCH)
				return io.EOF
			}, nil
		},
	},
	"--disable-virtual-terminal-processing": {
		U: "\nDo not use Windows10's native ESCAPE SEQUENCE.",
		F: func() {
			OptionEnableVirtualTerminalProcessing = false
		},
	},
	"--enable-virtual-terminal-processing": {
		U: "\nEnable Windows10's native ESCAPE SEQUENCE.\nIt should be used with `--no-go-colorable`.",
		F: func() {
			OptionEnableVirtualTerminalProcessing = true
		},
	},
	"--no-go-colorable": {
		U: "\nDo not use the ESCAPE SEQUENCE emulation with go-colorable library.",
		F: func() {
			OptionGoColorable = false
		},
	},
	"--go-colorable": {
		U: "\nUse the ESCAPE SEQUENCE emulation with go-colorable library.",
		F: func() {
			OptionGoColorable = true
		},
	},
	"--norc": {
		U: "\nDo not load the startup-scripts: `~\\.nyagos` , `~\\_nyagos`\nand `(BINDIR)\\nyagos.d\\*`.",
		F: func() {
			OptionNorc = true
		},
	},
	"--look-curdir-first": {
		U: "\nSearch for the executable from the current directory before %PATH%.\n(compatible with CMD.EXE)",
		F: func() {
			shell.LookCurdirOrder = dos.LookCurdirFirst
		},
	},
	"--look-curdir-last": {
		U: "\nSearch for the executable from the current directory after %PATH%.\n(compatible with PowerShell)",
		F: func() {
			shell.LookCurdirOrder = dos.LookCurdirLast
		},
	},
	"--look-curdir-never": {
		U: "\nNever search for the executable from the current directory\nunless %PATH% contains.\n(compatible with UNIX Shells)",
		F: func() {
			shell.LookCurdirOrder = dos.LookCurdirNever
		},
	},
}

func Title() {
	fmt.Printf("Nihongo Yet Another GOing Shell %s-%s by %s\n",
		VersionOrStamp(),
		runtime.GOARCH,
		runtime.Version())
	fmt.Println("(c) 2014-2018 NYAOS.ORG <http://www.nyaos.org>")
}

func help(p *optionArg) (func(context.Context) error, error) {
	OptionNorc = true
	return func(context.Context) error {
		Title()
		fmt.Println()
		for _, key := range texts.SortedKeys(optionMap) {
			val := optionMap[key]
			fmt.Printf("%s %s\n", key, strings.Replace(val.U, "\n", "\n\t", -1))
		}

		fmt.Println("\nThese script are called on startup")
		if me, err := os.Executable(); err == nil {
			binDir := filepath.Dir(me)
			nyagosD := filepath.Join(binDir, "nyagos.d")
			fmt.Printf("  %s\\*.lua\n", nyagosD)
			file1 := filepath.Join(binDir, ".nyagos")
			fmt.Printf("  %s (Lua)\n", file1)
			file1 = filepath.Join(binDir, "_nyagos")
			fmt.Printf("  %s (Command-lines)\n", file1)
		}

		home := strings.TrimSpace(os.Getenv("HOME"))
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		file1 := filepath.Join(home, ".nyagos")
		fmt.Printf("  %s (Lua)\n", file1)
		file1 = filepath.Join(home, "_nyagos")
		fmt.Printf("  %s (Command-lines)\n", file1)

		return io.EOF
	}, nil
}

func isDefault(value bool) string {
	if value {
		return " [default]"
	}
	return ""
}

func OptionParse(_ctx context.Context, sh *shell.Shell, e ScriptEngineForOption) (func(context.Context) error, error) {
	args := os.Args[1:]
	optionMap["-h"] = optionT{V: help, U: "\nPrint this usage"}
	optionMap["--help"] = optionT{V: help, U: "\nPrint this usage"}

	for key, val := range commands.BoolOptions {
		_key := strings.Replace(key, "_", "-", -1)
		_val := val
		optionMap["--"+_key] = optionT{
			F: func() {
				*_val.V = true
			},
			U: fmt.Sprintf("(lua: nyagos.option.%s=true)%s\n%s",
				key,
				isDefault(*val.V),
				_val.Usage),
		}
		optionMap["--no-"+_key] = optionT{
			F: func() {
				*_val.V = false
			},
			U: fmt.Sprintf("(lua: nyagos.option.%s=false)%s\n%s",
				key,
				isDefault(!*val.V),
				_val.NoUsage),
		}
	}

	for i := 0; i < len(args); i++ {
		if f, ok := optionMap[args[i]]; ok {
			if f.F != nil {
				f.F()
			}
			if f.V != nil {
				return f.V(&optionArg{
					args: args[i+1:],
					sh:   sh,
					e:    e,
					ctx:  _ctx,
				})
			}
		} else {
			fmt.Fprintf(os.Stderr, "%s: unknown parameter\n", args[i])
		}
	}
	return nil, nil
}

var SilentMode = false
<commit_msg>option -h: modify help about lua<commit_after>package frame

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/zetamatta/nyagos/commands"
	"github.com/zetamatta/nyagos/dos"
	"github.com/zetamatta/nyagos/shell"
	"github.com/zetamatta/nyagos/texts"
)

// OptionNorc is true, then rcfiles are not executed.
var OptionNorc = false

// OptionGoColorable is true,
// then escape sequences are interpreted by go-colorable library.
var OptionGoColorable = true

// OptionEnableVirtualTerminalProcessing is true,
// then Windows10's ENABLE_VIRTUAL_TERMINAL_PROCESSING is enabled.
var OptionEnableVirtualTerminalProcessing = false

type ScriptEngineForOption interface {
	SetArg([]string)
	RunFile(context.Context, string) ([]byte, error)
	RunString(context.Context, string) error
}

type optionArg struct {
	args []string
	sh   *shell.Shell
	e    ScriptEngineForOption
	ctx  context.Context // ctx is the Context object at parsing
}

type optionT struct {
	F func()
	V func(*optionArg) (func(context.Context) error, error)
	U string
}

var optionMap = map[string]optionT{
	"--lua-first": {
		U: "\"LUACODE\"\nExecute \"LUACODE\" before processing any rcfiles and continue shell",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("--lua-first: requires parameters")
			}
			if err := p.e.RunString(p.ctx, p.args[0]); err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
			}
			return nil, nil
		},
	},
	"--cmd-first": {
		U: "\"COMMAND\"\nExecute \"COMMAND\" before processing any rcfiles and continue shell",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("--cmd-first: requires parameters")
			}
			p.sh.Interpret(p.ctx, p.args[0])
			return nil, nil
		},
	},
	"-k": {
		U: "\"COMMAND\"\nExecute \"COMMAND\" and continue the command-line.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-k: requires parameters")
			}
			return func(ctx context.Context) error {
				p.sh.Interpret(ctx, p.args[0])
				return nil
			}, nil
		},
	},
	"-c": {
		U: "\"COMMAND\"\nExecute `COMMAND` and quit.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-c: requires parameters")
			}
			return func(ctx context.Context) error {
				p.sh.Interpret(ctx, p.args[0])
				return io.EOF
			}, nil
		},
	},
	"-b": {
		U: "\"BASE64edCOMMAND\"\nDecode and execute the command which is encoded with Base64.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-b: requires parameters")
			}
			data, err := base64.StdEncoding.DecodeString(p.args[0])
			if err != nil {
				return nil, err
			}
			text := string(data)
			return func(ctx context.Context) error {
				p.sh.Interpret(ctx, text)
				return io.EOF
			}, nil
		},
	},
	"-f": {
		U: "FILE ARG1 ARG2 ...\n" +
			"If FILE's suffix is .lua, execute Lua-code on it.\n" +
			"The script can refer arguments as `arg[]`.\n" +
			"Otherwise, read and execute commands on it.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-f: requires parameters")
			}
			if strings.HasSuffix(strings.ToLower(p.args[0]), ".lua") {
				// lua script
				return func(ctx context.Context) error {
					p.e.SetArg(p.args)
					_, err := p.e.RunFile(ctx, p.args[0])
					if err != nil {
						return err
					}
					return io.EOF
				}, nil
			}
			return func(ctx context.Context) error {
				// command script
				if err := p.sh.Source(ctx, p.args[0]); err != nil {
					return err
				}
				return io.EOF
			}, nil
		},
	},
	"-e": {
		U: "\"SCRIPTCODE\"\nExecute SCRIPTCODE with Lua interpreter and quit.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("-e: requires parameters")
			}
			return func(ctx context.Context) error {
				p.e.SetArg(p.args)
				err := p.e.RunString(ctx, p.args[0])
				if err != nil {
					return err
				}
				return io.EOF
			}, nil
		},
	},
	"--lua-file": {
		U: "FILE ARG1 ARG2...\n" +
			"Execute FILE as Lua Script even if FILE's suffix is not .lua .\n" +
			"The script can refer arguments as `arg[]`.\n" +
			"Lines starting with `@` are ignored to embed into batchfile.",
		V: func(p *optionArg) (func(context.Context) error, error) {
			if len(p.args) <= 0 {
				return nil, errors.New("--lua-file: requires parameters")
			}
			return func(ctx context.Context) error {
				p.e.SetArg(p.args)
				_, err := p.e.RunFile(ctx, p.args[0])
				if err != nil {
					return err
				}
				return io.EOF
			}, nil
		},
	},
	"--show-version-only": {
		U: "\nshow version only",
		V: func(p *optionArg) (func(context.Context) error, error) {
			OptionNorc = true
			return func(context.Context) error {
				fmt.Printf("%s-%s\n", Version, runtime.GOARCH)
				return io.EOF
			}, nil
		},
	},
	"--disable-virtual-terminal-processing": {
		U: "\nDo not use Windows10's native ESCAPE SEQUENCE.",
		F: func() {
			OptionEnableVirtualTerminalProcessing = false
		},
	},
	"--enable-virtual-terminal-processing": {
		U: "\nEnable Windows10's native ESCAPE SEQUENCE.\nIt should be used with `--no-go-colorable`.",
		F: func() {
			OptionEnableVirtualTerminalProcessing = true
		},
	},
	"--no-go-colorable": {
		U: "\nDo not use the ESCAPE SEQUENCE emulation with go-colorable library.",
		F: func() {
			OptionGoColorable = false
		},
	},
	"--go-colorable": {
		U: "\nUse the ESCAPE SEQUENCE emulation with go-colorable library.",
		F: func() {
			OptionGoColorable = true
		},
	},
	"--norc": {
		U: "\nDo not load the startup-scripts: `~\\.nyagos` , `~\\_nyagos`\nand `(BINDIR)\\nyagos.d\\*`.",
		F: func() {
			OptionNorc = true
		},
	},
	"--look-curdir-first": {
		U: "\nSearch for the executable from the current directory before %PATH%.\n(compatible with CMD.EXE)",
		F: func() {
			shell.LookCurdirOrder = dos.LookCurdirFirst
		},
	},
	"--look-curdir-last": {
		U: "\nSearch for the executable from the current directory after %PATH%.\n(compatible with PowerShell)",
		F: func() {
			shell.LookCurdirOrder = dos.LookCurdirLast
		},
	},
	"--look-curdir-never": {
		U: "\nNever search for the executable from the current directory\nunless %PATH% contains.\n(compatible with UNIX Shells)",
		F: func() {
			shell.LookCurdirOrder = dos.LookCurdirNever
		},
	},
}

func Title() {
	fmt.Printf("Nihongo Yet Another GOing Shell %s-%s by %s\n",
		VersionOrStamp(),
		runtime.GOARCH,
		runtime.Version())
	fmt.Println("(c) 2014-2018 NYAOS.ORG <http://www.nyaos.org>")
}

func help(p *optionArg) (func(context.Context) error, error) {
	OptionNorc = true
	return func(context.Context) error {
		Title()
		fmt.Println()
		for _, key := range texts.SortedKeys(optionMap) {
			val := optionMap[key]
			fmt.Printf("%s %s\n", key, strings.Replace(val.U, "\n", "\n\t", -1))
		}

		fmt.Println("\nThese script are called on startup")
		if me, err := os.Executable(); err == nil {
			binDir := filepath.Dir(me)
			nyagosD := filepath.Join(binDir, "nyagos.d")
			fmt.Printf("  %s\\*.lua\n", nyagosD)
			file1 := filepath.Join(binDir, ".nyagos")
			fmt.Printf("  %s (Lua)\n", file1)
			file1 = filepath.Join(binDir, "_nyagos")
			fmt.Printf("  %s (Command-lines)\n", file1)
		}

		home := strings.TrimSpace(os.Getenv("HOME"))
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		file1 := filepath.Join(home, ".nyagos")
		fmt.Printf("  %s (Lua)\n", file1)
		file1 = filepath.Join(home, "_nyagos")
		fmt.Printf("  %s (Command-lines)\n", file1)

		return io.EOF
	}, nil
}

func isDefault(value bool) string {
	if value {
		return " [default]"
	}
	return ""
}

func OptionParse(_ctx context.Context, sh *shell.Shell, e ScriptEngineForOption) (func(context.Context) error, error) {
	args := os.Args[1:]
	optionMap["-h"] = optionT{V: help, U: "\nPrint this usage"}
	optionMap["--help"] = optionT{V: help, U: "\nPrint this usage"}

	for key, val := range commands.BoolOptions {
		_key := strings.Replace(key, "_", "-", -1)
		_val := val
		optionMap["--"+_key] = optionT{
			F: func() {
				*_val.V = true
			},
			U: fmt.Sprintf("(lua: `nyagos.option.%s=true`)%s\n%s",
				key,
				isDefault(*val.V),
				_val.Usage),
		}
		optionMap["--no-"+_key] = optionT{
			F: func() {
				*_val.V = false
			},
			U: fmt.Sprintf("(lua: `nyagos.option.%s=false`)%s\n%s",
				key,
				isDefault(!*val.V),
				_val.NoUsage),
		}
	}

	for i := 0; i < len(args); i++ {
		if f, ok := optionMap[args[i]]; ok {
			if f.F != nil {
				f.F()
			}
			if f.V != nil {
				return f.V(&optionArg{
					args: args[i+1:],
					sh:   sh,
					e:    e,
					ctx:  _ctx,
				})
			}
		} else {
			fmt.Fprintf(os.Stderr, "%s: unknown parameter\n", args[i])
		}
	}
	return nil, nil
}

var SilentMode = false
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

import (
	"net/http"
	"strconv"

	"github.com/dgrijalva/jwt-go"
	"github.com/labstack/echo"
)

var (
	// ErrUnauthorized : HTTP 403 error
	ErrUnauthorized = echo.NewHTTPError(http.StatusForbidden, "")
	// ErrNotFound : HTTP 404 error
	ErrNotFound = echo.NewHTTPError(http.StatusNotFound, "")
	// ErrBadReqBody : HTTP 400 error
	ErrBadReqBody = echo.NewHTTPError(http.StatusBadRequest, "")
	// ErrGatewayTimeout : HTTP 504 error
	ErrGatewayTimeout = echo.NewHTTPError(http.StatusGatewayTimeout, "")
	// ErrInternal : HTTP 500 error
	ErrInternal = echo.NewHTTPError(http.StatusInternalServerError, "")
	// ErrNotImplemented : HTTP 405 error
	ErrNotImplemented = echo.NewHTTPError(http.StatusNotImplemented, "")
	// ErrExists : HTTP Error
	ErrExists = echo.NewHTTPError(http.StatusSeeOther, "")
)

// Get the authenticated user from the JWT Token
func authenticatedUser(c echo.Context) User {
	var u User

	user := c.Get("user").(*jwt.Token)

	claims, ok := user.Claims.(jwt.MapClaims)
	if ok {
		u.Username = claims["username"].(string)
		u.GroupID = int(claims["group_id"].(float64))
		u.Admin = claims["admin"].(bool)
	}

	return u
}

// Returns a filter based on parameters defined on the url stem
func getParamFilter(c echo.Context) map[string]interface{} {
	query := make(map[string]interface{})

	fields := []string{"group", "user", "group", "datacenter"}

	// Process ID's as int's
	for _, field := range fields {
		if val := c.Param(field); val != "" {
			id, err := strconv.Atoi(val)
			if err == nil {
				query["id"] = id
			}
		}
	}

	if c.Param("service") != "" {
		query["name"] = c.Param("service")
	}

	if c.Param("build") != "" {
		query["version"] = c.Param("build")
	}

	return query
}

// Returns a filter based on url query values from the request
func getSearchFilter(c echo.Context) map[string]interface{} {
	query := make(map[string]interface{})

	fields := []string{"id", "user_id", "group_id", "datacenter_id", "service_id"}

	// Process ID's as int's
	for _, field := range fields {
		if val := c.QueryParam(field); val != "" {
			id, err := strconv.Atoi(val)
			if err == nil {
				query[field] = id
			}
		}
	}

	if c.QueryParam("name") != "" {
		query["name"] = c.QueryParam("name")
	}

	return query
}
<commit_msg>build goes to id not version<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

import (
	"net/http"
	"strconv"

	"github.com/dgrijalva/jwt-go"
	"github.com/labstack/echo"
)

var (
	// ErrUnauthorized : HTTP 403 error
	ErrUnauthorized = echo.NewHTTPError(http.StatusForbidden, "")
	// ErrNotFound : HTTP 404 error
	ErrNotFound = echo.NewHTTPError(http.StatusNotFound, "")
	// ErrBadReqBody : HTTP 400 error
	ErrBadReqBody = echo.NewHTTPError(http.StatusBadRequest, "")
	// ErrGatewayTimeout : HTTP 504 error
	ErrGatewayTimeout = echo.NewHTTPError(http.StatusGatewayTimeout, "")
	// ErrInternal : HTTP 500 error
	ErrInternal = echo.NewHTTPError(http.StatusInternalServerError, "")
	// ErrNotImplemented : HTTP 405 error
	ErrNotImplemented = echo.NewHTTPError(http.StatusNotImplemented, "")
	// ErrExists : HTTP Error
	ErrExists = echo.NewHTTPError(http.StatusSeeOther, "")
)

// Get the authenticated user from the JWT Token
func authenticatedUser(c echo.Context) User {
	var u User

	user := c.Get("user").(*jwt.Token)

	claims, ok := user.Claims.(jwt.MapClaims)
	if ok {
		u.Username = claims["username"].(string)
		u.GroupID = int(claims["group_id"].(float64))
		u.Admin = claims["admin"].(bool)
	}

	return u
}

// Returns a filter based on parameters defined on the url stem
func getParamFilter(c echo.Context) map[string]interface{} {
	query := make(map[string]interface{})

	fields := []string{"group", "user", "group", "datacenter"}

	// Process ID's as int's
	for _, field := range fields {
		if val := c.Param(field); val != "" {
			id, err := strconv.Atoi(val)
			if err == nil {
				query["id"] = id
			}
		}
	}

	if c.Param("service") != "" {
		query["name"] = c.Param("service")
	}

	if c.Param("build") != "" {
		query["id"] = c.Param("build")
	}

	return query
}

// Returns a filter based on url query values from the request
func getSearchFilter(c echo.Context) map[string]interface{} {
	query := make(map[string]interface{})

	fields := []string{"id", "user_id", "group_id", "datacenter_id", "service_id"}

	// Process ID's as int's
	for _, field := range fields {
		if val := c.QueryParam(field); val != "" {
			id, err := strconv.Atoi(val)
			if err == nil {
				query[field] = id
			}
		}
	}

	if c.QueryParam("name") != "" {
		query["name"] = c.QueryParam("name")
	}

	return query
}
<|endoftext|>


<commit_before>package dhcp4

import (
	"encoding/binary"
	"net"
	"time"
)

// SelectOrderOrAll has same functionality as SelectOrder, except if the order
// param is nil, whereby all options are added (in arbitary order).
func (o Options) SelectOrderOrAll(order []byte) []Option {
	if order == nil {
		opts := make([]Option, 0, len(o))
		for i, v := range o {
			opts = append(opts, Option{Code: i, Value: v})
		}
		return opts
	}
	return o.SelectOrder(order)
}

// SelectOrder returns a slice of options ordered and selected by a byte array
// usually defined by OptionParameterRequestList.  This result is expected to be
// used in ReplyPacket()'s []Option parameter.
func (o Options) SelectOrder(order []byte) []Option {
	opts := make([]Option, 0, len(order))
	for _, v := range order {
		if data, ok := o[OptionCode(v)]; ok {
			opts = append(opts, Option{Code: OptionCode(v), Value: data})
		}
	}
	return opts
}

// IPRange returns how many ips in the ip range from start to stop (inclusive)
func IPRange(start, stop net.IP) int {
	//return int(Uint([]byte(stop))-Uint([]byte(start))) + 1
	return int(binary.BigEndian.Uint32(stop.To4())) - int(binary.BigEndian.Uint32(start.To4())) + 1
}

// IPAdd returns a copy of start + add.
// IPAdd(net.IP{192,168,1,1},30) returns net.IP{192.168.1.31}
func IPAdd(start net.IP, add int) net.IP { // IPv4 only
	start = start.To4()
	//v := Uvarint([]byte(start))
	result := make(net.IP, 4)
	binary.BigEndian.PutUint32(result, binary.BigEndian.Uint32(start)+uint32(add))
	//PutUint([]byte(result), v+uint64(add))
	return result
}

// IPLess returns where IP a is less than IP b.
func IPLess(a, b net.IP) bool {
	b = b.To4()
	for i, ai := range a.To4() {
		if ai != b[i] {
			return ai < b[i]
		}
	}
	return false
}

// IPInRange returns true if ip is between (inclusive) start and stop.
func IPInRange(start, stop, ip net.IP) bool {
	return !(IPLess(ip, start) || IPLess(stop, ip))
}

// OptionsLeaseTime - converts a time.Duration to a 4 byte slice, compatible
// with OptionIPAddressLeaseTime.
func OptionsLeaseTime(d time.Duration) []byte {
	leaseBytes := make([]byte, 4)
	binary.BigEndian.PutUint32(leaseBytes, uint32(d/time.Second))
	//PutUvarint(leaseBytes, uint64(d/time.Second))
	return leaseBytes
}

// JoinIPs returns a byte slice of IP addresses, one immediately after the other
// This may be useful for creating multiple IP options such as OptionRouter.
func JoinIPs(ips []net.IP) (b []byte) {
	for _, v := range ips {
		b = append(b, v.To4()...)
	}
	return
}

/*
// PutUint writes value to a byte slice.
func PutUint(data []byte, value uint64) {
	for i := len(data) - 1; i >= 0; i-- {
		data[i] = byte(value % 256)
		value /= 256
	}
}

// Uint returns a value from a byte slice.
// Values requiring more than 64bits, won't work correctly
func Uint(data []byte) (ans uint64) {
	for _, b := range data {
		ans <<= 8
		ans += uint64(b)
	}
	return
}*/
<commit_msg>Remove unused commented code<commit_after>package dhcp4

import (
	"encoding/binary"
	"net"
	"time"
)

// SelectOrderOrAll has same functionality as SelectOrder, except if the order
// param is nil, whereby all options are added (in arbitary order).
func (o Options) SelectOrderOrAll(order []byte) []Option {
	if order == nil {
		opts := make([]Option, 0, len(o))
		for i, v := range o {
			opts = append(opts, Option{Code: i, Value: v})
		}
		return opts
	}
	return o.SelectOrder(order)
}

// SelectOrder returns a slice of options ordered and selected by a byte array
// usually defined by OptionParameterRequestList.  This result is expected to be
// used in ReplyPacket()'s []Option parameter.
func (o Options) SelectOrder(order []byte) []Option {
	opts := make([]Option, 0, len(order))
	for _, v := range order {
		if data, ok := o[OptionCode(v)]; ok {
			opts = append(opts, Option{Code: OptionCode(v), Value: data})
		}
	}
	return opts
}

// IPRange returns how many ips in the ip range from start to stop (inclusive)
func IPRange(start, stop net.IP) int {
	//return int(Uint([]byte(stop))-Uint([]byte(start))) + 1
	return int(binary.BigEndian.Uint32(stop.To4())) - int(binary.BigEndian.Uint32(start.To4())) + 1
}

// IPAdd returns a copy of start + add.
// IPAdd(net.IP{192,168,1,1},30) returns net.IP{192.168.1.31}
func IPAdd(start net.IP, add int) net.IP { // IPv4 only
	start = start.To4()
	//v := Uvarint([]byte(start))
	result := make(net.IP, 4)
	binary.BigEndian.PutUint32(result, binary.BigEndian.Uint32(start)+uint32(add))
	//PutUint([]byte(result), v+uint64(add))
	return result
}

// IPLess returns where IP a is less than IP b.
func IPLess(a, b net.IP) bool {
	b = b.To4()
	for i, ai := range a.To4() {
		if ai != b[i] {
			return ai < b[i]
		}
	}
	return false
}

// IPInRange returns true if ip is between (inclusive) start and stop.
func IPInRange(start, stop, ip net.IP) bool {
	return !(IPLess(ip, start) || IPLess(stop, ip))
}

// OptionsLeaseTime - converts a time.Duration to a 4 byte slice, compatible
// with OptionIPAddressLeaseTime.
func OptionsLeaseTime(d time.Duration) []byte {
	leaseBytes := make([]byte, 4)
	binary.BigEndian.PutUint32(leaseBytes, uint32(d/time.Second))
	//PutUvarint(leaseBytes, uint64(d/time.Second))
	return leaseBytes
}

// JoinIPs returns a byte slice of IP addresses, one immediately after the other
// This may be useful for creating multiple IP options such as OptionRouter.
func JoinIPs(ips []net.IP) (b []byte) {
	for _, v := range ips {
		b = append(b, v.To4()...)
	}
	return
}
<|endoftext|>


<commit_before>package main

// Given a file with filenames, this times how fast we can stat them
// in parallel.  This is useful for benchmarking purposes.

import (
	"os"
	"flag"
	"time"
	"fmt"
	"encoding/line"
)

func main() {
	flag.Parse()

	filename := flag.Args()[0]
	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		panic("err" + err.String())
	}

	linelen := 1000
	reader := line.NewReader(f, linelen)

	files := make([]string, 0)
	for {
		l, _, err := reader.ReadLine()
		if err != nil {
			break
		}
		files = append(files, string(l))
	}

	parallel := 10
	todo := make(chan string, len(files))
	dts := make(chan int64, parallel)

	fmt.Printf("Statting %d files with %d threads\n", len(files), parallel)
	for i := 0; i < parallel; i++ {
		go func() {
			for {
				fn := <-todo
				if fn == "" {
					break
				}

				t := time.Nanoseconds()
				os.Lstat(fn)
				dts <- time.Nanoseconds() - t
			}
		}()
	}

	for _, v := range files {
		todo <- v
	}

	total := 0.0
	for i := 0; i < len(files); i++ {
		total += float64(<-dts) * 1e-6
	}

	fmt.Println("Average stat time (ms):", total/float64(len(files)))
}
<commit_msg>Do a couple of runs and show the average.<commit_after>package main

// Given a file with filenames, this times how fast we can stat them
// in parallel.  This is useful for benchmarking purposes.

import (
	"os"
	"flag"
	"time"
	"fmt"
	"encoding/line"
)

func main() {
	flag.Parse()

	filename := flag.Args()[0]
	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		panic("err" + err.String())
	}

	linelen := 1000
	reader := line.NewReader(f, linelen)

	files := make([]string, 0)
	for {
		l, _, err := reader.ReadLine()
		if err != nil {
			break
		}
		files = append(files, string(l))
	}

	runs := 10
	tot := 0.0
	sleeptime := 4.0
	for j := 0; j < runs; j++ {
		tot += BulkStat(10, files)
		fmt.Printf("Sleeping %.2f seconds\n", sleeptime)
		time.Sleep(int64(sleeptime * 1e9))
	}

	fmt.Printf("Average of %d runs: %f ms\n", runs, tot/float64(runs))
}

func BulkStat(parallelism int, files []string) float64 {
	parallel := 10
	todo := make(chan string, len(files))
	dts := make(chan int64, parallel)

	allStart := time.Nanoseconds()

	fmt.Printf("Statting %d files with %d threads\n", len(files), parallel)
	for i := 0; i < parallel; i++ {
		go func() {
			for {
				fn := <-todo
				if fn == "" {
					break
				}

				t := time.Nanoseconds()
				os.Lstat(fn)
				dts <- time.Nanoseconds() - t
			}
		}()
	}

	for _, v := range files {
		todo <- v
	}

	total := 0.0
	for i := 0; i < len(files); i++ {
		total += float64(<-dts) * 1e-6
	}

	allEnd := time.Nanoseconds()
	avg := total/float64(len(files))

	fmt.Printf("Elapsed: %f sec. Average stat %f ms\n",
		float64(allEnd-allStart)*1e-9, avg)

	return avg
}
<|endoftext|>


<commit_before>// Copyright 2017 bs authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package log

import (
	"encoding/json"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/Graylog2/go-gelf/gelf"
	"github.com/tsuru/bs/bslog"
	"github.com/tsuru/bs/config"
)

type gelfBackend struct {
	extra           json.RawMessage
	host            string
	fieldsWhitelist []string
	msgCh           chan<- LogMessage
	quitCh          chan<- bool
	nextNotify      *time.Timer
}

func (b *gelfBackend) initialize() error {
	bufferSize := config.IntEnvOrDefault(config.DefaultBufferSize, "LOG_GELF_BUFFER_SIZE", "LOG_BUFFER_SIZE")
	b.host = config.StringEnvOrDefault("localhost:12201", "LOG_GELF_HOST")
	extra := config.StringEnvOrDefault("", "LOG_GELF_EXTRA_TAGS")
	if extra != "" {
		data := map[string]interface{}{}
		if err := json.Unmarshal([]byte(extra), &data); err != nil {
			bslog.Warnf("unable to parse gelf extra tags: %s", err)
		} else {
			b.extra = json.RawMessage(extra)
		}
	}
	b.fieldsWhitelist = config.StringsEnvOrDefault([]string{
		"request_id",
		"request_time",
		"request_uri",
		"status",
		"method",
		"uri",
	}, "LOG_GELF_FIELDS_WHITELIST")
	b.nextNotify = time.NewTimer(0)
	var err error
	b.msgCh, b.quitCh, err = processMessages(b, bufferSize)
	if err != nil {
		return err
	}
	return nil
}

func (b *gelfBackend) sendMessage(parts *rawLogParts, appName, processName, container string) {
	level := gelf.LOG_INFO
	if s, err := strconv.Atoi(string(parts.priority)); err == nil {
		if int32(s)&gelf.LOG_ERR == gelf.LOG_ERR {
			level = gelf.LOG_ERR
		}
	}
	msg := &gelf.Message{
		Version: "1.1",
		Host:    container,
		Short:   string(parts.content),
		Level:   level,
		Extra: map[string]interface{}{
			"_app": appName,
			"_pid": processName,
		},
		RawExtra: b.extra,
	}
	select {
	case b.msgCh <- msg:
	default:
		select {
		case <-b.nextNotify.C:
			bslog.Errorf("Dropping log messages to gelf due to full channel buffer.")
			b.nextNotify.Reset(time.Minute)
		default:
		}
	}
}
func (b *gelfBackend) stop() {
	close(b.quitCh)
}

type gelfConnWrapper struct {
	net.Conn
	*gelf.Writer
}

func (w *gelfConnWrapper) Close() error {
	return w.Writer.Close()
}

func (w *gelfConnWrapper) Write(msg []byte) (int, error) {
	return 0, nil
}

func (b *gelfBackend) connect() (net.Conn, error) {
	writer, err := gelf.NewWriter(b.host)
	if err != nil {
		return nil, err
	}
	writer.CompressionType = gelf.CompressNone
	return &gelfConnWrapper{Writer: writer}, nil
}

func (b *gelfBackend) parseFields(gelfMsg *gelf.Message) {
	shortMsg := gelfMsg.Short
	if !strings.Contains(shortMsg, "=") {
		return
	}
	for _, field := range b.fieldsWhitelist {
		value := findFieldInMsg(shortMsg, field)
		if value == "" {
			continue
		}
		gelfMsg.Extra["_"+field] = value
	}

	level := strings.ToUpper(findFieldInMsg(shortMsg, "level"))

	switch level {
	case "EMERG", "PANIC":
		gelfMsg.Level = gelf.LOG_EMERG
	case "ALERT":
		gelfMsg.Level = gelf.LOG_ALERT
	case "CRIT", "CRITICAL", "FATAL":
		gelfMsg.Level = gelf.LOG_CRIT
	case "ERR", "ERROR":
		gelfMsg.Level = gelf.LOG_ERR
	case "WARN", "WARNING":
		gelfMsg.Level = gelf.LOG_WARNING
	case "NOTICE":
		gelfMsg.Level = gelf.LOG_NOTICE
	case "INFO":
		gelfMsg.Level = gelf.LOG_INFO
	case "DEBUG":
		gelfMsg.Level = gelf.LOG_DEBUG
	}
}

func (b *gelfBackend) process(conn net.Conn, msg LogMessage) error {
	gelfMsg := msg.(*gelf.Message)
	b.parseFields(gelfMsg)
	return conn.(*gelfConnWrapper).WriteMessage(gelfMsg)
}

func (b *gelfBackend) close(conn net.Conn) {
	conn.Close()
}

func findFieldInMsg(msg, field string) string {
	idx := strings.Index(msg, field+"=")
	if idx == -1 {
		return ""
	}
	if idx > 0 {
		switch msg[idx-1] {
		case ' ', '\t':
		default:
			return ""
		}
	}
	idx += len(field) + 1
	end := strings.IndexAny(msg[idx:], " \t")
	if end == -1 {
		end = len(msg) - idx
	}
	return msg[idx : idx+end]
}
<commit_msg>log: add timestamp do gelf message<commit_after>// Copyright 2017 bs authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package log

import (
	"encoding/json"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/Graylog2/go-gelf/gelf"
	"github.com/tsuru/bs/bslog"
	"github.com/tsuru/bs/config"
)

type gelfBackend struct {
	extra           json.RawMessage
	host            string
	fieldsWhitelist []string
	msgCh           chan<- LogMessage
	quitCh          chan<- bool
	nextNotify      *time.Timer
}

func (b *gelfBackend) initialize() error {
	bufferSize := config.IntEnvOrDefault(config.DefaultBufferSize, "LOG_GELF_BUFFER_SIZE", "LOG_BUFFER_SIZE")
	b.host = config.StringEnvOrDefault("localhost:12201", "LOG_GELF_HOST")
	extra := config.StringEnvOrDefault("", "LOG_GELF_EXTRA_TAGS")
	if extra != "" {
		data := map[string]interface{}{}
		if err := json.Unmarshal([]byte(extra), &data); err != nil {
			bslog.Warnf("unable to parse gelf extra tags: %s", err)
		} else {
			b.extra = json.RawMessage(extra)
		}
	}
	b.fieldsWhitelist = config.StringsEnvOrDefault([]string{
		"request_id",
		"request_time",
		"request_uri",
		"status",
		"method",
		"uri",
	}, "LOG_GELF_FIELDS_WHITELIST")
	b.nextNotify = time.NewTimer(0)
	var err error
	b.msgCh, b.quitCh, err = processMessages(b, bufferSize)
	if err != nil {
		return err
	}
	return nil
}

func (b *gelfBackend) sendMessage(parts *rawLogParts, appName, processName, container string) {
	level := gelf.LOG_INFO
	if s, err := strconv.Atoi(string(parts.priority)); err == nil {
		if int32(s)&gelf.LOG_ERR == gelf.LOG_ERR {
			level = gelf.LOG_ERR
		}
	}
	msg := &gelf.Message{
		Version: "1.1",
		Host:    container,
		Short:   string(parts.content),
		Level:   level,
		Extra: map[string]interface{}{
			"_app": appName,
			"_pid": processName,
		},
		RawExtra: b.extra,
		TimeUnix: float64(time.Now().UnixNano()) / float64(time.Second),
	}
	select {
	case b.msgCh <- msg:
	default:
		select {
		case <-b.nextNotify.C:
			bslog.Errorf("Dropping log messages to gelf due to full channel buffer.")
			b.nextNotify.Reset(time.Minute)
		default:
		}
	}
}
func (b *gelfBackend) stop() {
	close(b.quitCh)
}

type gelfConnWrapper struct {
	net.Conn
	*gelf.Writer
}

func (w *gelfConnWrapper) Close() error {
	return w.Writer.Close()
}

func (w *gelfConnWrapper) Write(msg []byte) (int, error) {
	return 0, nil
}

func (b *gelfBackend) connect() (net.Conn, error) {
	writer, err := gelf.NewWriter(b.host)
	if err != nil {
		return nil, err
	}
	writer.CompressionType = gelf.CompressNone
	return &gelfConnWrapper{Writer: writer}, nil
}

func (b *gelfBackend) parseFields(gelfMsg *gelf.Message) {
	shortMsg := gelfMsg.Short
	if !strings.Contains(shortMsg, "=") {
		return
	}
	for _, field := range b.fieldsWhitelist {
		value := findFieldInMsg(shortMsg, field)
		if value == "" {
			continue
		}
		gelfMsg.Extra["_"+field] = value
	}

	level := strings.ToUpper(findFieldInMsg(shortMsg, "level"))

	switch level {
	case "EMERG", "PANIC":
		gelfMsg.Level = gelf.LOG_EMERG
	case "ALERT":
		gelfMsg.Level = gelf.LOG_ALERT
	case "CRIT", "CRITICAL", "FATAL":
		gelfMsg.Level = gelf.LOG_CRIT
	case "ERR", "ERROR":
		gelfMsg.Level = gelf.LOG_ERR
	case "WARN", "WARNING":
		gelfMsg.Level = gelf.LOG_WARNING
	case "NOTICE":
		gelfMsg.Level = gelf.LOG_NOTICE
	case "INFO":
		gelfMsg.Level = gelf.LOG_INFO
	case "DEBUG":
		gelfMsg.Level = gelf.LOG_DEBUG
	}
}

func (b *gelfBackend) process(conn net.Conn, msg LogMessage) error {
	gelfMsg := msg.(*gelf.Message)
	b.parseFields(gelfMsg)
	return conn.(*gelfConnWrapper).WriteMessage(gelfMsg)
}

func (b *gelfBackend) close(conn net.Conn) {
	conn.Close()
}

func findFieldInMsg(msg, field string) string {
	idx := strings.Index(msg, field+"=")
	if idx == -1 {
		return ""
	}
	if idx > 0 {
		switch msg[idx-1] {
		case ' ', '\t':
		default:
			return ""
		}
	}
	idx += len(field) + 1
	end := strings.IndexAny(msg[idx:], " \t")
	if end == -1 {
		end = len(msg) - idx
	}
	return msg[idx : idx+end]
}
<|endoftext|>


<commit_before>package dbf

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

type StructCrack struct {
	Id         string            `json:"id"`
	Generator  string            `json:"generator"`
	Gen_config []string          `json:"gen_config"`
	Algo_id    string            `json:"algo_id"`
	Algo_name  string            `json:"algo_name"`
	Len_min    string            `json:"len_min"`
	Len_max    string            `json:"len_max"`
	Charset1   string            `json:"charset1"`
	Charset2   string            `json:"charset2"`
	Charset3   string            `json:"charset3"`
	Charset4   string            `json:"charset4"`
	Mask       string            `json:"mask"`
	Target     string            `json:"target"`
	Has_dep    bool              `json:"has_dep"`
	info       []StructCrackInfo `json:"has_dep"`
}

type StructCrackInfo struct {
	Platform     string `json:"platform"`
	Cracker      string `json:"cracker"`
	Internal_gen bool   `json:"internal_gen"`
}

type StructCrackerNEmbed struct {
	Stdin  []string `json:"stdin"`
	Infile []string `json:"infile"`
}

type StructCrackerEmbed struct {
	Generator []StructCrackerGen `json:"generator"`
}

type StructCrackerGen struct {
	Name string   `json:"name"`
	Args []string `json:"args"`
}

func processCrack(task *StructCrackTask, crackInfoPath *string) bool {
	var vendorPath, cmdJsonStr string
	var cmdArg []string
	var resultStatus int

	resultStatus = -1
	taskPath := getPath(_PATH_TASK) + task.Platform + PATH_SEPARATOR

	defer func() {
		resultByte, err := ioutil.ReadFile(taskPath + "result")
		if err != nil {
			resultByte = nil

			if os.IsNotExist(err) {
				resultStatus = 0 // Was not cracked
			} else {
				Log.Printf("%s\n", err)
				resultStatus = -2
			}
		}

		fmt.Printf("Sending result of crack #%s...\n", task.Crack_id)

		if sendResult(`[{"crack_id":"`+task.Crack_id+`","start":"`+task.Start+`","offset":"`+task.Offset+`","result":"`+string(resultByte)+`","status":"`+strconv.Itoa(resultStatus)+`"}]`) == true {
			fmt.Printf("Removing task info of crack #%s (%s)...\n", task.Crack_id, task.Platform)
			err = os.RemoveAll(taskPath)
			if err != nil {
				Log.Printf("%s\n", err)
			}
		}
	}()

	crackJson, err := ioutil.ReadFile(*crackInfoPath)
	if err != nil {
		Log.Printf("%s\n", err)
		resultStatus = -3
		return false
	}

	var crack StructCrack
	err = json.Unmarshal(crackJson, &crack)
	if err != nil {
		Log.Printf("%s\n", err)
		resultStatus = -4
		return false
	}

	/* Process crack */
	// Check cracker
	var cracker string
	var internal_gen bool
	for _, crackInfo := range crack.info {
		if crackInfo.Platform == task.Platform {
			cracker = crackInfo.Cracker
			internal_gen = crackInfo.Internal_gen
			break
		}
	}

	if cracker == "" {
		resultStatus = -0
		return false
	}

	vendorPath = getPath(_PATH_VENDOR) + _VENDOR_TYPE_CRACKER + PATH_SEPARATOR + cracker + PATH_SEPARATOR + task.Platform + PATH_SEPARATOR + _VENDOR_TYPE_CRACKER + extExecutable
	if checkVendor(_VENDOR_TYPE_CRACKER, &cracker, &task.Platform, &vendorPath) == false {
		resultStatus = -5
		return false
	}

	// Check hashfile
	*crackInfoPath = filepath.Dir(*crackInfoPath) + PATH_SEPARATOR + "hashfile"
	if _, err := os.Stat(*crackInfoPath); err != nil {
		if os.IsNotExist(err) { // Does not exist, so create it
			err = ioutil.WriteFile(*crackInfoPath, []byte(crack.Target), 0664)
			if err != nil {
				Log.Printf("%s\n", err) // Error in creating
				resultStatus = -6
				return false
			}
		} else {
			Log.Printf("%s\n", err) // Error in accessing
			resultStatus = -7
			return false
		}
	}

	generatorReplacer := strings.NewReplacer("START", task.Start, "OFFSET", task.Offset, `"IN_FILE"`, strconv.Quote(taskPath+"file.fifo"))
	crackerReplacer := strings.NewReplacer(`"HASH_FILE"`, strconv.Quote(*crackInfoPath), `"OUT_FILE"`, strconv.Quote(taskPath+"result"), `"IN_FILE"`, strconv.Quote(taskPath+"file.fifo"))

	if internal_gen { // Embeded
		vendorPath = filepath.Dir(vendorPath) + PATH_SEPARATOR + "info.json"
		crackerEmbedJson, err := ioutil.ReadFile(vendorPath)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -0
			return false
		}

		var crackerEmbed StructCrackerEmbed
		err = json.Unmarshal(crackerEmbedJson, &crackerEmbed)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -0
			return false
		}

		brk := false
		for _, crackerGen := range crackerEmbed.Generator {
			if crackerGen.Name == crack.Generator {
				cmdArg = crackerGen.Args
				brk = true
				break
			}
		}
		if brk == false {
			Log.Printf("No args for cracker '%s' in info.json!\n", cracker)
			resultStatus = -0
			return false
		}

		cmdJsonByte, err := json.Marshal(&cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -0
			return false
		}
		cmdJsonStr = string(cmdJsonByte)

		cmdJsonStr = generatorReplacer.Replace(cmdJsonStr)
		cmdJsonStr = crackerReplacer.Replace(cmdJsonStr)
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -8
			return false
		}

		fmt.Printf("Performing crack #%s...\n", task.Crack_id)

		vendorPath = filepath.Dir(vendorPath) + PATH_SEPARATOR + _VENDOR_TYPE_CRACKER + extExecutable
		err = exec.Command(vendorPath, cmdArg...).Run()
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -9
			return false
		} else {
			resultStatus = 0
			return true
		}
	} else { // Not embeded
		// Prepare cracker
		cmdJsonStr = crackerReplacer.Replace(crack.Cmd_cracker)
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -10
			return false
		}
		execCracker := exec.Command(vendorPath, cmdArg...)

		// Check generator
		vendorPath = getPath(_PATH_VENDOR) + _VENDOR_TYPE_GENERATOR + PATH_SEPARATOR + crack.Generator + PATH_SEPARATOR + task.Platform + PATH_SEPARATOR + _VENDOR_TYPE_GENERATOR + extExecutable
		if checkVendor(_VENDOR_TYPE_GENERATOR, &crack.Generator, &task.Platform, &vendorPath) == false {
			resultStatus = -11
			return false
		}

		// Prepare generator
		cmdJsonStr = generatorReplacer.Replace(crack.Cmd_generator)
		if strings.Contains(cmdJsonStr, "DEP_GEN") {
			// Check if dependency exists in crack location
			*crackInfoPath = filepath.Dir(*crackInfoPath) + PATH_SEPARATOR + "dep" + PATH_SEPARATOR + "dep-gen"
			if _, err := os.Stat(*crackInfoPath); err == nil { // dep-gen file exists in crack location and is accessible
				cmdJsonStr = strings.Replace(cmdJsonStr, `"DEP_GEN"`, strconv.Quote(*crackInfoPath), -1)
			} else { // Check if dependency exists in generator location
				vendorPath = filepath.Dir(vendorPath) + PATH_SEPARATOR + "dep-gen"
				if _, err := os.Stat(vendorPath); err == nil { // dep-gen file exists in generator location and is accessible
					cmdJsonStr = strings.Replace(cmdJsonStr, `"DEP_GEN"`, strconv.Quote(vendorPath), -1)
				} else {
					resultStatus = -12
					return false
				}
				vendorPath = filepath.Dir(vendorPath) + PATH_SEPARATOR + _VENDOR_TYPE_GENERATOR + extExecutable // Rename back to generator executable
			}
		}
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -13
			return false
		}
		execGenerator := exec.Command(vendorPath, cmdArg...)

		fmt.Printf("Performing crack #%s...\n", task.Crack_id)

		if crack.Type == "stdin" {
			r, err := execGenerator.StdoutPipe()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -14
				return false
			}
			execCracker.Stdin = r

			err = execGenerator.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -15
				return false
			}

			err = execCracker.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -16
				return false
			}

			execCracker.Wait()
			execGenerator.Process.Signal(syscall.SIGINT) // ^C (Control-C)
			r.Close()

			resultStatus = 0
			return true
		} else { // Infile
			err = exec.Command("mkfifo", taskPath+"file.fifo").Run()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -18
				return false
			}

			err = execGenerator.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -19
				return false
			}

			err = execCracker.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -20
				return false
			}

			execCracker.Wait()
			execGenerator.Process.Signal(syscall.SIGINT) // ^C (Control-C)

			resultStatus = 0
			return true
		}
	}

	return true
}
<commit_msg>Embedded cracking is ready<commit_after>package dbf

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	//	"syscall"
)

type StructCrack struct {
	Id         string            `json:"id"`
	Generator  string            `json:"generator"`
	Gen_config []string          `json:"gen_config"`
	Algo_id    string            `json:"algo_id"`
	Algo_name  string            `json:"algo_name"`
	Len_min    string            `json:"len_min"`
	Len_max    string            `json:"len_max"`
	Charset1   string            `json:"charset1"`
	Charset2   string            `json:"charset2"`
	Charset3   string            `json:"charset3"`
	Charset4   string            `json:"charset4"`
	Mask       string            `json:"mask"`
	Target     string            `json:"target"`
	Has_dep    bool              `json:"has_dep"`
	Info       []StructCrackInfo `json:"info"`
}

type StructCrackInfo struct {
	Platform     string `json:"platform"`
	Cracker      string `json:"cracker"`
	Internal_gen bool   `json:"internal_gen"`
}

type StructCrackerNEmbed struct {
	Stdin  []string `json:"stdin"`
	Infile []string `json:"infile"`
}

type StructCrackerEmbed struct {
	Generator []StructCrackerGen `json:"generator"`
}

type StructCrackerGen struct {
	Name string   `json:"name"`
	Arg  []string `json:"arg"`
}

func processCrack(task *StructCrackTask, crackInfoPath *string) bool {
	var vendorPath, cmdJsonStr string
	var cmdArg []string
	var resultStatus int

	resultStatus = -1
	taskPath := getPath(_PATH_TASK) + task.Platform + PATH_SEPARATOR

	defer func() {
		resultByte, err := ioutil.ReadFile(taskPath + "result")
		if err != nil {
			resultByte = nil

			if os.IsNotExist(err) {
				resultStatus = 0 // Was not cracked
			} else {
				Log.Printf("%s\n", err)
				resultStatus = -2
			}
		}

		fmt.Printf("Sending result of crack #%s (status: %d)...\n", task.Crack_id, resultStatus)

		if sendResult(`[{"crack_id":"`+task.Crack_id+`","start":"`+task.Start+`","offset":"`+task.Offset+`","result":"`+string(resultByte)+`","status":"`+strconv.Itoa(resultStatus)+`"}]`) == true {
			fmt.Printf("Removing task info of crack #%s (%s)...\n", task.Crack_id, task.Platform)
			err = os.RemoveAll(taskPath)
			if err != nil {
				Log.Printf("%s\n", err)
			}
		}
	}()

	crackJson, err := ioutil.ReadFile(*crackInfoPath)
	if err != nil {
		Log.Printf("%s\n", err)
		resultStatus = -3
		return false
	}

	var crack StructCrack
	err = json.Unmarshal(crackJson, &crack)
	if err != nil {
		Log.Printf("%s\n", err)
		resultStatus = -4
		return false
	}

	/* Process crack */
	// Check cracker
	var cracker string
	var internal_gen bool
	for _, crackInfo := range crack.Info {
		if crackInfo.Platform == task.Platform {
			cracker = crackInfo.Cracker
			internal_gen = crackInfo.Internal_gen
			break
		}
	}

	if cracker == "" {
		resultStatus = -5
		return false
	}

	vendorPath = getPath(_PATH_VENDOR) + _VENDOR_TYPE_CRACKER + PATH_SEPARATOR + cracker + PATH_SEPARATOR + task.Platform + PATH_SEPARATOR + _VENDOR_TYPE_CRACKER + extExecutable
	if checkVendor(_VENDOR_TYPE_CRACKER, &cracker, &task.Platform, &vendorPath) == false {
		resultStatus = -6
		return false
	}

	// Check hashfile
	*crackInfoPath = filepath.Dir(*crackInfoPath) + PATH_SEPARATOR + "hashfile"
	if _, err := os.Stat(*crackInfoPath); err != nil {
		if os.IsNotExist(err) { // Does not exist, so create it
			err = ioutil.WriteFile(*crackInfoPath, []byte(crack.Target), 0664)
			if err != nil {
				Log.Printf("%s\n", err) // Error in creating
				resultStatus = -7
				return false
			}
		} else {
			Log.Printf("%s\n", err) // Error in accessing
			resultStatus = -8
			return false
		}
	}

	/* Quote question mark in mask! */
	crack.Mask = strings.Replace(crack.Mask, "?", "??", -1)
	crack.Mask = strings.Replace(crack.Mask, "??l", "?l", -1)
	crack.Mask = strings.Replace(crack.Mask, "??u", "?u", -1)
	crack.Mask = strings.Replace(crack.Mask, "??d", "?d", -1)
	crack.Mask = strings.Replace(crack.Mask, "??s", "?s", -1)
	crack.Mask = strings.Replace(crack.Mask, "??a", "?a", -1)
	crack.Mask = strings.Replace(crack.Mask, "??b", "?b", -1)
	crack.Mask = strings.Replace(crack.Mask, "??1", "?1", -1)
	crack.Mask = strings.Replace(crack.Mask, "??2", "?2", -1)
	crack.Mask = strings.Replace(crack.Mask, "??3", "?3", -1)
	crack.Mask = strings.Replace(crack.Mask, "??4", "?4", -1)

	/* Handle replacement of custom charsets */
	var char1, char2, char3, char4 string
	if len(crack.Charset1) > 0 {
		char1 = `,"-1",` + strconv.Quote(strings.Replace(crack.Charset1, "?", "??", -1))
	} else {
		char1 = ``
	}

	if len(crack.Charset2) > 0 {
		char2 = `,"-2",` + strconv.Quote(strings.Replace(crack.Charset2, "?", "??", -1))
	} else {
		char2 = ``
	}

	if len(crack.Charset3) > 0 {
		char3 = `,"-3",` + strconv.Quote(strings.Replace(crack.Charset3, "?", "??", -1))
	} else {
		char3 = ``
	}

	if len(crack.Charset4) > 0 {
		char4 = `,"-4",` + strconv.Quote(strings.Replace(crack.Charset4, "?", "??", -1))
	} else {
		char4 = ``
	}

	generatorReplacer := strings.NewReplacer("START", task.Start, "OFFSET", task.Offset, "LEN_MIN", crack.Len_min, "LEN_MAX", crack.Len_max, `,"CHAR1"`, char1, `,"CHAR2"`, char2, `,"CHAR3"`, char3, `,"CHAR4"`, char4, "MASK", crack.Mask, `"IN_FILE"`, strconv.Quote(taskPath+"file.fifo"))
	crackerReplacer := strings.NewReplacer("ALGO_ID", crack.Algo_id, "ALGO_NAME", crack.Algo_name, `"HASH_FILE"`, strconv.Quote(*crackInfoPath), `"OUT_FILE"`, strconv.Quote(taskPath+"result"), `"IN_FILE"`, strconv.Quote(taskPath+"file.fifo"))

	if internal_gen { // Embeded
		vendorPath = filepath.Dir(vendorPath) + PATH_SEPARATOR + "info.json"
		crackerJson, err := ioutil.ReadFile(vendorPath)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -9
			return false
		}

		var crackerEmbed StructCrackerEmbed
		err = json.Unmarshal(crackerJson, &crackerEmbed)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -10
			return false
		}

		brk := false
		for _, crackerGen := range crackerEmbed.Generator {
			if crackerGen.Name == crack.Generator {
				cmdArg = crackerGen.Arg
				brk = true
				break
			}
		}
		if brk == false {
			Log.Printf("No args for cracker '%s' in info.json!\n", cracker)
			resultStatus = -11
			return false
		}

		cmdJsonByte, err := json.Marshal(&cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -12
			return false
		}
		cmdJsonStr = string(cmdJsonByte)

		cmdJsonStr = generatorReplacer.Replace(cmdJsonStr)
		cmdJsonStr = crackerReplacer.Replace(cmdJsonStr)
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -13
			return false
		}

		fmt.Printf("Performing crack #%s...\n", task.Crack_id)

		vendorPath = filepath.Dir(vendorPath) + PATH_SEPARATOR + _VENDOR_TYPE_CRACKER + extExecutable
		err = exec.Command(vendorPath, cmdArg...).Run()
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -14
			return false
		} else {
			resultStatus = 0
			return true
		}
	}
	//	 else { // Not embeded
	//		// Prepare cracker
	//		cmdJsonStr = crackerReplacer.Replace(crack.Cmd_cracker)
	//		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
	//		if err != nil {
	//			Log.Printf("%s\n", err)
	//			resultStatus = -10
	//			return false
	//		}
	//		execCracker := exec.Command(vendorPath, cmdArg...)
	//
	//		// Check generator
	//		vendorPath = getPath(_PATH_VENDOR) + _VENDOR_TYPE_GENERATOR + PATH_SEPARATOR + crack.Generator + PATH_SEPARATOR + task.Platform + PATH_SEPARATOR + _VENDOR_TYPE_GENERATOR + extExecutable
	//		if checkVendor(_VENDOR_TYPE_GENERATOR, &crack.Generator, &task.Platform, &vendorPath) == false {
	//			resultStatus = -11
	//			return false
	//		}
	//
	//		// Prepare generator
	//		cmdJsonStr = generatorReplacer.Replace(crack.Cmd_generator)
	//		if strings.Contains(cmdJsonStr, "DEP_GEN") {
	//			// Check if dependency exists in crack location
	//			*crackInfoPath = filepath.Dir(*crackInfoPath) + PATH_SEPARATOR + "dep" + PATH_SEPARATOR + "dep-gen"
	//			if _, err := os.Stat(*crackInfoPath); err == nil { // dep-gen file exists in crack location and is accessible
	//				cmdJsonStr = strings.Replace(cmdJsonStr, `"DEP_GEN"`, strconv.Quote(*crackInfoPath), -1)
	//			} else { // Check if dependency exists in generator location
	//				vendorPath = filepath.Dir(vendorPath) + PATH_SEPARATOR + "dep-gen"
	//				if _, err := os.Stat(vendorPath); err == nil { // dep-gen file exists in generator location and is accessible
	//					cmdJsonStr = strings.Replace(cmdJsonStr, `"DEP_GEN"`, strconv.Quote(vendorPath), -1)
	//				} else {
	//					resultStatus = -12
	//					return false
	//				}
	//				vendorPath = filepath.Dir(vendorPath) + PATH_SEPARATOR + _VENDOR_TYPE_GENERATOR + extExecutable // Rename back to generator executable
	//			}
	//		}
	//		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
	//		if err != nil {
	//			Log.Printf("%s\n", err)
	//			resultStatus = -13
	//			return false
	//		}
	//		execGenerator := exec.Command(vendorPath, cmdArg...)
	//
	//		fmt.Printf("Performing crack #%s...\n", task.Crack_id)
	//
	//		if crack.Type == "stdin" {
	//			r, err := execGenerator.StdoutPipe()
	//			if err != nil {
	//				Log.Printf("%s\n", err)
	//				resultStatus = -14
	//				return false
	//			}
	//			execCracker.Stdin = r
	//
	//			err = execGenerator.Start()
	//			if err != nil {
	//				Log.Printf("%s\n", err)
	//				resultStatus = -15
	//				return false
	//			}
	//
	//			err = execCracker.Start()
	//			if err != nil {
	//				Log.Printf("%s\n", err)
	//				resultStatus = -16
	//				return false
	//			}
	//
	//			execCracker.Wait()
	//			execGenerator.Process.Signal(syscall.SIGINT) // ^C (Control-C)
	//			r.Close()
	//
	//			resultStatus = 0
	//			return true
	//		} else { // Infile
	//			err = exec.Command("mkfifo", taskPath+"file.fifo").Run()
	//			if err != nil {
	//				Log.Printf("%s\n", err)
	//				resultStatus = -18
	//				return false
	//			}
	//
	//			err = execGenerator.Start()
	//			if err != nil {
	//				Log.Printf("%s\n", err)
	//				resultStatus = -19
	//				return false
	//			}
	//
	//			err = execCracker.Start()
	//			if err != nil {
	//				Log.Printf("%s\n", err)
	//				resultStatus = -20
	//				return false
	//			}
	//
	//			execCracker.Wait()
	//			execGenerator.Process.Signal(syscall.SIGINT) // ^C (Control-C)
	//
	//			resultStatus = 0
	//			return true
	//		}
	//	}

	return true
}
<|endoftext|>


<commit_before>package translators_test

import (
	"github.com/markbates/pop"
	"github.com/markbates/pop/fizz"
	"github.com/markbates/pop/fizz/translators"
)

var _ fizz.Translator = (*translators.MySQL)(nil)
var myt = translators.NewMySQL("", "")

func init() {
	myconn, err := pop.Connect("mysql")
	if err != nil {
		panic(err.Error())
	}
	deets := myconn.Dialect.Details()
	myt = translators.NewMySQL(myconn.URL(), deets.Database)
}

func (p *MySQLSuite) Test_MySQL_SchemaMigration() {
	r := p.Require()
	ddl := `CREATE TABLE schema_migrations (
version VARCHAR (255) NOT NULL
) ENGINE=InnoDB;
CREATE UNIQUE INDEX version_idx ON schema_migrations (version);`

	res, err := myt.CreateTable(fizz.Table{
		Name: "schema_migrations",
		Columns: []fizz.Column{
			{Name: "version", ColType: "string"},
		},
		Indexes: []fizz.Index{
			{Name: "version_idx", Columns: []string{"version"}, Unique: true},
		},
	})
	r.NoError(err)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_CreateTable() {
	r := p.Require()
	ddl := `CREATE TABLE users (
id integer NOT NULL AUTO_INCREMENT,
first_name VARCHAR (255) NOT NULL,
last_name VARCHAR (255) NOT NULL,
email VARCHAR (20) NOT NULL,
permissions text,
age integer DEFAULT 40,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL,
PRIMARY KEY(id)
) ENGINE=InnoDB;`

	res, _ := fizz.AString(`
	create_table("users", func(t) {
		t.Column("first_name", "string", {})
		t.Column("last_name", "string", {})
		t.Column("email", "string", {"size":20})
		t.Column("permissions", "text", {"null": true})
		t.Column("age", "integer", {"null": true, "default": 40})
	})
	`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_CreateTable_UUID() {
	r := p.Require()
	ddl := `CREATE TABLE users (
first_name VARCHAR (255) NOT NULL,
last_name VARCHAR (255) NOT NULL,
email VARCHAR (20) NOT NULL,
permissions text,
age integer DEFAULT 40,
company_id char(36) NOT NULL DEFAULT 'test',
uuid char(36) NOT NULL,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL,
PRIMARY KEY(uuid)
) ENGINE=InnoDB;`

	res, _ := fizz.AString(`
	create_table("users", func(t) {
		t.Column("first_name", "string", {})
		t.Column("last_name", "string", {})
		t.Column("email", "string", {"size":20})
		t.Column("permissions", "text", {"null": true})
		t.Column("age", "integer", {"null": true, "default": 40})
		t.Column("company_id", "uuid", {"default_raw": "'test'"})
		t.Column("uuid", "uuid", {"primary": true})
	})
	`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_DropTable() {
	r := p.Require()

	ddl := `DROP TABLE users;`

	res, _ := fizz.AString(`drop_table("users")`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_RenameTable() {
	r := p.Require()

	ddl := `ALTER TABLE users RENAME TO people;`

	res, _ := fizz.AString(`rename_table("users", "people")`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_RenameTable_NotEnoughValues() {
	r := p.Require()

	_, err := myt.RenameTable([]fizz.Table{})
	r.Error(err)
}

func (p *MySQLSuite) Test_MySQL_ChangeColumn() {
	r := p.Require()
	ddl := `ALTER TABLE users MODIFY mycolumn VARCHAR (50) NOT NULL DEFAULT 'foo';`

	res, _ := fizz.AString(`change_column("users", "mycolumn", "string", {"default": "foo", "size": 50})`, myt)

	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddColumn() {
	r := p.Require()
	ddl := `ALTER TABLE users ADD COLUMN mycolumn VARCHAR (50) NOT NULL DEFAULT 'foo';`

	res, _ := fizz.AString(`add_column("users", "mycolumn", "string", {"default": "foo", "size": 50})`, myt)

	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_DropColumn() {
	r := p.Require()
	ddl := `ALTER TABLE users DROP COLUMN mycolumn;`

	res, _ := fizz.AString(`drop_column("users", "mycolumn")`, myt)

	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_RenameColumn() {
	r := p.Require()
	ddl := `ALTER TABLE users CHANGE email email_address varchar(50) NOT NULL DEFAULT 'foo@example.com';`

	res, _ := fizz.AString(`rename_column("users", "email", "email_address")`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddIndex() {
	r := p.Require()
	ddl := `CREATE INDEX users_email_idx ON users (email);`

	res, _ := fizz.AString(`add_index("users", "email", {})`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddIndex_Unique() {
	r := p.Require()
	ddl := `CREATE UNIQUE INDEX users_email_idx ON users (email);`

	res, _ := fizz.AString(`add_index("users", "email", {"unique": true})`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddIndex_MultiColumn() {
	r := p.Require()
	ddl := `CREATE INDEX users_id_email_idx ON users (id, email);`

	res, _ := fizz.AString(`add_index("users", ["id", "email"], {})`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddIndex_CustomName() {
	r := p.Require()
	ddl := `CREATE INDEX email_index ON users (email);`

	res, _ := fizz.AString(`add_index("users", "email", {"name": "email_index"})`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_DropIndex() {
	r := p.Require()
	ddl := `DROP INDEX email_idx ON users;`

	res, _ := fizz.AString(`drop_index("users", "email_idx")`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_RenameIndex() {
	r := p.Require()

	ddl := `ALTER TABLE users RENAME INDEX email_idx TO email_address_ix;`

	res, _ := fizz.AString(`rename_index("users", "email_idx", "email_address_ix")`, myt)
	r.Equal(ddl, res)
}
<commit_msg>fixed mysql tests<commit_after>package translators_test

import (
	"github.com/markbates/pop"
	"github.com/markbates/pop/fizz"
	"github.com/markbates/pop/fizz/translators"
)

var _ fizz.Translator = (*translators.MySQL)(nil)
var myt = translators.NewMySQL("", "")

func init() {
	myconn, err := pop.Connect("mysql")
	if err != nil {
		panic(err.Error())
	}
	deets := myconn.Dialect.Details()
	myt = translators.NewMySQL(myconn.URL(), deets.Database)
}

func (p *MySQLSuite) Test_MySQL_SchemaMigration() {
	r := p.Require()
	ddl := `CREATE TABLE schema_migrations (
version VARCHAR (255) NOT NULL
) ENGINE=InnoDB;
CREATE UNIQUE INDEX version_idx ON schema_migrations (version);`

	res, err := myt.CreateTable(fizz.Table{
		Name: "schema_migrations",
		Columns: []fizz.Column{
			{Name: "version", ColType: "string"},
		},
		Indexes: []fizz.Index{
			{Name: "version_idx", Columns: []string{"version"}, Unique: true},
		},
	})
	r.NoError(err)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_CreateTable() {
	r := p.Require()
	ddl := `CREATE TABLE users (
id integer NOT NULL AUTO_INCREMENT,
PRIMARY KEY(id),
first_name VARCHAR (255) NOT NULL,
last_name VARCHAR (255) NOT NULL,
email VARCHAR (20) NOT NULL,
permissions text,
age integer DEFAULT 40,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
) ENGINE=InnoDB;`

	res, _ := fizz.AString(`
	create_table("users", func(t) {
		t.Column("first_name", "string", {})
		t.Column("last_name", "string", {})
		t.Column("email", "string", {"size":20})
		t.Column("permissions", "text", {"null": true})
		t.Column("age", "integer", {"null": true, "default": 40})
	})
	`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_CreateTable_UUID() {
	r := p.Require()
	ddl := `CREATE TABLE users (
first_name VARCHAR (255) NOT NULL,
last_name VARCHAR (255) NOT NULL,
email VARCHAR (20) NOT NULL,
permissions text,
age integer DEFAULT 40,
company_id char(36) NOT NULL DEFAULT 'test',
uuid char(36) NOT NULL,
PRIMARY KEY(uuid),
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
) ENGINE=InnoDB;`

	res, _ := fizz.AString(`
	create_table("users", func(t) {
		t.Column("first_name", "string", {})
		t.Column("last_name", "string", {})
		t.Column("email", "string", {"size":20})
		t.Column("permissions", "text", {"null": true})
		t.Column("age", "integer", {"null": true, "default": 40})
		t.Column("company_id", "uuid", {"default_raw": "'test'"})
		t.Column("uuid", "uuid", {"primary": true})
	})
	`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_DropTable() {
	r := p.Require()

	ddl := `DROP TABLE users;`

	res, _ := fizz.AString(`drop_table("users")`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_RenameTable() {
	r := p.Require()

	ddl := `ALTER TABLE users RENAME TO people;`

	res, _ := fizz.AString(`rename_table("users", "people")`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_RenameTable_NotEnoughValues() {
	r := p.Require()

	_, err := myt.RenameTable([]fizz.Table{})
	r.Error(err)
}

func (p *MySQLSuite) Test_MySQL_ChangeColumn() {
	r := p.Require()
	ddl := `ALTER TABLE users MODIFY mycolumn VARCHAR (50) NOT NULL DEFAULT 'foo';`

	res, _ := fizz.AString(`change_column("users", "mycolumn", "string", {"default": "foo", "size": 50})`, myt)

	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddColumn() {
	r := p.Require()
	ddl := `ALTER TABLE users ADD COLUMN mycolumn VARCHAR (50) NOT NULL DEFAULT 'foo';`

	res, _ := fizz.AString(`add_column("users", "mycolumn", "string", {"default": "foo", "size": 50})`, myt)

	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_DropColumn() {
	r := p.Require()
	ddl := `ALTER TABLE users DROP COLUMN mycolumn;`

	res, _ := fizz.AString(`drop_column("users", "mycolumn")`, myt)

	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_RenameColumn() {
	r := p.Require()
	ddl := `ALTER TABLE users CHANGE email email_address varchar(50) NOT NULL DEFAULT 'foo@example.com';`

	res, _ := fizz.AString(`rename_column("users", "email", "email_address")`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddIndex() {
	r := p.Require()
	ddl := `CREATE INDEX users_email_idx ON users (email);`

	res, _ := fizz.AString(`add_index("users", "email", {})`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddIndex_Unique() {
	r := p.Require()
	ddl := `CREATE UNIQUE INDEX users_email_idx ON users (email);`

	res, _ := fizz.AString(`add_index("users", "email", {"unique": true})`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddIndex_MultiColumn() {
	r := p.Require()
	ddl := `CREATE INDEX users_id_email_idx ON users (id, email);`

	res, _ := fizz.AString(`add_index("users", ["id", "email"], {})`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_AddIndex_CustomName() {
	r := p.Require()
	ddl := `CREATE INDEX email_index ON users (email);`

	res, _ := fizz.AString(`add_index("users", "email", {"name": "email_index"})`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_DropIndex() {
	r := p.Require()
	ddl := `DROP INDEX email_idx ON users;`

	res, _ := fizz.AString(`drop_index("users", "email_idx")`, myt)
	r.Equal(ddl, res)
}

func (p *MySQLSuite) Test_MySQL_RenameIndex() {
	r := p.Require()

	ddl := `ALTER TABLE users RENAME INDEX email_idx TO email_address_ix;`

	res, _ := fizz.AString(`rename_index("users", "email_idx", "email_address_ix")`, myt)
	r.Equal(ddl, res)
}
<|endoftext|>