<commit_before>package gmime

/*
#cgo pkg-config: gmime-2.6
#include <stdlib.h>
#include <gmime/gmime.h>
*/
import "C"
import (
	"unsafe"
)

type Multipart interface {
	Object
	AddPart(Object)
	Boundary() string
	GetPart(int) Object
	Count() int
	Clear()
}

type aMultipart struct {
	*anObject
}

type rawMultipart interface {
	Multipart
	rawMultipart() *C.GMimeMultipart
}

func CastMultipart(mp *C.GMimeMultipart) Multipart {
	oPtr := (*C.GMimeObject)(unsafe.Pointer(mp))
	return &aMultipart{CastObject(oPtr)}
}

func NewMultipart() Multipart {
	multipart := C.g_mime_multipart_new()
	defer unref(C.gpointer(multipart))
	return CastMultipart(multipart)
}

func NewMultipartWithSubtype(subtype string) Multipart {
	var _csubtype *C.char = C.CString(subtype)
	defer C.free(unsafe.Pointer(_csubtype))
	multipart := C.g_mime_multipart_new_with_subtype(_csubtype)
	defer unref(C.gpointer(multipart))
	return CastMultipart(multipart)
}

func (m *aMultipart) AddPart(part Object) {
	rawObject := part.(rawObject)
	C.g_mime_multipart_add(m.rawMultipart(), rawObject.rawObject())
}

func (m *aMultipart) GetPart(index int) Object {
	part := C.g_mime_multipart_get_part(m.rawMultipart(), C.int(index))
	return objectAsSubclass(part)
}

func (m *aMultipart) Clear() {
	C.g_mime_multipart_clear(m.rawMultipart())
}

func (m *aMultipart) Count() int {
	return int(C.g_mime_multipart_get_count(m.rawMultipart()))
}

func (m *aMultipart) Boundary() string {
	return C.GoString(C.g_mime_multipart_get_boundary(m.rawMultipart()))
}

func (m *aMultipart) rawMultipart() *C.GMimeMultipart {
	return (*C.GMimeMultipart)(m.pointer())
}
<commit_msg>Add .Walk() method to Multipart<commit_after>package gmime

/*
#cgo pkg-config: gmime-2.6
#include <stdlib.h>
#include <gmime/gmime.h>
*/
import "C"
import (
	"unsafe"
)

type Multipart interface {
	Object
	AddPart(Object)
	Boundary() string
	GetPart(int) Object
	Count() int
	Clear()
    Walk(func (Object) error) error
}

type aMultipart struct {
	*anObject
}

type rawMultipart interface {
	Multipart
	rawMultipart() *C.GMimeMultipart
}

func CastMultipart(mp *C.GMimeMultipart) Multipart {
	oPtr := (*C.GMimeObject)(unsafe.Pointer(mp))
	return &aMultipart{CastObject(oPtr)}
}

func NewMultipart() Multipart {
	multipart := C.g_mime_multipart_new()
	defer unref(C.gpointer(multipart))
	return CastMultipart(multipart)
}

func NewMultipartWithSubtype(subtype string) Multipart {
	var _csubtype *C.char = C.CString(subtype)
	defer C.free(unsafe.Pointer(_csubtype))
	multipart := C.g_mime_multipart_new_with_subtype(_csubtype)
	defer unref(C.gpointer(multipart))
	return CastMultipart(multipart)
}

func (m *aMultipart) AddPart(part Object) {
	rawObject := part.(rawObject)
	C.g_mime_multipart_add(m.rawMultipart(), rawObject.rawObject())
}

func (m *aMultipart) GetPart(index int) Object {
	part := C.g_mime_multipart_get_part(m.rawMultipart(), C.int(index))
	return objectAsSubclass(part)
}

func (m *aMultipart) Clear() {
	C.g_mime_multipart_clear(m.rawMultipart())
}

func (m *aMultipart) Count() int {
	return int(C.g_mime_multipart_get_count(m.rawMultipart()))
}

func (m *aMultipart) Boundary() string {
	return C.GoString(C.g_mime_multipart_get_boundary(m.rawMultipart()))
}

// FIXME: need tests
func (m *aMultipart) Walk(callback func(Object) error) error {
	for i := 0; i < m.Count(); i++ {
		if ok := callback(m.GetPart(i)); ok != nil {
			return ok
		}
	}
	return nil
}

func (m *aMultipart) rawMultipart() *C.GMimeMultipart {
	return (*C.GMimeMultipart)(m.pointer())
}
<|endoftext|>


<commit_before>// Package tproxy provides the TcpDial and TcpListen tproxy equivalent of the
// net package Dial and Listen with tproxy support for linux ONLY.
package tproxy

import (
	"fmt"
	"net"
	"os"

	"golang.org/x/sys/unix"
)

const big = 0xFFFFFF
const IP_ORIGADDRS = 20

// Debug outs the library in Debug mode
var Debug = false

func ipToSocksAddr(family int, ip net.IP, port int, zone string) (unix.Sockaddr, error) {
	switch family {
	case unix.AF_INET:
		if len(ip) == 0 {
			ip = net.IPv4zero
		}
		if ip = ip.To4(); ip == nil {
			return nil, net.InvalidAddrError("non-IPv4 address")
		}
		sa := new(unix.SockaddrInet4)
		for i := 0; i < net.IPv4len; i++ {
			sa.Addr[i] = ip[i]
		}
		sa.Port = port
		return sa, nil
	case unix.AF_INET6:
		if len(ip) == 0 {
			ip = net.IPv6zero
		}
		// IPv4 callers use 0.0.0.0 to mean "announce on any available address".
		// In IPv6 mode, Linux treats that as meaning "announce on 0.0.0.0",
		// which it refuses to do.  Rewrite to the IPv6 unspecified address.
		if ip.Equal(net.IPv4zero) {
			ip = net.IPv6zero
		}
		if ip = ip.To16(); ip == nil {
			return nil, net.InvalidAddrError("non-IPv6 address")
		}
		sa := new(unix.SockaddrInet6)
		for i := 0; i < net.IPv6len; i++ {
			sa.Addr[i] = ip[i]
		}
		sa.Port = port
		sa.ZoneId = uint32(zoneToInt(zone))
		return sa, nil
	}
	return nil, net.InvalidAddrError("unexpected socket family")
}

func zoneToInt(zone string) int {
	if zone == "" {
		return 0
	}
	if ifi, err := net.InterfaceByName(zone); err == nil {
		return ifi.Index
	}
	n, _, _ := dtoi(zone, 0)
	return n
}

func dtoi(s string, i0 int) (n int, i int, ok bool) {
	n = 0
	for i = i0; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ {
		n = n*10 + int(s[i]-'0')
		if n >= big {
			return 0, i, false
		}
	}
	if i == i0 {
		return 0, i, false
	}
	return n, i, true
}

// IPv6TcpAddrToUnixSocksAddr ---
func IPv6TcpAddrToUnixSocksAddr(addr string) (sa unix.Sockaddr, err error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp6", addr)
	if err != nil {
		return nil, err
	}
	return ipToSocksAddr(unix.AF_INET6, tcpAddr.IP, tcpAddr.Port, tcpAddr.Zone)
}

// IPv6UdpAddrToUnixSocksAddr ---
func IPv6UdpAddrToUnixSocksAddr(addr string) (sa unix.Sockaddr, err error) {
	tcpAddr, err := net.ResolveTCPAddr("udp6", addr)
	if err != nil {
		return nil, err
	}
	return ipToSocksAddr(unix.AF_INET6, tcpAddr.IP, tcpAddr.Port, tcpAddr.Zone)
}

// TCPListen is listening for incoming IP packets which are being intercepted.
// In conflict to regular Listen mehtod the socket destination and source addresses
// are of the intercepted connection.
// Else then that it works exactly like net package net.Listen.
func TCPListen(listenAddr string) (listener net.Listener, err error) {
	s, err := unix.Socket(unix.AF_INET6, unix.SOCK_STREAM, 0)
	if err != nil {
		return nil, err
	}
	defer unix.Close(s)
	err = unix.SetsockoptInt(s, unix.SOL_IP, unix.IP_TRANSPARENT, 1)
	if err != nil {
		return nil, err
	}

	sa, err := IPv6TcpAddrToUnixSocksAddr(listenAddr)
	if err != nil {
		return nil, err
	}
	err = unix.Bind(s, sa)
	if err != nil {
		return nil, err
	}
	err = unix.Listen(s, unix.SOMAXCONN)
	if err != nil {
		return nil, err
	}
	f := os.NewFile(uintptr(s), "TProxy")
	defer f.Close()
	return net.FileListener(f)
}

// TCPDial is a special tcp connection which binds a non local address as the source.
// Except then the option to bind to a specific local address which the machine doesn't posses
// it is exactly like any other net.Conn connection.
// It is advised to use port numbered 0 in the localAddr and leave the kernel to choose which
// Local port to use in order to avoid errors and binding conflicts.
func TCPDial(localAddr, remoteAddr string) (conn net.Conn, err error) {
	if Debug {
		fmt.Println("TcpDial from:", localAddr, "to:", remoteAddr)
	}
	s, err := unix.Socket(unix.AF_INET6, unix.SOCK_STREAM, 0)

	//In a case there was a need for a non-blocking socket an example
	//s, err := unix.Socket(unix.AF_INET6, unix.SOCK_STREAM |unix.SOCK_NONBLOCK, 0)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}
	defer unix.Close(s)
	err = unix.SetsockoptInt(s, unix.SOL_IP, unix.IP_TRANSPARENT, 1)
	if err != nil {
		if Debug {
			fmt.Println("ERROR setting the socket in IP_TRANSPARENT mode", err)
		}

		return nil, err
	}

	err = unix.SetsockoptInt(s, unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
	if err != nil {
		if Debug {
			fmt.Println("ERROR setting the socket in unix.SO_REUSEADDR mode", err)
		}
		return nil, err
	}

	rhost, _, err := net.SplitHostPort(localAddr)
	if err != nil {
		if Debug {
			// fmt.Fprintln(os.Stderr, err)
			fmt.Println("ERROR", err, "running net.SplitHostPort on address:", localAddr)
		}
	}

	sa, err := IPv6TcpAddrToUnixSocksAddr(rhost + ":0")
	if err != nil {
		if Debug {
			fmt.Println("ERROR creating a hostaddres for the socker with IPv6TcpAddrToUnixSocksAddr", err)
		}
		return nil, err
	}

	remoteSocket, err := IPv6TcpAddrToUnixSocksAddr(remoteAddr)
	if err != nil {
		if Debug {
			fmt.Println("ERROR creating a remoteSocket for the socker with IPv6TcpAddrToUnixSocksAddr on the remote addres", err)
		}
		return nil, err
	}

	err = unix.Bind(s, sa)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	err = unix.Connect(s, remoteSocket)
	if err != nil {
		if Debug {
			fmt.Println("ERROR Connecting from", s, "to:", remoteSocket, "ERROR:", err)
		}
		return nil, err
	}

	f := os.NewFile(uintptr(s), "TProxyTCPClient")
	client, err := net.FileConn(f)
	if err != nil {
		if Debug {
			fmt.Println("ERROR os.NewFile", err)
		}
		return nil, err
	}
	if Debug {
		fmt.Println("FINISHED Creating net.coo from:", client.LocalAddr().String(), "to:", client.RemoteAddr().String())
	}
	return client, err
}
<commit_msg>Added another debug.<commit_after>// Package tproxy provides the TcpDial and TcpListen tproxy equivalent of the
// net package Dial and Listen with tproxy support for linux ONLY.
package tproxy

import (
	"fmt"
	"net"
	"os"

	"golang.org/x/sys/unix"
)

const big = 0xFFFFFF
const IP_ORIGADDRS = 20

// Debug outs the library in Debug mode
var Debug = false

func ipToSocksAddr(family int, ip net.IP, port int, zone string) (unix.Sockaddr, error) {
	switch family {
	case unix.AF_INET:
		if len(ip) == 0 {
			ip = net.IPv4zero
		}
		if ip = ip.To4(); ip == nil {
			return nil, net.InvalidAddrError("non-IPv4 address")
		}
		sa := new(unix.SockaddrInet4)
		for i := 0; i < net.IPv4len; i++ {
			sa.Addr[i] = ip[i]
		}
		sa.Port = port
		return sa, nil
	case unix.AF_INET6:
		if len(ip) == 0 {
			ip = net.IPv6zero
		}
		// IPv4 callers use 0.0.0.0 to mean "announce on any available address".
		// In IPv6 mode, Linux treats that as meaning "announce on 0.0.0.0",
		// which it refuses to do.  Rewrite to the IPv6 unspecified address.
		if ip.Equal(net.IPv4zero) {
			ip = net.IPv6zero
		}
		if ip = ip.To16(); ip == nil {
			return nil, net.InvalidAddrError("non-IPv6 address")
		}
		sa := new(unix.SockaddrInet6)
		for i := 0; i < net.IPv6len; i++ {
			sa.Addr[i] = ip[i]
		}
		sa.Port = port
		sa.ZoneId = uint32(zoneToInt(zone))
		return sa, nil
	}
	return nil, net.InvalidAddrError("unexpected socket family")
}

func zoneToInt(zone string) int {
	if zone == "" {
		return 0
	}
	if ifi, err := net.InterfaceByName(zone); err == nil {
		return ifi.Index
	}
	n, _, _ := dtoi(zone, 0)
	return n
}

func dtoi(s string, i0 int) (n int, i int, ok bool) {
	n = 0
	for i = i0; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ {
		n = n*10 + int(s[i]-'0')
		if n >= big {
			return 0, i, false
		}
	}
	if i == i0 {
		return 0, i, false
	}
	return n, i, true
}

// IPv6TcpAddrToUnixSocksAddr ---
func IPv6TcpAddrToUnixSocksAddr(addr string) (sa unix.Sockaddr, err error) {
	if Debug {
		fmt.Println("DEBUG: IPv6TcpAddrToUnixSocksAddr recieved address:", addr)
	}
	tcpAddr, err := net.ResolveTCPAddr("tcp6", addr)
	if err != nil {
		return nil, err
	}
	return ipToSocksAddr(unix.AF_INET6, tcpAddr.IP, tcpAddr.Port, tcpAddr.Zone)
}

// IPv6UdpAddrToUnixSocksAddr ---
func IPv6UdpAddrToUnixSocksAddr(addr string) (sa unix.Sockaddr, err error) {
	tcpAddr, err := net.ResolveTCPAddr("udp6", addr)
	if err != nil {
		return nil, err
	}
	return ipToSocksAddr(unix.AF_INET6, tcpAddr.IP, tcpAddr.Port, tcpAddr.Zone)
}

// TCPListen is listening for incoming IP packets which are being intercepted.
// In conflict to regular Listen mehtod the socket destination and source addresses
// are of the intercepted connection.
// Else then that it works exactly like net package net.Listen.
func TCPListen(listenAddr string) (listener net.Listener, err error) {
	s, err := unix.Socket(unix.AF_INET6, unix.SOCK_STREAM, 0)
	if err != nil {
		return nil, err
	}
	defer unix.Close(s)
	err = unix.SetsockoptInt(s, unix.SOL_IP, unix.IP_TRANSPARENT, 1)
	if err != nil {
		return nil, err
	}

	sa, err := IPv6TcpAddrToUnixSocksAddr(listenAddr)
	if err != nil {
		return nil, err
	}
	err = unix.Bind(s, sa)
	if err != nil {
		return nil, err
	}
	err = unix.Listen(s, unix.SOMAXCONN)
	if err != nil {
		return nil, err
	}
	f := os.NewFile(uintptr(s), "TProxy")
	defer f.Close()
	return net.FileListener(f)
}

// TCPDial is a special tcp connection which binds a non local address as the source.
// Except then the option to bind to a specific local address which the machine doesn't posses
// it is exactly like any other net.Conn connection.
// It is advised to use port numbered 0 in the localAddr and leave the kernel to choose which
// Local port to use in order to avoid errors and binding conflicts.
func TCPDial(localAddr, remoteAddr string) (conn net.Conn, err error) {
	if Debug {
		fmt.Println("TcpDial from:", localAddr, "to:", remoteAddr)
	}
	s, err := unix.Socket(unix.AF_INET6, unix.SOCK_STREAM, 0)

	//In a case there was a need for a non-blocking socket an example
	//s, err := unix.Socket(unix.AF_INET6, unix.SOCK_STREAM |unix.SOCK_NONBLOCK, 0)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}
	defer unix.Close(s)
	err = unix.SetsockoptInt(s, unix.SOL_IP, unix.IP_TRANSPARENT, 1)
	if err != nil {
		if Debug {
			fmt.Println("ERROR setting the socket in IP_TRANSPARENT mode", err)
		}

		return nil, err
	}

	err = unix.SetsockoptInt(s, unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
	if err != nil {
		if Debug {
			fmt.Println("ERROR setting the socket in unix.SO_REUSEADDR mode", err)
		}
		return nil, err
	}

	rhost, _, err := net.SplitHostPort(localAddr)
	if err != nil {
		if Debug {
			// fmt.Fprintln(os.Stderr, err)
			fmt.Println("ERROR", err, "running net.SplitHostPort on address:", localAddr)
		}
	}

	sa, err := IPv6TcpAddrToUnixSocksAddr(rhost + ":0")
	if err != nil {
		if Debug {
			fmt.Println("ERROR creating a hostaddres for the socker with IPv6TcpAddrToUnixSocksAddr", err)
		}
		return nil, err
	}

	remoteSocket, err := IPv6TcpAddrToUnixSocksAddr(remoteAddr)
	if err != nil {
		if Debug {
			fmt.Println("ERROR creating a remoteSocket for the socker with IPv6TcpAddrToUnixSocksAddr on the remote addres", err)
		}
		return nil, err
	}

	err = unix.Bind(s, sa)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	err = unix.Connect(s, remoteSocket)
	if err != nil {
		if Debug {
			fmt.Println("ERROR Connecting from", s, "to:", remoteSocket, "ERROR:", err)
		}
		return nil, err
	}

	f := os.NewFile(uintptr(s), "TProxyTCPClient")
	client, err := net.FileConn(f)
	if err != nil {
		if Debug {
			fmt.Println("ERROR os.NewFile", err)
		}
		return nil, err
	}
	if Debug {
		fmt.Println("FINISHED Creating net.coo from:", client.LocalAddr().String(), "to:", client.RemoteAddr().String())
	}
	return client, err
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package keybase

import (
	"encoding/base64"
	"errors"
	"fmt"
	"net"
	"sync"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/logger"
	"github.com/keybase/client/go/protocol"
	"github.com/keybase/client/go/service"
	"github.com/keybase/go-framed-msgpack-rpc"
	"github.com/keybase/kbfs/fsrpc"
	"github.com/keybase/kbfs/libkbfs"
)

var kbCtx *libkb.GlobalContext
var conn net.Conn
var startOnce sync.Once
var logSendContext libkb.LogSendContext
var kbfsConfig libkbfs.Config

// InitOnce runs the Keybase services (only runs one time)
func InitOnce(homeDir string, logFile string, runModeStr string, accessGroupOverride bool) {
	startOnce.Do(func() {
		if err := Init(homeDir, logFile, runModeStr, accessGroupOverride); err != nil {
			kbCtx.Log.Errorf("Init error: %s", err)
		}
	})
}

// Init runs the Keybase services
func Init(homeDir string, logFile string, runModeStr string, accessGroupOverride bool) error {
	fmt.Println("Go: Initializing")
	fmt.Printf("Go: Using log: %s\n", logFile)
	kbCtx = libkb.G
	kbCtx.Init()
	usage := libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
	runMode, err := libkb.StringToRunMode(runModeStr)
	if err != nil {
		return err
	}
	config := libkb.AppConfig{
		HomeDir:                     homeDir,
		LogFile:                     logFile,
		RunMode:                     runMode,
		Debug:                       true,
		LocalRPCDebug:               "Acsvip",
		SecurityAccessGroupOverride: accessGroupOverride,
	}
	err = kbCtx.Configure(config, usage)
	if err != nil {
		return err
	}

	svc := service.NewService(kbCtx, false)
	svc.StartLoopbackServer()
	kbCtx.SetService()
	kbCtx.SetUIRouter(service.NewUIRouter(kbCtx))

	serviceLog := config.GetLogFile()
	logs := libkb.Logs{
		Service: serviceLog,
	}

	logSendContext = libkb.LogSendContext{
		Contextified: libkb.NewContextified(kbCtx),
		Logs:         logs,
	}

	// FIXME (MBG): This is causing RPC responses to sometimes not be recieved
	// on iOS. Repro by hooking up getExtendedStatus to a button in the iOS
	// client and watching JS logs. Disabling until we have a root cause / fix.
	//
	//kbfsParams := libkbfs.DefaultInitParams(kbCtx)
	//kbfsConfig, err = libkbfs.Init(kbCtx, kbfsParams, newKeybaseDaemon, func() {}, kbCtx.Log)
	//if err != nil {
	//	return err
	//}

	return Reset()
}

func newKeybaseDaemon(config libkbfs.Config, params libkbfs.InitParams, ctx libkbfs.Context, log logger.Logger) (libkbfs.KeybaseService, error) {
	keybaseService := libkbfs.NewKeybaseDaemonRPC(config, ctx, log, true)
	keybaseService.AddProtocols([]rpc.Protocol{
		keybase1.FsProtocol(fsrpc.NewFS(config, log)),
	})
	return keybaseService, nil
}

// LogSend sends a log to Keybase
func LogSend(uiLogPath string) (string, error) {
	logSendContext.Logs.Desktop = uiLogPath
	return logSendContext.LogSend("", 10000)
}

// WriteB64 sends a base64 encoded msgpack rpc payload
func WriteB64(str string) error {
	data, err := base64.StdEncoding.DecodeString(str)
	if err != nil {
		return fmt.Errorf("Base64 decode error: %s; %s", err, str)
	}
	n, err := conn.Write(data)
	if err != nil {
		return fmt.Errorf("Write error: %s", err)
	}
	if n != len(data) {
		return errors.New("Did not write all the data")
	}
	return nil
}

const targetBufferSize = 50 * 1024

// bufferSize must be divisible by 3 to ensure that we don't split
// our b64 encode across a payload boundary if we go over our buffer
// size.
const bufferSize = targetBufferSize - (targetBufferSize % 3)

// ReadB64 is a blocking read for base64 encoded msgpack rpc data.
func ReadB64() (string, error) {
	data := make([]byte, bufferSize)

	n, err := conn.Read(data)
	if n > 0 && err == nil {
		str := base64.StdEncoding.EncodeToString(data[0:n])
		return str, nil
	}

	if err != nil {
		// Attempt to fix the connection
		Reset()
		return "", fmt.Errorf("Read error: %s", err)
	}

	return "", nil
}

// Reset resets the socket connection
func Reset() error {
	if conn != nil {
		conn.Close()
	}

	var err error
	conn, _, _, err = kbCtx.ResetSocket(false)
	if err != nil {
		return fmt.Errorf("Socket error: %s", err)
	}
	return nil
}
<commit_msg>This should fix the deadlock that max goodman found (#3747)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package keybase

import (
	"encoding/base64"
	"errors"
	"fmt"
	"net"
	"sync"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/logger"
	"github.com/keybase/client/go/protocol"
	"github.com/keybase/client/go/service"
	"github.com/keybase/go-framed-msgpack-rpc"
	"github.com/keybase/kbfs/fsrpc"
	"github.com/keybase/kbfs/libkbfs"
)

var kbCtx *libkb.GlobalContext
var conn net.Conn
var startOnce sync.Once
var logSendContext libkb.LogSendContext
var kbfsConfig libkbfs.Config

// InitOnce runs the Keybase services (only runs one time)
func InitOnce(homeDir string, logFile string, runModeStr string, accessGroupOverride bool) {
	startOnce.Do(func() {
		if err := Init(homeDir, logFile, runModeStr, accessGroupOverride); err != nil {
			kbCtx.Log.Errorf("Init error: %s", err)
		}
	})
}

// Init runs the Keybase services
func Init(homeDir string, logFile string, runModeStr string, accessGroupOverride bool) error {
	fmt.Println("Go: Initializing")
	fmt.Printf("Go: Using log: %s\n", logFile)
	kbCtx = libkb.G
	kbCtx.Init()
	usage := libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
	runMode, err := libkb.StringToRunMode(runModeStr)
	if err != nil {
		return err
	}
	config := libkb.AppConfig{
		HomeDir:                     homeDir,
		LogFile:                     logFile,
		RunMode:                     runMode,
		Debug:                       true,
		LocalRPCDebug:               "Acsvip",
		SecurityAccessGroupOverride: accessGroupOverride,
	}
	err = kbCtx.Configure(config, usage)
	if err != nil {
		return err
	}

	svc := service.NewService(kbCtx, false)
	svc.StartLoopbackServer()
	kbCtx.SetService()
	kbCtx.SetUIRouter(service.NewUIRouter(kbCtx))

	serviceLog := config.GetLogFile()
	logs := libkb.Logs{
		Service: serviceLog,
	}

	logSendContext = libkb.LogSendContext{
		Contextified: libkb.NewContextified(kbCtx),
		Logs:         logs,
	}

	// FIXME (MBG): This is causing RPC responses to sometimes not be recieved
	// on iOS. Repro by hooking up getExtendedStatus to a button in the iOS
	// client and watching JS logs. Disabling until we have a root cause / fix.
	kbfsParams := libkbfs.DefaultInitParams(kbCtx)
	kbfsConfig, err = libkbfs.Init(kbCtx, kbfsParams, newKeybaseDaemon, func() {}, kbCtx.Log)
	if err != nil {
		return err
	}

	return Reset()
}

func newKeybaseDaemon(config libkbfs.Config, params libkbfs.InitParams, ctx libkbfs.Context, log logger.Logger) (libkbfs.KeybaseService, error) {
	keybaseService := libkbfs.NewKeybaseDaemonRPC(config, ctx, log, true)
	keybaseService.AddProtocols([]rpc.Protocol{
		keybase1.FsProtocol(fsrpc.NewFS(config, log)),
	})
	return keybaseService, nil
}

// LogSend sends a log to Keybase
func LogSend(uiLogPath string) (string, error) {
	logSendContext.Logs.Desktop = uiLogPath
	return logSendContext.LogSend("", 10000)
}

// WriteB64 sends a base64 encoded msgpack rpc payload
func WriteB64(str string) error {
	data, err := base64.StdEncoding.DecodeString(str)
	if err != nil {
		return fmt.Errorf("Base64 decode error: %s; %s", err, str)
	}
	n, err := conn.Write(data)
	if err != nil {
		return fmt.Errorf("Write error: %s", err)
	}
	if n != len(data) {
		return errors.New("Did not write all the data")
	}
	return nil
}

const targetBufferSize = 50 * 1024

// bufferSize must be divisible by 3 to ensure that we don't split
// our b64 encode across a payload boundary if we go over our buffer
// size.
const bufferSize = targetBufferSize - (targetBufferSize % 3)

// ReadB64 is a blocking read for base64 encoded msgpack rpc data.
func ReadB64() (string, error) {
	data := make([]byte, bufferSize)

	n, err := conn.Read(data)
	if n > 0 && err == nil {
		str := base64.StdEncoding.EncodeToString(data[0:n])
		return str, nil
	}

	if err != nil {
		// Attempt to fix the connection
		Reset()
		return "", fmt.Errorf("Read error: %s", err)
	}

	return "", nil
}

// Reset resets the socket connection
func Reset() error {
	if conn != nil {
		conn.Close()
	}

	var err error
	conn, err = kbCtx.LoopbackListener.Dial()
	if err != nil {
		return fmt.Errorf("Socket error: %s", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
)

const (
	baseURL = "http://%s:8091/settings/rbac/users/local/%s"
)

var (
	numUsers, numWorkers      int
	hostname, admin, password string
	httpClient                *http.Client
)

func newBody() io.Reader {
	form := url.Values{}
	form.Add("roles", "data_writer[*],data_reader[*],query_select[*]")
	form.Add("password", "Pa$$w0rd")

	return strings.NewReader(form.Encode())
}

func addUser(user string) error {
	log.Printf("Adding a new user: %s\n", user)

	req, err := http.NewRequest("PUT", fmt.Sprintf(baseURL, hostname, user), newBody())
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.SetBasicAuth(admin, password)

	resp, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	io.Copy(ioutil.Discard, resp.Body)

	if resp.StatusCode != 200 {
		return fmt.Errorf("unexpected status code: %v", resp.StatusCode)
	}

	return nil
}

func addUsers(users <-chan string, results chan<- error) {
	for u := range users {
		err := addUser(u)
		if err != nil {
			results <- err
			return
		}
	}
	results <- nil
}

func newUsers() <-chan string {
	users := make(chan string, 2*numWorkers)

	go func() {
		defer close(users)

		for i := 0; i < numUsers; i++ {
			users <- fmt.Sprintf("user-%d", i)
		}
	}()

	return users
}

func init() {
	t := &http.Transport{MaxIdleConnsPerHost: numWorkers}
	httpClient = &http.Client{Transport: t}
}

func main() {
	flag.IntVar(&numUsers, "users", 30000, "Number of users to create")
	flag.IntVar(&numWorkers, "workers", 500, "Number of workers to use")

	flag.StringVar(&hostname, "hostname", "127.0.0.1", "Target hostname")
	flag.StringVar(&admin, "admin", "Administrator", "Administrator username")
	flag.StringVar(&password, "password", "password", "Administrator password")

	flag.Parse()

	users := newUsers()
	results := make(chan error, numWorkers)

	for w := 0; w < numWorkers; w++ {
		go addUsers(users, results)
	}

	for w := 0; w < numWorkers; w++ {
		r := <-results
		if r != nil {
			log.Fatalln(r)
		}
	}
}
<commit_msg>Do not hard-code ns_server port<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
)

const (
	baseURL = "http://%s/settings/rbac/users/local/%s"
)

var (
	numUsers, numWorkers     int
	address, admin, password string
	httpClient               *http.Client
)

func newBody() io.Reader {
	form := url.Values{}
	form.Add("roles", "data_writer[*],data_reader[*],query_select[*]")
	form.Add("password", "Pa$$w0rd")

	return strings.NewReader(form.Encode())
}

func addUser(user string) error {
	log.Printf("Adding a new user: %s\n", user)

	req, err := http.NewRequest("PUT", fmt.Sprintf(baseURL, address, user), newBody())
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.SetBasicAuth(admin, password)

	resp, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	io.Copy(ioutil.Discard, resp.Body)

	if resp.StatusCode != 200 {
		return fmt.Errorf("unexpected status code: %v", resp.StatusCode)
	}

	return nil
}

func addUsers(users <-chan string, results chan<- error) {
	for u := range users {
		err := addUser(u)
		if err != nil {
			results <- err
			return
		}
	}
	results <- nil
}

func newUsers() <-chan string {
	users := make(chan string, 2*numWorkers)

	go func() {
		defer close(users)

		for i := 0; i < numUsers; i++ {
			users <- fmt.Sprintf("user-%d", i)
		}
	}()

	return users
}

func init() {
	t := &http.Transport{MaxIdleConnsPerHost: numWorkers}
	httpClient = &http.Client{Transport: t}
}

func main() {
	flag.IntVar(&numUsers, "users", 30000, "Number of users to create")
	flag.IntVar(&numWorkers, "workers", 500, "Number of workers to use")

	flag.StringVar(&address, "address", "127.0.0.1:8091", "Target address")
	flag.StringVar(&admin, "admin", "Administrator", "Administrator username")
	flag.StringVar(&password, "password", "password", "Administrator password")

	flag.Parse()

	users := newUsers()
	results := make(chan error, numWorkers)

	for w := 0; w < numWorkers; w++ {
		go addUsers(users, results)
	}

	for w := 0; w < numWorkers; w++ {
		r := <-results
		if r != nil {
			log.Fatalln(r)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pb "github.com/brotherlogic/gobuildslave/proto"
	pbgs "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

// RunJob - runs the job
func (s *Server) RunJob(ctx context.Context, req *pb.RunRequest) (*pb.RunResponse, error) {
	if !req.GetJob().GetBreakout() &&
		(s.Registry.Identifier == "clust6" ||
			s.Registry.Identifier == "clust3" ||
			s.Registry.Identifier == "clust7" ||
			s.Registry.Identifier == "clust8" ||
			s.Registry.Identifier == "clust4") {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "we only run the basic set of jobs")
	}

	if req.GetBits() > 0 && s.Bits != int(req.GetBits()) {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Cannot run %v bits on this server", req.GetBits())
	}

	if !s.doesBuild && !req.Job.Breakout {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Refusing to build")
	}
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; ok {
		return &pb.RunResponse{}, fmt.Errorf("Already running this job!")
	}

	if len(s.njobs) > s.maxJobs && !req.GetJob().GetBreakout() {
		return nil, status.Errorf(codes.FailedPrecondition, "We're running %v jobs, can't run no more", len(s.njobs))
	}

	s.CtxLog(ctx, "Running %v")

	s.njobs[req.GetJob().GetName()] = &pb.JobAssignment{Job: req.GetJob(), LastTransitionTime: time.Now().Unix(), Bits: int32(s.Bits)}
	go s.nmonitor(s.njobs[req.GetJob().GetName()])

	return &pb.RunResponse{}, nil
}

// KillJob - kills the job
func (s *Server) KillJob(ctx context.Context, req *pb.KillRequest) (*pb.KillResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()

	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_KILLING
	return &pb.KillResponse{}, nil
}

//UpdateJob - updates the job
func (s *Server) UpdateJob(ctx context.Context, req *pb.UpdateRequest) (*pb.UpdateResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_UPDATE_STARTING
	return &pb.UpdateResponse{}, nil
}

// ListJobs - lists the jobs
func (s *Server) ListJobs(ctx context.Context, req *pb.ListRequest) (*pb.ListResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	resp := &pb.ListResponse{}
	for _, job := range s.njobs {
		resp.Jobs = append(resp.Jobs, job)
	}
	return resp, nil
}

func extractBitRate(output string) (string, string) {
	matcher := regexp.MustCompile("Rate=(.*?) ")
	matches := matcher.FindStringSubmatch(output)

	matcher2 := regexp.MustCompile("Access Point. ([A-F0-9:]*)")
	matches2 := matcher2.FindStringSubmatch(output)
	if len(matches) > 0 && len(matches2) > 0 {
		return strings.TrimRight(matches[1], " "), strings.TrimRight(matches2[1], " ")
	}
	return "", ""
}

// SlaveConfig gets the config for this slave
func (s *Server) SlaveConfig(ctx context.Context, req *pb.ConfigRequest) (*pb.ConfigResponse, error) {
	disks := s.disker.getDisks()
	requirements := make([]*pb.Requirement, 0)
	for _, disk := range disks {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_DISK, Properties: disk})
	}
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_SERVER, Properties: s.Registry.Identifier})
	if s.Registry.Identifier == "argon" {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_EXTERNAL, Properties: "external_ready"})
	}

	data, err := exec.Command("/usr/bin/lsusb").Output()
	if err != nil {
		return nil, fmt.Errorf("error listing usb components: %v", err)
	}
	s.CtxLog(ctx, fmt.Sprintf("USBRES: %v", string(data)))
	if strings.Contains(string(data), "TSP100II") {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_RECEIPT_PRINTER})
	}

	out, _ := exec.Command("/sbin/iwconfig").Output()
	br, ap := extractBitRate(string(out))
	s.accessPoint = ap
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_NETWORK, Properties: br})
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ACCESS_POINT, Properties: ap})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_BITS, Properties: fmt.Sprintf("%v", s.Bits)})

	out, _ = exec.Command("cat", "/sys/firmware/devicetree/base/model").Output()
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_HOST_TYPE, Properties: string(out)})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ZONE, Properties: s.Registry.Zone})

	return &pb.ConfigResponse{Config: &pb.SlaveConfig{Requirements: requirements}}, nil
}

func (s *Server) FullShutdown(ctx context.Context, req *pb.ShutdownRequest) (*pb.ShutdownResponse, error) {
	defer func() {
		time.Sleep(time.Minute)

		exec.Command("sudo", "shutdown", "-h", "now")
	}()

	jobs, err := s.ListJobs(ctx, &pb.ListRequest{})
	if err != nil {
		return nil, err
	}

	for _, job := range jobs.GetJobs() {
		if job.GetPort() != 0 {
			conn, err := utils.LFDial(fmt.Sprintf("%v:%v", job.GetHost(), job.GetPort()))
			if err != nil {
				return nil, err
			}
			s.CtxLog(ctx, fmt.Sprintf("Calling shutdown on %v", job))
			gsclient := pbgs.NewGoserverServiceClient(conn)
			_, err = gsclient.Shutdown(ctx, &pbgs.ShutdownRequest{})
			if err != nil {
				return nil, err
			}
		} else {
			s.CtxLog(ctx, fmt.Sprintf("Not shutting down %v", job))
		}
	}

	return &pb.ShutdownResponse{}, nil
}
<commit_msg>Also this<commit_after>package main

import (
	"context"
	"fmt"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pb "github.com/brotherlogic/gobuildslave/proto"
	pbgs "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

// RunJob - runs the job
func (s *Server) RunJob(ctx context.Context, req *pb.RunRequest) (*pb.RunResponse, error) {
	if !req.GetJob().GetBreakout() &&
		(s.Registry.Identifier == "clust6" ||
			s.Registry.Identifier == "clust3" ||
			s.Registry.Identifier == "clust7" ||
			s.Registry.Identifier == "clust8" ||
			s.Registry.Identifier == "clust4") {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "we only run the basic set of jobs")
	}

	if req.GetBits() > 0 && s.Bits != int(req.GetBits()) {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Cannot run %v bits on this server", req.GetBits())
	}

	if !s.doesBuild && !req.Job.Breakout {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Refusing to build")
	}
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; ok {
		return &pb.RunResponse{}, fmt.Errorf("Already running this job!")
	}

	if len(s.njobs) > s.maxJobs && !req.GetJob().GetBreakout() {
		return nil, status.Errorf(codes.FailedPrecondition, "We're running %v jobs, can't run no more", len(s.njobs))
	}

	s.CtxLog(ctx, "Running %v")

	s.njobs[req.GetJob().GetName()] = &pb.JobAssignment{Job: req.GetJob(), LastTransitionTime: time.Now().Unix(), Bits: int32(s.Bits)}
	go s.nmonitor(s.njobs[req.GetJob().GetName()])

	return &pb.RunResponse{}, nil
}

// KillJob - kills the job
func (s *Server) KillJob(ctx context.Context, req *pb.KillRequest) (*pb.KillResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()

	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_KILLING
	return &pb.KillResponse{}, nil
}

//UpdateJob - updates the job
func (s *Server) UpdateJob(ctx context.Context, req *pb.UpdateRequest) (*pb.UpdateResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_UPDATE_STARTING
	return &pb.UpdateResponse{}, nil
}

// ListJobs - lists the jobs
func (s *Server) ListJobs(ctx context.Context, req *pb.ListRequest) (*pb.ListResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	resp := &pb.ListResponse{}
	for _, job := range s.njobs {
		resp.Jobs = append(resp.Jobs, job)
	}
	return resp, nil
}

func extractBitRate(output string) (string, string) {
	matcher := regexp.MustCompile("Rate=(.*?) ")
	matches := matcher.FindStringSubmatch(output)

	matcher2 := regexp.MustCompile("Access Point. ([A-F0-9:]*)")
	matches2 := matcher2.FindStringSubmatch(output)
	if len(matches) > 0 && len(matches2) > 0 {
		return strings.TrimRight(matches[1], " "), strings.TrimRight(matches2[1], " ")
	}
	return "", ""
}

// SlaveConfig gets the config for this slave
func (s *Server) SlaveConfig(ctx context.Context, req *pb.ConfigRequest) (*pb.ConfigResponse, error) {
	disks := s.disker.getDisks()
	requirements := make([]*pb.Requirement, 0)
	for _, disk := range disks {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_DISK, Properties: disk})
	}
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_SERVER, Properties: s.Registry.Identifier})
	if s.Registry.Identifier == "argon" {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_EXTERNAL, Properties: "external_ready"})
	}

	data, err := exec.Command("/usr/bin/lsusb").Output()
	if err != nil {
		return nil, fmt.Errorf("error listing usb components: %v", err)
	}
	s.CtxLog(ctx, fmt.Sprintf("USBRES: %v", string(data)))
	if strings.Contains(string(data), "TSP100II") {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_RECEIPT_PRINTER})
	}

	out, _ := exec.Command("/sbin/iwconfig").Output()
	br, ap := extractBitRate(string(out))
	s.accessPoint = ap
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_NETWORK, Properties: br})
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ACCESS_POINT, Properties: ap})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_BITS, Properties: fmt.Sprintf("%v", s.Bits)})

	out, _ = exec.Command("cat", "/sys/firmware/devicetree/base/model").Output()
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_HOST_TYPE, Properties: string(out)})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ZONE, Properties: s.Registry.Zone})

	return &pb.ConfigResponse{Config: &pb.SlaveConfig{Requirements: requirements}}, nil
}

func (s *Server) FullShutdown(ctx context.Context, req *pb.ShutdownRequest) (*pb.ShutdownResponse, error) {
	defer func() {
		time.Sleep(time.Minute)

		exec.Command("sudo", "shutdown", "-h", "now")
	}()

	jobs, err := s.ListJobs(ctx, &pb.ListRequest{})
	if err != nil {
		return nil, err
	}

	s.CtxLog(ctx, fmt.Sprintf("Shutting down %v jobs", len(jobs.GetJobs())))

	for _, job := range jobs.GetJobs() {
		if job.GetPort() != 0 {
			conn, err := utils.LFDial(fmt.Sprintf("%v:%v", job.GetHost(), job.GetPort()))
			if err != nil {
				return nil, err
			}
			s.CtxLog(ctx, fmt.Sprintf("Calling shutdown on %v", job))
			gsclient := pbgs.NewGoserverServiceClient(conn)
			_, err = gsclient.Shutdown(ctx, &pbgs.ShutdownRequest{})
			if err != nil {
				return nil, err
			}
		} else {
			s.CtxLog(ctx, fmt.Sprintf("Not shutting down %v", job))
		}
	}

	return &pb.ShutdownResponse{}, nil
}
<|endoftext|>


<commit_before>package field

import (
	"strconv"
	"time"

	"github.com/graphql-go/graphql"
	"github.com/samuelngs/hyper/gql/interfaces"
	"github.com/samuelngs/hyper/gql/server"
	"github.com/samuelngs/hyper/router"
)

type field struct {
	name, description, deprecated string
	typ                           graphql.Output
	args                          []interfaces.Argument
	resolve                       interfaces.ResolveHandler
}

func (v *field) Name(s string) interfaces.Field {
	v.name = s
	return v
}

func (v *field) Description(s string) interfaces.Field {
	v.description = s
	return v
}

func (v *field) DeprecationReason(s string) interfaces.Field {
	v.deprecated = s
	return v
}

func (v *field) Type(t interface{}) interfaces.Field {
	switch o := t.(type) {
	case graphql.Output:
		v.typ = o
	case interfaces.Object:
		v.typ = o.ToObject()
	}
	return v
}

func (v *field) Args(args ...interfaces.Argument) interfaces.Field {
	v.args = append(v.args, args...)
	return v
}

func (v *field) Resolve(h interfaces.ResolveHandler) interfaces.Field {
	v.resolve = h
	return v
}

func (v *field) ResolveParameters(params map[string]interface{}, values []interfaces.Value, args []interfaces.Argument) {
	for i, arg := range args {
		name, conf := arg.ToArgumentConfig()
		data := &value{
			key: name,
		}
		switch conf.Type {
		case graphql.Int:
			data.fmt = router.Int
		case graphql.Float:
			data.fmt = router.F64
		case graphql.String:
			data.fmt = router.Text
		case graphql.Boolean:
			data.fmt = router.Bool
		case graphql.ID:
			data.fmt = router.Text
		case graphql.DateTime:
			data.fmt = router.DateTimeRFC3339
		default:
			data.fmt = router.Any
		}
		if k, ok := params[name]; ok {
			switch o := k.(type) {
			case string:
				data.val = []byte(o)
				data.has = true
			case int:
				data.val = []byte(strconv.Itoa(o))
				data.has = true
				data.parsed = o
			case float64:
				data.val = []byte(strconv.FormatFloat(o, 'E', -1, 64))
				data.has = true
				data.parsed = o
			case bool:
				data.val = []byte(strconv.FormatBool(o))
				data.has = true
				data.parsed = o
			case time.Time:
				data.val = []byte(o.String())
				data.has = true
				data.parsed = o
			case map[string]interface{}:
				data.val = nil
				data.has = true
				if args := arg.InputObject().ExportArgs(); len(args) > 0 {
					arr := make([]interfaces.Value, len(args))
					v.ResolveParameters(o, arr, args)
					data.parsed = arr
				}
			}
		}
		switch data.has {
		case true && data.parsed == nil:
			if parsed, ok := router.Val(data.fmt, data.val); ok {
				data.parsed = parsed
			}
		case false:
			if o, ok := conf.DefaultValue.([]byte); ok {
				data.val = o
			}
		}
		values[i] = data
	}
}

func (v *field) Compile() *graphql.Field {
	args := graphql.FieldConfigArgument{}
	for _, arg := range v.args {
		k, v := arg.ToArgumentConfig()
		args[k] = v
	}
	hdfn := func(params graphql.ResolveParams) (interface{}, error) {
		c := server.FromContext(params.Context)
		r := &resolve{
			context: c,
			params:  params,
			values:  make([]interfaces.Value, len(v.args)),
		}
		v.ResolveParameters(params.Args, r.values, v.args)
		o, err := v.resolve(r)
		if err != nil {
			r.Context().GraphQLError(err)
			return nil, err
		}
		return o, nil
	}
	return &graphql.Field{
		Name:              v.name,
		Description:       v.description,
		DeprecationReason: v.deprecated,
		Type:              v.typ,
		Args:              args,
		Resolve:           hdfn,
	}
}

// NewField creates new field instance
func NewField(s string) interfaces.Field {
	return &field{name: s}
}
<commit_msg>bugfix for default value<commit_after>package field

import (
	"strconv"
	"time"

	"github.com/graphql-go/graphql"
	"github.com/samuelngs/hyper/gql/interfaces"
	"github.com/samuelngs/hyper/gql/server"
	"github.com/samuelngs/hyper/router"
)

type field struct {
	name, description, deprecated string
	typ                           graphql.Output
	args                          []interfaces.Argument
	resolve                       interfaces.ResolveHandler
}

func (v *field) Name(s string) interfaces.Field {
	v.name = s
	return v
}

func (v *field) Description(s string) interfaces.Field {
	v.description = s
	return v
}

func (v *field) DeprecationReason(s string) interfaces.Field {
	v.deprecated = s
	return v
}

func (v *field) Type(t interface{}) interfaces.Field {
	switch o := t.(type) {
	case graphql.Output:
		v.typ = o
	case interfaces.Object:
		v.typ = o.ToObject()
	}
	return v
}

func (v *field) Args(args ...interfaces.Argument) interfaces.Field {
	v.args = append(v.args, args...)
	return v
}

func (v *field) Resolve(h interfaces.ResolveHandler) interfaces.Field {
	v.resolve = h
	return v
}

func (v *field) ResolveParameters(params map[string]interface{}, values []interfaces.Value, args []interfaces.Argument) {
	for i, arg := range args {
		name, conf := arg.ToArgumentConfig()
		data := &value{
			key: name,
		}
		switch conf.Type {
		case graphql.Int:
			data.fmt = router.Int
		case graphql.Float:
			data.fmt = router.F64
		case graphql.String:
			data.fmt = router.Text
		case graphql.Boolean:
			data.fmt = router.Bool
		case graphql.ID:
			data.fmt = router.Text
		case graphql.DateTime:
			data.fmt = router.DateTimeRFC3339
		default:
			data.fmt = router.Any
		}
		if k, ok := params[name]; ok {
			switch o := k.(type) {
			case string:
				data.val = []byte(o)
				data.has = true
			case int:
				data.val = []byte(strconv.Itoa(o))
				data.has = true
				data.parsed = o
			case float64:
				data.val = []byte(strconv.FormatFloat(o, 'E', -1, 64))
				data.has = true
				data.parsed = o
			case bool:
				data.val = []byte(strconv.FormatBool(o))
				data.has = true
				data.parsed = o
			case time.Time:
				data.val = []byte(o.String())
				data.has = true
				data.parsed = o
			case map[string]interface{}:
				data.val = nil
				data.has = true
				if args := arg.InputObject().ExportArgs(); len(args) > 0 {
					arr := make([]interfaces.Value, len(args))
					v.ResolveParameters(o, arr, args)
					data.parsed = arr
				}
			}
		}
		if !data.has {
			if o, ok := conf.DefaultValue.([]byte); ok {
				data.val = o
			}
		}
		if data.val != nil && data.parsed == nil {
			if parsed, ok := router.Val(data.fmt, data.val); ok {
				data.parsed = parsed
			}
		}
		values[i] = data
	}
}

func (v *field) Compile() *graphql.Field {
	args := graphql.FieldConfigArgument{}
	for _, arg := range v.args {
		k, v := arg.ToArgumentConfig()
		args[k] = v
	}
	hdfn := func(params graphql.ResolveParams) (interface{}, error) {
		c := server.FromContext(params.Context)
		r := &resolve{
			context: c,
			params:  params,
			values:  make([]interfaces.Value, len(v.args)),
		}
		v.ResolveParameters(params.Args, r.values, v.args)
		o, err := v.resolve(r)
		if err != nil {
			r.Context().GraphQLError(err)
			return nil, err
		}
		return o, nil
	}
	return &graphql.Field{
		Name:              v.name,
		Description:       v.description,
		DeprecationReason: v.deprecated,
		Type:              v.typ,
		Args:              args,
		Resolve:           hdfn,
	}
}

// NewField creates new field instance
func NewField(s string) interfaces.Field {
	return &field{name: s}
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	gitc "git_comment"
	ex "git_comment/exec"
	"strings"
	"time"
)

const (
	commentFull          = "%C"
	commentShort         = "%c"
	commitFull           = "%H"
	commitShort          = "%h"
	filePath             = "%f"
	lineNumber           = "%L"
	authorName           = "%an"
	authorEmail          = "%ae"
	authorDateISO8601    = "%ad"
	authorDateUnix       = "%aU"
	committerName        = "%kn"
	committerEmail       = "%ke"
	committerDateISO8601 = "%kd"
	committerDateUnix    = "%kU"
	bodyContent          = "%b"
	titleLine            = "%t"
	newLine              = "%n"
	dividerLine          = "%d"
	black                = "black("
	red                  = "red("
	green                = "green("
	yellow               = "yellow("
	blue                 = "blue("
	magenta              = "magenta("
	cyan                 = "cyan("
	white                = "white("
	resetColor           = ")"
)

const (
	Short         = "short"
	Full          = "full"
	Raw           = "raw"
	ShortFormat   = "[%h] %c %an <%ae>\n%t\n\n"
	FullFormat    = "commit  %H\ncomment %C\nAuthor: %an <%ae>\n%b\n\n"
	RawFormat     = "comment %C\n%v\n\n"
	formatPrefix  = "format:"
	invalidFormat = "Unknown pretty format."
	lineNumberMax = 5
)

type Formatter struct {
	format         string
	useLineNumbers bool
	useColor       bool
	termWidth      uint16
	colorMapping   map[string]string
	indent         string
}

func NewFormatter(format string, useLineNumbers, useColor bool, termWidth uint16) *Formatter {
	var indent = "\n  "
	var colorMapping map[string]string
	if useLineNumbers {
		indent = "\n            "
	}
	if useColor {
		colorMapping = map[string]string{
			black:      ex.Black,
			red:        ex.Red,
			green:      ex.Green,
			yellow:     ex.Yellow,
			blue:       ex.Blue,
			magenta:    ex.Magenta,
			cyan:       ex.Cyan,
			white:      ex.White,
			resetColor: ex.Clear,
		}
	} else {
		colorMapping = map[string]string{
			black:      "",
			red:        "",
			green:      "",
			yellow:     "",
			blue:       "",
			magenta:    "",
			cyan:       "",
			white:      "",
			resetColor: "",
		}
	}
	return &Formatter{format, useLineNumbers, useColor, termWidth, colorMapping, indent}
}

func (f *Formatter) FormatLine(line *gitc.DiffLine) string {
	prefix := f.formatLinePrefix(line)
	number := f.formatLineNumbers(line.OldLineNumber, line.NewLineNumber)
	content := f.formatLineContent(line)
	return fmt.Sprintf("%v %v %v", prefix, number, content)
}

func (f *Formatter) FormatComment(comment *gitc.Comment) string {
	switch {
	case f.format == Short || len(f.format) == 0:
		return f.substituteVariables(ShortFormat, comment)
	case f.format == Full:
		return f.substituteVariables(FullFormat, comment)
	case f.format == Raw:
		format := string(f.substituteVariables(RawFormat, comment))
		return fmt.Sprintf(format, *comment.ID)
	case strings.HasPrefix(f.format, formatPrefix):
		format := f.substituteVariables(f.format[len(formatPrefix):], comment)
		return f.substituteColors(format)
	}
	return ""
}

func (f *Formatter) formatLineNumber(number int) string {
	var line string
	if number < 0 {
		line = " "
	} else {
		line = fmt.Sprintf("%d", number)
	}
	for len(line) < lineNumberMax {
		line = fmt.Sprintf(" %v", line)
	}
	return line
}

func (f *Formatter) formatLineNumbers(oldNum, newNum int) string {
	if !f.useLineNumbers {
		return ""
	}
	var newLine string
	oldLine := f.formatLineNumber(oldNum)
	if oldNum == newNum || newNum == -1 {
		newLine = f.formatLineNumber(-1)
	} else {
		newLine = ex.Colorize(ex.Green, f.formatLineNumber(newNum), f.useColor)
	}
	if oldNum != -1 {
		oldLine = ex.Colorize(ex.Red, oldLine, f.useColor)
	}
	return fmt.Sprintf("%v%v", oldLine, newLine)
}

func (f *Formatter) FormatFilePath(file *gitc.DiffFile) string {
	var path string
	if file.OldPath == file.NewPath {
		path = file.OldPath
	} else if len(file.OldPath) > 0 && len(file.NewPath) > 0 {
		path = fmt.Sprintf("%v -> %v", file.OldPath, file.NewPath)
	} else if len(file.OldPath) > 0 {
		path = file.OldPath
	} else {
		path = file.NewPath
	}
	return fmt.Sprintf("%v\n", path)
}

func (f *Formatter) formatLinePrefix(line *gitc.DiffLine) string {
	switch line.Type {
	case gitc.DiffAdd, gitc.DiffAddNewline:
		return ex.Colorize(ex.Green, "+", f.useColor)
	case gitc.DiffRemove, gitc.DiffRemoveNewline:
		return ex.Colorize(ex.Red, "-", f.useColor)
	default:
		return " "
	}
}

func (f *Formatter) formatLineContent(line *gitc.DiffLine) string {
	switch line.Type {
	case gitc.DiffAddNewline, gitc.DiffRemoveNewline:
		return "↵"
	case gitc.DiffAdd:
		return ex.Colorize(ex.Green, line.Content, f.useColor)
	case gitc.DiffRemove:
		return ex.Colorize(ex.Red, line.Content, f.useColor)
	default:
		return line.Content
	}
}

func (f *Formatter) substituteColors(format string) string {
	return replaceAll(format, f.colorMapping)
}

func (f *Formatter) substituteVariables(format string, comment *gitc.Comment) string {
	return replaceAll(format, f.commentMapping(comment))
}

func (f *Formatter) commentMapping(comment *gitc.Comment) map[string]string {
	var path = ""
	var line = ""
	if comment.FileRef != nil {
		path = comment.FileRef.Path
		line = fmt.Sprintf("%v", comment.FileRef.Line)
	}
	return map[string]string{
		authorName:           comment.Author.Name,
		authorEmail:          comment.Author.Email,
		authorDateISO8601:    comment.CreateTime.Format(time.RFC3339),
		authorDateUnix:       fmt.Sprintf("%v", comment.CreateTime.Unix()),
		committerName:        comment.Amender.Name,
		committerEmail:       comment.Amender.Email,
		committerDateISO8601: comment.AmendTime.Format(time.RFC3339),
		committerDateUnix:    fmt.Sprintf("%v", comment.AmendTime.Unix()),
		commentFull:          *comment.ID,
		commentShort:         (*comment.ID)[:7],
		commitFull:           *comment.Commit,
		commitShort:          (*comment.Commit)[:7],
		bodyContent:          comment.Content,
		titleLine:            strings.Split(comment.Content, "\n")[0],
		filePath:             path,
		lineNumber:           line,
		newLine:              f.indent,
		dividerLine:          strings.Repeat("-", int(f.termWidth)),
	}
}

func replaceAll(format string, substitutions map[string]string) string {
	for key, value := range substitutions {
		format = strings.Replace(format, key, value, -1)
	}
	return format
}
<commit_msg>Fix extra indentation after line number display<commit_after>package log

import (
	"fmt"
	gitc "git_comment"
	ex "git_comment/exec"
	"strings"
	"time"
)

const (
	commentFull          = "%C"
	commentShort         = "%c"
	commitFull           = "%H"
	commitShort          = "%h"
	filePath             = "%f"
	lineNumber           = "%L"
	authorName           = "%an"
	authorEmail          = "%ae"
	authorDateISO8601    = "%ad"
	authorDateUnix       = "%aU"
	committerName        = "%kn"
	committerEmail       = "%ke"
	committerDateISO8601 = "%kd"
	committerDateUnix    = "%kU"
	bodyContent          = "%b"
	titleLine            = "%t"
	newLine              = "%n"
	dividerLine          = "%d"
	black                = "black("
	red                  = "red("
	green                = "green("
	yellow               = "yellow("
	blue                 = "blue("
	magenta              = "magenta("
	cyan                 = "cyan("
	white                = "white("
	resetColor           = ")"
)

const (
	Short         = "short"
	Full          = "full"
	Raw           = "raw"
	ShortFormat   = "[%h] %c %an <%ae>\n%t\n\n"
	FullFormat    = "commit  %H\ncomment %C\nAuthor: %an <%ae>\n%b\n\n"
	RawFormat     = "comment %C\n%v\n\n"
	formatPrefix  = "format:"
	invalidFormat = "Unknown pretty format."
	lineNumberMax = 5
)

type Formatter struct {
	format         string
	useLineNumbers bool
	useColor       bool
	termWidth      uint16
	colorMapping   map[string]string
	indent         string
}

func NewFormatter(format string, useLineNumbers, useColor bool, termWidth uint16) *Formatter {
	var indent = "\n  "
	var colorMapping map[string]string
	if useLineNumbers {
		indent = "\n            "
	}
	if useColor {
		colorMapping = map[string]string{
			black:      ex.Black,
			red:        ex.Red,
			green:      ex.Green,
			yellow:     ex.Yellow,
			blue:       ex.Blue,
			magenta:    ex.Magenta,
			cyan:       ex.Cyan,
			white:      ex.White,
			resetColor: ex.Clear,
		}
	} else {
		colorMapping = map[string]string{
			black:      "",
			red:        "",
			green:      "",
			yellow:     "",
			blue:       "",
			magenta:    "",
			cyan:       "",
			white:      "",
			resetColor: "",
		}
	}
	return &Formatter{format, useLineNumbers, useColor, termWidth, colorMapping, indent}
}

func (f *Formatter) FormatLine(line *gitc.DiffLine) string {
	prefix := f.formatLinePrefix(line)
	number := f.formatLineNumbers(line.OldLineNumber, line.NewLineNumber)
	content := f.formatLineContent(line)
	return fmt.Sprintf("%v %v %v", prefix, number, content)
}

func (f *Formatter) FormatComment(comment *gitc.Comment) string {
	switch {
	case f.format == Short || len(f.format) == 0:
		return f.substituteVariables(ShortFormat, comment)
	case f.format == Full:
		return f.substituteVariables(FullFormat, comment)
	case f.format == Raw:
		format := string(f.substituteVariables(RawFormat, comment))
		return fmt.Sprintf(format, *comment.ID)
	case strings.HasPrefix(f.format, formatPrefix):
		format := f.substituteVariables(f.format[len(formatPrefix):], comment)
		return fmt.Sprintf("%v\n", f.substituteColors(format))
	}
	return ""
}

func (f *Formatter) formatLineNumber(number int) string {
	var line string
	if number < 0 {
		line = " "
	} else {
		line = fmt.Sprintf("%d", number)
	}
	for len(line) < lineNumberMax {
		line = fmt.Sprintf(" %v", line)
	}
	return line
}

func (f *Formatter) formatLineNumbers(oldNum, newNum int) string {
	if !f.useLineNumbers {
		return ""
	}
	var newLine string
	oldLine := f.formatLineNumber(oldNum)
	if oldNum == newNum || newNum == -1 {
		newLine = f.formatLineNumber(-1)
	} else {
		newLine = ex.Colorize(ex.Green, f.formatLineNumber(newNum), f.useColor)
	}
	if oldNum != -1 {
		oldLine = ex.Colorize(ex.Red, oldLine, f.useColor)
	}
	return fmt.Sprintf("%v%v", oldLine, newLine)
}

func (f *Formatter) FormatFilePath(file *gitc.DiffFile) string {
	var path string
	if file.OldPath == file.NewPath {
		path = file.OldPath
	} else if len(file.OldPath) > 0 && len(file.NewPath) > 0 {
		path = fmt.Sprintf("%v -> %v", file.OldPath, file.NewPath)
	} else if len(file.OldPath) > 0 {
		path = file.OldPath
	} else {
		path = file.NewPath
	}
	return fmt.Sprintf("%v\n", path)
}

func (f *Formatter) formatLinePrefix(line *gitc.DiffLine) string {
	switch line.Type {
	case gitc.DiffAdd, gitc.DiffAddNewline:
		return ex.Colorize(ex.Green, "+", f.useColor)
	case gitc.DiffRemove, gitc.DiffRemoveNewline:
		return ex.Colorize(ex.Red, "-", f.useColor)
	default:
		return " "
	}
}

func (f *Formatter) formatLineContent(line *gitc.DiffLine) string {
	switch line.Type {
	case gitc.DiffAddNewline, gitc.DiffRemoveNewline:
		return "↵"
	case gitc.DiffAdd:
		return ex.Colorize(ex.Green, line.Content, f.useColor)
	case gitc.DiffRemove:
		return ex.Colorize(ex.Red, line.Content, f.useColor)
	default:
		return line.Content
	}
}

func (f *Formatter) substituteColors(format string) string {
	return replaceAll(format, f.colorMapping)
}

func (f *Formatter) substituteVariables(format string, comment *gitc.Comment) string {
	return replaceAll(format, f.commentMapping(comment))
}

func (f *Formatter) commentMapping(comment *gitc.Comment) map[string]string {
	var path = ""
	var line = ""
	if comment.FileRef != nil {
		path = comment.FileRef.Path
		line = fmt.Sprintf("%v", comment.FileRef.Line)
	}
	return map[string]string{
		authorName:           comment.Author.Name,
		authorEmail:          comment.Author.Email,
		authorDateISO8601:    comment.CreateTime.Format(time.RFC3339),
		authorDateUnix:       fmt.Sprintf("%v", comment.CreateTime.Unix()),
		committerName:        comment.Amender.Name,
		committerEmail:       comment.Amender.Email,
		committerDateISO8601: comment.AmendTime.Format(time.RFC3339),
		committerDateUnix:    fmt.Sprintf("%v", comment.AmendTime.Unix()),
		commentFull:          *comment.ID,
		commentShort:         (*comment.ID)[:7],
		commitFull:           *comment.Commit,
		commitShort:          (*comment.Commit)[:7],
		bodyContent:          comment.Content,
		titleLine:            strings.Split(comment.Content, "\n")[0],
		filePath:             path,
		lineNumber:           line,
		newLine:              f.indent,
		dividerLine:          strings.Repeat("-", int(f.termWidth)),
	}
}

func replaceAll(format string, substitutions map[string]string) string {
	for key, value := range substitutions {
		format = strings.Replace(format, key, value, -1)
	}
	return format
}
<|endoftext|>


<commit_before>// @author       sigu-399
// @description  An implementation of JSON Reference - Go language
// @created      26-02-2013

package gojsonreference

import ()

func NewJsonReference(jsonReferenceString string) (JsonReference, error) {

	var r JsonReference
	err := r.parse(jsonReferenceString)
	return r, err

}

type JsonReference struct {
}

func (r *JsonReference) parse(JsonReferenceString string) error {

	var err error

	return err
}
<commit_msg>implemented parsing<commit_after>// @author       sigu-399
// @description  An implementation of JSON Reference - Go language
// @created      26-02-2013

package gojsonreference

import (
	"gojsonpointer"
	"net/url"
)

func NewJsonReference(jsonReferenceString string) (JsonReference, error) {

	var r JsonReference
	err := r.parse(jsonReferenceString)
	return r, err

}

type JsonReference struct {
	referenceUrl     *url.URL
	referencePointer gojsonpointer.JsonPointer
}

func (r *JsonReference) GetUrl() *url.URL {
	return r.referenceUrl
}

func (r *JsonReference) GetPointer() *gojsonpointer.JsonPointer {
	return &r.referencePointer
}

func (r *JsonReference) parse(jsonReferenceString string) error {

	var err error

	r.referenceUrl, err = url.Parse(jsonReferenceString)
	if err == nil {
		r.referencePointer, err = gojsonpointer.NewJsonPointer(r.referenceUrl.Fragment)
	}

	return err
}<|endoftext|>


<commit_before>package processor

import (
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/wanelo/image-server/core"
	adapter "github.com/wanelo/image-server/processor/cli"
)

type ProcessorResult struct {
	ResizedPath string
	Error       error
}

var ImageProcessings map[string][]chan ProcessorResult

func init() {
	ImageProcessings = make(map[string][]chan ProcessorResult)
}

type Processor struct {
	Source             string
	Destination        string
	ImageConfiguration *core.ImageConfiguration
	Channels           *ProcessorChannels
}

type ProcessorChannels struct {
	ImageProcessed chan *core.ImageConfiguration
	Skipped        chan string
}

func (p *Processor) CreateImage() error {
	c := make(chan ProcessorResult)
	go p.uniqueCreateImage(c)
	ipr := <-c
	return ipr.Error
}

func (p *Processor) uniqueCreateImage(c chan ProcessorResult) {
	key := p.Destination
	_, present := ImageProcessings[key]

	if present {
		ImageProcessings[key] = append(ImageProcessings[key], c)
		p.notifySkipped(key)
	} else {
		ImageProcessings[key] = []chan ProcessorResult{c}
		err := p.createIfNotAvailable()

		for _, cc := range ImageProcessings[key] {
			cc <- ProcessorResult{p.Destination, err}
		}
		delete(ImageProcessings, key)
		p.notifyProcessed()
	}
}

func (p *Processor) createIfNotAvailable() error {
	if _, err := os.Stat(p.Destination); os.IsNotExist(err) {
		start := time.Now()

		dir := filepath.Dir(p.Destination)
		os.MkdirAll(dir, 0700)

		processor := &adapter.Processor{}
		err = processor.CreateImage(p.Source, p.Destination, p.ImageConfiguration)

		if err != nil {
			log.Println(err)
			return err
		}

		elapsed := time.Since(start)
		log.Printf("Took %s to generate image: %s", elapsed, p.Destination)
		p.notifyProcessed()
	} else {
		p.notifySkipped(p.Destination)
	}

	return nil
}

func (p *Processor) notifyProcessed() {
	go func() {
		p.Channels.ImageProcessed <- p.ImageConfiguration
	}()
}

func (p *Processor) notifySkipped(path string) {
	go func() {
		p.Channels.Skipped <- path
	}()
}
<commit_msg>Processer uses a mutex around global hash access<commit_after>package processor

import (
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/wanelo/image-server/core"
	adapter "github.com/wanelo/image-server/processor/cli"
)

type ProcessorResult struct {
	ResizedPath string
	Error       error
}

var ImageProcessings map[string][]chan ProcessorResult
var processingMutex sync.RWMutex // To protect ImageProcessings

func init() {
	ImageProcessings = make(map[string][]chan ProcessorResult)
}

type Processor struct {
	Source             string
	Destination        string
	ImageConfiguration *core.ImageConfiguration
	Channels           *ProcessorChannels
}

type ProcessorChannels struct {
	ImageProcessed chan *core.ImageConfiguration
	Skipped        chan string
}

func (p *Processor) CreateImage() error {
	c := make(chan ProcessorResult)
	go p.uniqueCreateImage(c)
	ipr := <-c
	return ipr.Error
}

func (p *Processor) uniqueCreateImage(c chan ProcessorResult) {
	key := p.Destination
	_, present := ImageProcessings[key]

	processingMutex.Lock()

	if present {
		ImageProcessings[key] = append(ImageProcessings[key], c)
		processingMutex.Unlock()
		p.notifySkipped(key)
	} else {
		ImageProcessings[key] = []chan ProcessorResult{c}
		processingMutex.Unlock()

		err := p.createIfNotAvailable()

		for _, cc := range ImageProcessings[key] {
			cc <- ProcessorResult{p.Destination, err}
		}
		processingMutex.Lock()
		delete(ImageProcessings, key)
		processingMutex.Unlock()
		p.notifyProcessed()
	}

}

func (p *Processor) createIfNotAvailable() error {
	if _, err := os.Stat(p.Destination); os.IsNotExist(err) {
		start := time.Now()

		dir := filepath.Dir(p.Destination)
		os.MkdirAll(dir, 0700)

		processor := &adapter.Processor{}
		err = processor.CreateImage(p.Source, p.Destination, p.ImageConfiguration)

		if err != nil {
			log.Println(err)
			return err
		}

		elapsed := time.Since(start)
		log.Printf("Took %s to generate image: %s", elapsed, p.Destination)
		p.notifyProcessed()
	} else {
		p.notifySkipped(p.Destination)
	}

	return nil
}

func (p *Processor) notifyProcessed() {
	go func() {
		p.Channels.ImageProcessed <- p.ImageConfiguration
	}()
}

func (p *Processor) notifySkipped(path string) {
	go func() {
		p.Channels.Skipped <- path
	}()
}
<|endoftext|>


<commit_before>package config

import (
	"reflect"
	"testing"

	"github.com/mitchellh/reflectwalk"
)

func BenchmarkVariableDetectWalker(b *testing.B) {
	w := new(variableDetectWalker)
	str := reflect.ValueOf(`foo ${var.bar} bar ${bar.baz.bing} $${escaped}`)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		w.Variables = nil
		w.Primitive(str)
	}
}

func TestVariableDetectWalker(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo ${var.bar}`
	if err := w.Primitive(reflect.ValueOf(str)); err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(w.Variables) != 1 {
		t.Fatalf("bad: %#v", w.Variables)
	}
	if w.Variables["var.bar"].(*UserVariable).FullKey() != "var.bar" {
		t.Fatalf("bad: %#v", w.Variables)
	}
}

func TestVariableDetectWalker_resource(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo ${ec2.foo.bar}`
	if err := w.Primitive(reflect.ValueOf(str)); err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(w.Variables) != 1 {
		t.Fatalf("bad: %#v", w.Variables)
	}
	if w.Variables["ec2.foo.bar"].(*ResourceVariable).FullKey() != "ec2.foo.bar" {
		t.Fatalf("bad: %#v", w.Variables)
	}
}

func TestVariableDetectWalker_bad(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo ${bar}`
	if err := w.Primitive(reflect.ValueOf(str)); err == nil {
		t.Fatal("should error")
	}
}

func TestVariableDetectWalker_escaped(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo $${var.bar}`
	if err := w.Primitive(reflect.ValueOf(str)); err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(w.Variables) > 0 {
		t.Fatalf("bad: %#v", w.Variables)
	}
}

func TestVariableDetectWalker_empty(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo`
	if err := w.Primitive(reflect.ValueOf(str)); err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(w.Variables) > 0 {
		t.Fatalf("bad: %#v", w.Variables)
	}
}

func TestVariableReplaceWalker(t *testing.T) {
	w := &variableReplaceWalker{
		Values: map[string]string{
			"var.bar": "bar",
		},
	}

	cases := []struct {
		Input  interface{}
		Output interface{}
	}{
		{
			`foo ${var.bar}`,
			"foo bar",
		},
		{
			[]string{"foo", "${var.bar}"},
			[]string{"foo", "bar"},
		},
		{
			map[string]interface{}{
				"ami": "${var.bar}",
				"security_groups": []interface{}{
					"foo",
					"${var.bar}",
				},
			},
			map[string]interface{}{
				"ami": "bar",
				"security_groups": []interface{}{
					"foo",
					"bar",
				},
			},
		},
	}

	for i, tc := range cases {
		var input interface{} = tc.Input
		if reflect.ValueOf(tc.Input).Kind() == reflect.String {
			input = &tc.Input
		}

		if err := reflectwalk.Walk(input, w); err != nil {
			t.Fatalf("err: %s", err)
		}

		if !reflect.DeepEqual(tc.Input, tc.Output) {
			t.Fatalf("bad %d: %#v", i, tc.Input)
		}
	}
}
<commit_msg>config: add benchmark for replace walker<commit_after>package config

import (
	"reflect"
	"testing"

	"github.com/mitchellh/reflectwalk"
)

func BenchmarkVariableDetectWalker(b *testing.B) {
	w := new(variableDetectWalker)
	str := reflect.ValueOf(`foo ${var.bar} bar ${bar.baz.bing} $${escaped}`)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		w.Variables = nil
		w.Primitive(str)
	}
}

func BenchmarkVariableReplaceWalker(b *testing.B) {
	w := &variableReplaceWalker{
		Values: map[string]string{
			"var.bar":      "bar",
			"bar.baz.bing": "baz",
		},
	}

	str := `foo ${var.bar} bar ${bar.baz.bing} $${escaped}`

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err := reflectwalk.Walk(&str, w); err != nil {
			panic(err)
		}
	}
}

func TestVariableDetectWalker(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo ${var.bar}`
	if err := w.Primitive(reflect.ValueOf(str)); err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(w.Variables) != 1 {
		t.Fatalf("bad: %#v", w.Variables)
	}
	if w.Variables["var.bar"].(*UserVariable).FullKey() != "var.bar" {
		t.Fatalf("bad: %#v", w.Variables)
	}
}

func TestVariableDetectWalker_resource(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo ${ec2.foo.bar}`
	if err := w.Primitive(reflect.ValueOf(str)); err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(w.Variables) != 1 {
		t.Fatalf("bad: %#v", w.Variables)
	}
	if w.Variables["ec2.foo.bar"].(*ResourceVariable).FullKey() != "ec2.foo.bar" {
		t.Fatalf("bad: %#v", w.Variables)
	}
}

func TestVariableDetectWalker_bad(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo ${bar}`
	if err := w.Primitive(reflect.ValueOf(str)); err == nil {
		t.Fatal("should error")
	}
}

func TestVariableDetectWalker_escaped(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo $${var.bar}`
	if err := w.Primitive(reflect.ValueOf(str)); err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(w.Variables) > 0 {
		t.Fatalf("bad: %#v", w.Variables)
	}
}

func TestVariableDetectWalker_empty(t *testing.T) {
	w := new(variableDetectWalker)

	str := `foo`
	if err := w.Primitive(reflect.ValueOf(str)); err != nil {
		t.Fatalf("err: %s", err)
	}

	if len(w.Variables) > 0 {
		t.Fatalf("bad: %#v", w.Variables)
	}
}

func TestVariableReplaceWalker(t *testing.T) {
	w := &variableReplaceWalker{
		Values: map[string]string{
			"var.bar": "bar",
		},
	}

	cases := []struct {
		Input  interface{}
		Output interface{}
	}{
		{
			`foo ${var.bar}`,
			"foo bar",
		},
		{
			[]string{"foo", "${var.bar}"},
			[]string{"foo", "bar"},
		},
		{
			map[string]interface{}{
				"ami": "${var.bar}",
				"security_groups": []interface{}{
					"foo",
					"${var.bar}",
				},
			},
			map[string]interface{}{
				"ami": "bar",
				"security_groups": []interface{}{
					"foo",
					"bar",
				},
			},
		},
	}

	for i, tc := range cases {
		var input interface{} = tc.Input
		if reflect.ValueOf(tc.Input).Kind() == reflect.String {
			input = &tc.Input
		}

		if err := reflectwalk.Walk(input, w); err != nil {
			t.Fatalf("err: %s", err)
		}

		if !reflect.DeepEqual(tc.Input, tc.Output) {
			t.Fatalf("bad %d: %#v", i, tc.Input)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"runtime"
	"taskrunner-app/taskrunner-gtk/gui"
	"taskrunner-app/taskrunnerdal"
	"time"

	"github.com/alecthomas/kingpin"
	"github.com/mattn/go-gtk/gtk"

	"github.com/mattn/go-gtk/glib"

	"github.com/jamesrr39/goutil/user"
	"github.com/mattn/go-gtk/gdk"
)

var (
	taskrunnerDAL         *taskrunnerdal.TaskrunnerDAL
	taskrunnerApplication *kingpin.Application
	jobLogMaxLines        *uint
)

func main() {

	taskrunnerApplication = kingpin.New("Taskrunner GUI", "gtk gui for taskrunner")
	setupApplicationFlags()
	kingpin.MustParse(taskrunnerApplication.Parse(os.Args[1:]))

	go monitor()

	glib.ThreadInit(nil)
	gdk.ThreadsInit()
	gdk.ThreadsEnter()
	gtk.Init(nil)

	taskrunnerGUI := gui.NewTaskrunnerGUI(taskrunnerDAL, gui.TaskrunnerGUIOptions{*jobLogMaxLines})
	taskrunnerGUI.RenderScene(taskrunnerGUI.NewHomeScene())

	gtk.Main()
}

func setupApplicationFlags() {
	taskrunnerDir := taskrunnerApplication.
		Flag("taskrunner-dir", "Directory the taskruner uses to store job configs and logs of job runs.").
		Default("~/.local/share/github.com/jamesrr39/taskrunner-app").
		String()

	jobLogMaxLines = taskrunnerApplication.Flag("job-log-max-lines", "maximum number of lines to display in the job output. Lines after that are not shown in the UI, but the UI indicates where the whole log file is instead").
		Default("10000").
		Uint()

	taskrunnerApplication.Action(func(context *kingpin.ParseContext) error {
		expandedDir, err := user.ExpandUser(*taskrunnerDir)
		if nil != err {
			return err
		}

		taskrunnerDAL, err = taskrunnerdal.NewTaskrunnerDALAndEnsureDirectories(expandedDir)
		if nil != err {
			return err
		}

		return nil
	})
}

func monitor() {
	for {
		time.Sleep(time.Second)
		log.Printf("using %d goroutines\n", runtime.NumGoroutine())
	}
}
<commit_msg>make monitoring optional<commit_after>package main

import (
	"log"
	"os"
	"runtime"
	"taskrunner-app/taskrunner-gtk/gui"
	"taskrunner-app/taskrunnerdal"
	"time"

	"github.com/alecthomas/kingpin"
	"github.com/mattn/go-gtk/gtk"

	"github.com/mattn/go-gtk/glib"

	"github.com/jamesrr39/goutil/user"
	"github.com/mattn/go-gtk/gdk"
)

var (
	taskrunnerDAL         *taskrunnerdal.TaskrunnerDAL
	taskrunnerApplication *kingpin.Application
	jobLogMaxLines        *uint
	shouldMonitor         *bool
)

func main() {

	taskrunnerApplication = kingpin.New("Taskrunner GUI", "gtk gui for taskrunner")
	setupApplicationFlags()
	kingpin.MustParse(taskrunnerApplication.Parse(os.Args[1:]))

	if *shouldMonitor {
		go monitor()
	}

	glib.ThreadInit(nil)
	gdk.ThreadsInit()
	gdk.ThreadsEnter()
	gtk.Init(nil)

	taskrunnerGUI := gui.NewTaskrunnerGUI(taskrunnerDAL, gui.TaskrunnerGUIOptions{*jobLogMaxLines})
	taskrunnerGUI.RenderScene(taskrunnerGUI.NewHomeScene())

	gtk.Main()
}

func setupApplicationFlags() {
	taskrunnerDir := taskrunnerApplication.
		Flag("taskrunner-dir", "Directory the taskruner uses to store job configs and logs of job runs.").
		Default("~/.local/share/github.com/jamesrr39/taskrunner-app").
		String()

	shouldMonitor = taskrunnerApplication.Flag("monitor", "print information about the number of goroutines used to the log output").
		Bool()

	jobLogMaxLines = taskrunnerApplication.Flag("job-log-max-lines", "maximum number of lines to display in the job output. Lines after that are not shown in the UI, but the UI indicates where the whole log file is instead").
		Default("10000").
		Uint()

	taskrunnerApplication.Action(func(context *kingpin.ParseContext) error {
		expandedDir, err := user.ExpandUser(*taskrunnerDir)
		if nil != err {
			return err
		}

		taskrunnerDAL, err = taskrunnerdal.NewTaskrunnerDALAndEnsureDirectories(expandedDir)
		if nil != err {
			return err
		}

		return nil
	})
}

func monitor() {
	for {
		time.Sleep(time.Second)
		log.Printf("using %d goroutines (including 1 for monitoring).\n", runtime.NumGoroutine())
	}
}
<|endoftext|>


<commit_before>package routes

import (
	"../glue"
	"../scheduler"
	"bufio"
	"errors"
	"fmt"
	"github.com/DeedleFake/Go-PhysicsFS/physfs"
	"github.com/gin-gonic/contrib/gzip"
	"github.com/gin-gonic/gin"
	"github.com/pmylund/go-cache"
	"github.com/vifino/golua/lua"
	"github.com/vifino/luar"
	"net/http"
	"time"
)

// Cache
var cbc *cache.Cache
var LDumper *lua.State

func cacheDump(file string) (string, error, bool) {
	data_tmp, found := cbc.Get(file)
	if found == false {
		data, err := fileRead(file)
		if err != nil {
			return "", err, false
		}
		res, err := bcdump(data)
		if err != nil {
			return "", err, true
		}
		cbc.Set(file, res, cache.DefaultExpiration)
		return res, nil, false
	} else {
		//debug("Using Bytecode-cache for " + file)
		return data_tmp.(string), nil, false
	}
}
func bcdump(data string) (string, error) {
	if LDumper.LoadString(data) != 0 {
		return "", errors.New(LDumper.ToString(-1))
	}
	defer LDumper.Pop(1)
	return LDumper.FDump(), nil
}

// FS
var filesystem http.FileSystem

func fileRead(file string) (string, error) {
	f, err := filesystem.Open(file)
	defer f.Close()
	if err != nil {
		return "", err
	}
	fi, err := f.Stat()
	if err != nil {
		return "", err
	}
	r := bufio.NewReader(f)
	buf := make([]byte, fi.Size())
	_, err = r.Read(buf)
	if err != nil {
		return "", err
	}
	return string(buf), err
}

// Preloader/Starter
var jobs int
var Preloaded chan *lua.State

func Preloader() {
	Preloaded = make(chan *lua.State, jobs)
	for {
		//fmt.Println("preloading")
		state := luar.Init()
		err := state.DoString(glue.MainGlue())
		if err != nil {
			fmt.Println(err)
		}
		Preloaded <- state
	}
}
func GetInstance() *lua.State {
	//fmt.Println("grabbing instance")
	L := <-Preloaded
	//fmt.Println("Done")
	return L
}

// Init
func Init(j int) {
	jobs = j
	filesystem = physfs.FileSystem()
	cbc = cache.New(5*time.Minute, 30*time.Second) // Initialize cache with 5 minute lifetime and purge every 30 seconds
	LDumper = luar.Init()
}

// PHP-like lua scripts
func Lua() func(*gin.Context) {
	//LDumper := luar.Init()
	return func(context *gin.Context) {
		//fmt.Println("start")
		L := GetInstance()
		//fmt.Println("after start")
		defer scheduler.Add(func() {
			L.Close()
		})
		file := context.Request.URL.Path
		//fmt.Println("after after start")
		luar.Register(L, "", luar.Map{
			"context": context,
			"req":     context.Request,
		})
		//fmt.Println("before cache")
		code, err, lerr := cacheDump(file)
		//fmt.Println("after cache")
		if err != nil {
			if lerr == false {
				context.Next()
				return
			} else {
				context.HTMLString(http.StatusInternalServerError, `
				<title>Syntax Error in `+context.Request.URL.Path+`</title>

Syntax Error in file `+context.Request.URL.Path+`:

					`+string(err.Error())+`

				`)
				context.Abort()
				return
			}
		}
		//fmt.Println("before loadbuffer")
		L.LoadBuffer(code, len(code), file) // This shouldn't error, was checked earlier.
		if L.Pcall(0, 0, 0) != 0 {          // != 0 means error in execution
			context.HTMLString(http.StatusInternalServerError, `
			<title>Runtime Error in `+context.Request.URL.Path+`</title>

Runtime Error in file `+context.Request.URL.Path+`:

				`+L.ToString(-1)+`

			`)
			context.Abort()
			return
		}
		/*L.DoString("return CONTENT_TO_RETURN")
		v := luar.CopyTableToMap(L, nil, -1)
		m := v.(map[string]interface{})
		i := int(m["code"].(float64))
		if err != nil {
			i = http.StatusOK
		}*/
		//context.HTMLString(i, m["content"].(string))
	}
}

// Route creation by lua
func New(bcode string) (func(*gin.Context), error) {
	/*code, err := bcdump(code)
	if err != nil {
		return func(*gin.Context) {}, err
	}*/
	return func(context *gin.Context) {
		L := GetInstance()
		L.DoString(glue.RouteGlue())
		luar.Register(L, "", luar.Map{
			"context": context,
			"req":     context.Request,
		})
		luar.Register(L, "fs", luar.Map{ // PhysFS
			"mount":       physfs.Mount,
			"exits":       physfs.Exists,
			"getFS":       physfs.FileSystem,
			"mkdir":       physfs.Mkdir,
			"umount":      physfs.RemoveFromSearchPath,
			"delete":      physfs.Delete,
			"setWriteDir": physfs.SetWriteDir,
			"getWriteDir": physfs.GetWriteDir,
		})
		luar.Register(L, "mw", luar.Map{
			"Lua": Lua,
			"ExtRoute": (func(plan map[string]interface{}) func(*gin.Context) {
				newplan := make(Plan, len(plan))
				for k, v := range plan {
					newplan[k] = v.(func(*gin.Context))
				}
				return ExtRoute(newplan)
			}),
			"Logger":   gin.Logger,
			"Recovery": gin.Recovery,
			"GZip": func() func(*gin.Context) {
				return gzip.Gzip(gzip.DefaultCompression)
			},
			"New": New,
		})
		//fmt.Println("before loadbuffer")
		if L.LoadBuffer(bcode, len(bcode), "route") != 0 {
			context.HTMLString(http.StatusInternalServerError, `
			<title>Syntax Error in `+context.Request.URL.Path+`</title>

Syntax Error in Lua Route on `+context.Request.URL.Path+`:

				`+L.ToString(-1)+`

			`)
			context.Abort()
			return
		}
		if L.Pcall(0, 0, 0) != 0 { // != 0 means error in execution
			context.HTMLString(http.StatusInternalServerError, `
			<title>Runtime Error on `+context.Request.URL.Path+`</title>

Runtime Error in Lua Route on `+context.Request.URL.Path+`:

				`+L.ToString(-1)+`

			`)
			context.Abort()
			return
		}
	}, nil
}
<commit_msg>Speed improvement. (Useless bindings)<commit_after>package routes

import (
	"../glue"
	"../scheduler"
	"bufio"
	"errors"
	"fmt"
	"github.com/DeedleFake/Go-PhysicsFS/physfs"
	"github.com/gin-gonic/gin"
	"github.com/pmylund/go-cache"
	"github.com/vifino/golua/lua"
	"github.com/vifino/luar"
	"net/http"
	"time"
)

// Cache
var cbc *cache.Cache
var LDumper *lua.State

func cacheDump(file string) (string, error, bool) {
	data_tmp, found := cbc.Get(file)
	if found == false {
		data, err := fileRead(file)
		if err != nil {
			return "", err, false
		}
		res, err := bcdump(data)
		if err != nil {
			return "", err, true
		}
		cbc.Set(file, res, cache.DefaultExpiration)
		return res, nil, false
	} else {
		//debug("Using Bytecode-cache for " + file)
		return data_tmp.(string), nil, false
	}
}
func bcdump(data string) (string, error) {
	if LDumper.LoadString(data) != 0 {
		return "", errors.New(LDumper.ToString(-1))
	}
	defer LDumper.Pop(1)
	return LDumper.FDump(), nil
}

// FS
var filesystem http.FileSystem

func fileRead(file string) (string, error) {
	f, err := filesystem.Open(file)
	defer f.Close()
	if err != nil {
		return "", err
	}
	fi, err := f.Stat()
	if err != nil {
		return "", err
	}
	r := bufio.NewReader(f)
	buf := make([]byte, fi.Size())
	_, err = r.Read(buf)
	if err != nil {
		return "", err
	}
	return string(buf), err
}

// Preloader/Starter
var jobs int
var Preloaded chan *lua.State

func Preloader() {
	Preloaded = make(chan *lua.State, jobs)
	for {
		//fmt.Println("preloading")
		state := luar.Init()
		err := state.DoString(glue.MainGlue())
		if err != nil {
			fmt.Println(err)
		}
		Preloaded <- state
	}
}
func GetInstance() *lua.State {
	//fmt.Println("grabbing instance")
	L := <-Preloaded
	//fmt.Println("Done")
	return L
}

// Init
func Init(j int) {
	jobs = j
	filesystem = physfs.FileSystem()
	cbc = cache.New(5*time.Minute, 30*time.Second) // Initialize cache with 5 minute lifetime and purge every 30 seconds
	LDumper = luar.Init()
}

// PHP-like lua scripts
func Lua() func(*gin.Context) {
	//LDumper := luar.Init()
	return func(context *gin.Context) {
		//fmt.Println("start")
		L := GetInstance()
		//fmt.Println("after start")
		defer scheduler.Add(func() {
			L.Close()
		})
		file := context.Request.URL.Path
		//fmt.Println("after after start")
		luar.Register(L, "", luar.Map{
			"context": context,
			"req":     context.Request,
		})
		//fmt.Println("before cache")
		code, err, lerr := cacheDump(file)
		//fmt.Println("after cache")
		if err != nil {
			if lerr == false {
				context.Next()
				return
			} else {
				context.HTMLString(http.StatusInternalServerError, `
				<title>Syntax Error in `+context.Request.URL.Path+`</title>

Syntax Error in file `+context.Request.URL.Path+`:

					`+string(err.Error())+`

				`)
				context.Abort()
				return
			}
		}
		//fmt.Println("before loadbuffer")
		L.LoadBuffer(code, len(code), file) // This shouldn't error, was checked earlier.
		if L.Pcall(0, 0, 0) != 0 {          // != 0 means error in execution
			context.HTMLString(http.StatusInternalServerError, `
			<title>Runtime Error in `+context.Request.URL.Path+`</title>

Runtime Error in file `+context.Request.URL.Path+`:

				`+L.ToString(-1)+`

			`)
			context.Abort()
			return
		}
		/*L.DoString("return CONTENT_TO_RETURN")
		v := luar.CopyTableToMap(L, nil, -1)
		m := v.(map[string]interface{})
		i := int(m["code"].(float64))
		if err != nil {
			i = http.StatusOK
		}*/
		//context.HTMLString(i, m["content"].(string))
	}
}

// Route creation by lua
func New(bcode string) (func(*gin.Context), error) {
	/*code, err := bcdump(code)
	if err != nil {
		return func(*gin.Context) {}, err
	}*/
	return func(context *gin.Context) {
		L := GetInstance()
		L.DoString(glue.RouteGlue())
		luar.Register(L, "", luar.Map{
			"context": context,
			"req":     context.Request,
		})
		/*luar.Register(L, "fs", luar.Map{ // PhysFS
			"mount":       physfs.Mount,
			"exits":       physfs.Exists,
			"getFS":       physfs.FileSystem,
			"mkdir":       physfs.Mkdir,
			"umount":      physfs.RemoveFromSearchPath,
			"delete":      physfs.Delete,
			"setWriteDir": physfs.SetWriteDir,
			"getWriteDir": physfs.GetWriteDir,
		})
		luar.Register(L, "mw", luar.Map{
			"Lua": Lua,
			"ExtRoute": (func(plan map[string]interface{}) func(*gin.Context) {
				newplan := make(Plan, len(plan))
				for k, v := range plan {
					newplan[k] = v.(func(*gin.Context))
				}
				return ExtRoute(newplan)
			}),
			"Logger":   gin.Logger,
			"Recovery": gin.Recovery,
			"GZip": func() func(*gin.Context) {
				return gzip.Gzip(gzip.DefaultCompression)
			},
			"New": New,
		})*/
		//fmt.Println("before loadbuffer")
		if L.LoadBuffer(bcode, len(bcode), "route") != 0 {
			context.HTMLString(http.StatusInternalServerError, `
			<title>Syntax Error in `+context.Request.URL.Path+`</title>

Syntax Error in Lua Route on `+context.Request.URL.Path+`:

				`+L.ToString(-1)+`

			`)
			context.Abort()
			return
		}
		if L.Pcall(0, 0, 0) != 0 { // != 0 means error in execution
			context.HTMLString(http.StatusInternalServerError, `
			<title>Runtime Error on `+context.Request.URL.Path+`</title>

Runtime Error in Lua Route on `+context.Request.URL.Path+`:

				`+L.ToString(-1)+`

			`)
			context.Abort()
			return
		}
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package setting

import (
	"path"
	"path/filepath"
	"strings"
	"time"

	"code.gitea.io/gitea/modules/log"

	"github.com/gobwas/glob"
)

// enumerates all the indexer queue types
const (
	LevelQueueType   = "levelqueue"
	ChannelQueueType = "channel"
	RedisQueueType   = "redis"
)

var (
	// Indexer settings
	Indexer = struct {
		IssueType             string
		IssuePath             string
		IssueConnStr          string
		IssueIndexerName      string
		IssueQueueType        string
		IssueQueueDir         string
		IssueQueueConnStr     string
		IssueQueueBatchNumber int
		StartupTimeout        time.Duration

		RepoIndexerEnabled bool
		RepoType           string
		RepoPath           string
		RepoConnStr        string
		RepoIndexerName    string
		UpdateQueueLength  int
		MaxIndexerFileSize int64
		IncludePatterns    []glob.Glob
		ExcludePatterns    []glob.Glob
		ExcludeVendored    bool
	}{
		IssueType:             "bleve",
		IssuePath:             "indexers/issues.bleve",
		IssueConnStr:          "",
		IssueIndexerName:      "gitea_issues",
		IssueQueueType:        LevelQueueType,
		IssueQueueDir:         "indexers/issues.queue",
		IssueQueueConnStr:     "",
		IssueQueueBatchNumber: 20,

		RepoIndexerEnabled: false,
		RepoType:           "bleve",
		RepoPath:           "indexers/repos.bleve",
		RepoConnStr:        "",
		RepoIndexerName:    "gitea_codes",
		MaxIndexerFileSize: 1024 * 1024,
		ExcludeVendored:    true,
	}
)

func newIndexerService() {
	sec := Cfg.Section("indexer")
	Indexer.IssueType = sec.Key("ISSUE_INDEXER_TYPE").MustString("bleve")
	Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString(path.Join(AppDataPath, "indexers/issues.bleve"))
	if !filepath.IsAbs(Indexer.IssuePath) {
		Indexer.IssuePath = path.Join(AppWorkPath, Indexer.IssuePath)
	}
	Indexer.IssueConnStr = sec.Key("ISSUE_INDEXER_CONN_STR").MustString(Indexer.IssueConnStr)
	Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName)

	Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType)
	Indexer.IssueQueueDir = sec.Key("ISSUE_INDEXER_QUEUE_DIR").MustString(path.Join(AppDataPath, "indexers/issues.queue"))
	Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString(path.Join(AppDataPath, ""))
	Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20)

	Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false)
	Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve")
	Indexer.RepoPath = sec.Key("REPO_INDEXER_PATH").MustString(path.Join(AppDataPath, "indexers/repos.bleve"))
	if !filepath.IsAbs(Indexer.RepoPath) {
		Indexer.RepoPath = path.Join(AppWorkPath, Indexer.RepoPath)
	}
	Indexer.RepoConnStr = sec.Key("REPO_INDEXER_CONN_STR").MustString("")
	Indexer.RepoIndexerName = sec.Key("REPO_INDEXER_NAME").MustString("gitea_codes")

	Indexer.IncludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_INCLUDE").MustString(""))
	Indexer.ExcludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_EXCLUDE").MustString(""))
	Indexer.ExcludeVendored = sec.Key("REPO_INDEXER_EXCLUDE_VENDORED").MustBool(true)
	Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
	Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024)
	Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second)
}

// IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing
func IndexerGlobFromString(globstr string) []glob.Glob {
	extarr := make([]glob.Glob, 0, 10)
	for _, expr := range strings.Split(strings.ToLower(globstr), ",") {
		expr = strings.TrimSpace(expr)
		if expr != "" {
			if g, err := glob.Compile(expr, '.', '/'); err != nil {
				log.Info("Invalid glob expresion '%s' (skipped): %v", expr, err)
			} else {
				extarr = append(extarr, g)
			}
		}
	}
	return extarr
}
<commit_msg>Avoid setting the CONN_STR in issue indexer queue unless it is meant to be set (#13069)<commit_after>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package setting

import (
	"path"
	"path/filepath"
	"strings"
	"time"

	"code.gitea.io/gitea/modules/log"

	"github.com/gobwas/glob"
)

// enumerates all the indexer queue types
const (
	LevelQueueType   = "levelqueue"
	ChannelQueueType = "channel"
	RedisQueueType   = "redis"
)

var (
	// Indexer settings
	Indexer = struct {
		IssueType             string
		IssuePath             string
		IssueConnStr          string
		IssueIndexerName      string
		IssueQueueType        string
		IssueQueueDir         string
		IssueQueueConnStr     string
		IssueQueueBatchNumber int
		StartupTimeout        time.Duration

		RepoIndexerEnabled bool
		RepoType           string
		RepoPath           string
		RepoConnStr        string
		RepoIndexerName    string
		UpdateQueueLength  int
		MaxIndexerFileSize int64
		IncludePatterns    []glob.Glob
		ExcludePatterns    []glob.Glob
		ExcludeVendored    bool
	}{
		IssueType:             "bleve",
		IssuePath:             "indexers/issues.bleve",
		IssueConnStr:          "",
		IssueIndexerName:      "gitea_issues",
		IssueQueueType:        LevelQueueType,
		IssueQueueDir:         "indexers/issues.queue",
		IssueQueueConnStr:     "",
		IssueQueueBatchNumber: 20,

		RepoIndexerEnabled: false,
		RepoType:           "bleve",
		RepoPath:           "indexers/repos.bleve",
		RepoConnStr:        "",
		RepoIndexerName:    "gitea_codes",
		MaxIndexerFileSize: 1024 * 1024,
		ExcludeVendored:    true,
	}
)

func newIndexerService() {
	sec := Cfg.Section("indexer")
	Indexer.IssueType = sec.Key("ISSUE_INDEXER_TYPE").MustString("bleve")
	Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString(path.Join(AppDataPath, "indexers/issues.bleve"))
	if !filepath.IsAbs(Indexer.IssuePath) {
		Indexer.IssuePath = path.Join(AppWorkPath, Indexer.IssuePath)
	}
	Indexer.IssueConnStr = sec.Key("ISSUE_INDEXER_CONN_STR").MustString(Indexer.IssueConnStr)
	Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName)

	Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType)
	Indexer.IssueQueueDir = sec.Key("ISSUE_INDEXER_QUEUE_DIR").MustString(path.Join(AppDataPath, "indexers/issues.queue"))
	Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString("")
	Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20)

	Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false)
	Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve")
	Indexer.RepoPath = sec.Key("REPO_INDEXER_PATH").MustString(path.Join(AppDataPath, "indexers/repos.bleve"))
	if !filepath.IsAbs(Indexer.RepoPath) {
		Indexer.RepoPath = path.Join(AppWorkPath, Indexer.RepoPath)
	}
	Indexer.RepoConnStr = sec.Key("REPO_INDEXER_CONN_STR").MustString("")
	Indexer.RepoIndexerName = sec.Key("REPO_INDEXER_NAME").MustString("gitea_codes")

	Indexer.IncludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_INCLUDE").MustString(""))
	Indexer.ExcludePatterns = IndexerGlobFromString(sec.Key("REPO_INDEXER_EXCLUDE").MustString(""))
	Indexer.ExcludeVendored = sec.Key("REPO_INDEXER_EXCLUDE_VENDORED").MustBool(true)
	Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
	Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024)
	Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second)
}

// IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing
func IndexerGlobFromString(globstr string) []glob.Glob {
	extarr := make([]glob.Glob, 0, 10)
	for _, expr := range strings.Split(strings.ToLower(globstr), ",") {
		expr = strings.TrimSpace(expr)
		if expr != "" {
			if g, err := glob.Compile(expr, '.', '/'); err != nil {
				log.Info("Invalid glob expresion '%s' (skipped): %v", expr, err)
			} else {
				extarr = append(extarr, g)
			}
		}
	}
	return extarr
}
<|endoftext|>