<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	flags "github.com/btcsuite/go-flags"
	"github.com/roasbeef/btcutil"
)

const (
	defaultConfigFilename = "lnd.conf"
	defaultDataDirname    = "data"
	defaultLogLevel       = "info"
	defaultLogDirname     = "logs"
	defaultLogFilename    = "lnd.log"
	defaultRPCPort        = 10009
	defaultSPVMode        = false
	defaultPeerPort       = 10011
	defaultRPCHost        = "localhost:18334"
	defaultRPCUser        = "user"
	defaultRPCPass        = "passwd"
	defaultSPVHostAdr     = "localhost:18333"
)

var (
	lndHomeDir        = btcutil.AppDataDir("lnd", false)
	defaultConfigFile = filepath.Join(lndHomeDir, defaultConfigFilename)
	defaultDataDir    = filepath.Join(lndHomeDir, defaultDataDirname)
	defaultLogDir     = filepath.Join(lndHomeDir, defaultLogDirname)

	btcdHomeDir        = btcutil.AppDataDir("btcd", false)
	defaultRPCKeyFile  = filepath.Join(btcdHomeDir, "rpc.key")
	defaultRPCCertFile = filepath.Join(btcdHomeDir, "rpc.cert")
)

// config defines the configuration options for lnd.
//
// See loadConfig for further details regarding the configuration
// loading+parsing process.
type config struct {
	ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"`

	ConfigFile string `long:"C" long:"configfile" description:"Path to configuration file"`
	DataDir    string `short:"b" long:"datadir" description:"The directory to store lnd's data within"`
	LogDir     string `long:"logdir" description:"Directory to log output."`

	Listeners   []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 10011)"`
	ExternalIPs []string `long:"externalip" description:"Add an ip to the list of local addresses we claim to listen on to peers"`

	DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify <subsystem>=<level>,<subsystem2>=<level>,... to set the log level for individual subsystems -- Use show to list available subsystems"`

	Profile string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`

	PeerPort int    `long:"peerport" description:"The port to listen on for incoming p2p connections"`
	RPCPort  int    `long:"rpcport" description:"The port for the rpc server"`
	SPVMode  bool   `long:"spv" description:"assert to enter spv wallet mode"`
	RPCHost  string `long:"btcdhost" description:"The btcd rpc listening address. "`
	RPCUser  string `short:"u" long:"rpcuser" description:"Username for RPC connections"`
	RPCPass  string `short:"P" long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`

	RPCCert    string `long:"rpccert" description:"File containing btcd's certificate file"`
	RPCKey     string `long:"rpckey" description:"File containing btcd's certificate key"`
	SPVHostAdr string `long:"spvhostadr" description:"Address of full bitcoin node. It is used in SPV mode."`
	TestNet3   bool   `long:"testnet" description:"Use the test network"`
	SimNet     bool   `long:"simnet" description:"Use the simulation test network"`
	SegNet     bool   `long:"segnet" description:"Use the segragated witness test network"`
}

// loadConfig initializes and parses the config using a config file and command
// line options.
//
// The configuration proceeds as follows:
// 	1) Start with a default config with sane settings
// 	2) Pre-parse the command line to check for an alternative config file
// 	3) Load configuration file overwriting defaults with any specified options
// 	4) Parse CLI options and overwrite/add any specified options
func loadConfig() (*config, error) {
	defaultCfg := config{
		ConfigFile: defaultConfigFile,
		DataDir:    defaultDataDir,
		DebugLevel: defaultLogLevel,
		LogDir:     defaultLogDir,
		PeerPort:   defaultPeerPort,
		RPCPort:    defaultRPCPort,
		SPVMode:    defaultSPVMode,
		RPCHost:    defaultRPCHost,
		RPCUser:    defaultRPCUser,
		RPCPass:    defaultRPCPass,
		RPCCert:    defaultRPCCertFile,
		RPCKey:     defaultRPCKeyFile,
		SPVHostAdr: defaultSPVHostAdr,
	}

	// Pre-parse the command line options to pick up an alternative config
	// file.
	preCfg := defaultCfg
	if _, err := flags.Parse(&preCfg); err != nil {
		return nil, err
	}

	// Show the version and exit if the version flag was specified.
	appName := filepath.Base(os.Args[0])
	appName = strings.TrimSuffix(appName, filepath.Ext(appName))
	usageMessage := fmt.Sprintf("Use %s -h to show usage", appName)
	if preCfg.ShowVersion {
		fmt.Println(appName, "version", version())
		os.Exit(0)
	}

	// Create the home directory if it doesn't already exist.
	funcName := "loadConfig"
	if err := os.MkdirAll(lndHomeDir, 0700); err != nil {
		// Show a nicer error message if it's because a symlink is
		// linked to a directory that does not exist (probably because
		// it's not mounted).
		if e, ok := err.(*os.PathError); ok && os.IsExist(err) {
			if link, lerr := os.Readlink(e.Path); lerr == nil {
				str := "is symlink %s -> %s mounted?"
				err = fmt.Errorf(str, e.Path, link)
			}
		}

		str := "%s: Failed to create home directory: %v"
		err := fmt.Errorf(str, funcName, err)
		fmt.Fprintln(os.Stderr, err)
		return nil, err
	}

	// Next, load any additional configuration options from the file.
	cfg := defaultCfg
	if err := flags.IniParse(preCfg.ConfigFile, &cfg); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}

	// Finally, parse the remaining command line options again to ensure
	// they take precedence.
	if _, err := flags.Parse(&cfg); err != nil {
		return nil, err
	}

	// Multiple networks can't be selected simultaneously.
	// Count number of network flags passed; assign active network params
	// while we're at it
	numNets := 0
	if cfg.TestNet3 {
		numNets++
		activeNetParams = testNetParams
	}
	if cfg.SimNet {
		numNets++
		activeNetParams = simNetParams
	}
	if numNets > 1 {
		str := "%s: The testnet, segnet, and simnet params can't be " +
			"used together -- choose one of the three"
		err := fmt.Errorf(str, funcName)
		return nil, err
	}

	// Validate profile port number
	if cfg.Profile != "" {
		profilePort, err := strconv.Atoi(cfg.Profile)
		if err != nil || profilePort < 1024 || profilePort > 65535 {
			str := "%s: The profile port must be between 1024 and 65535"
			err := fmt.Errorf(str, funcName)
			fmt.Fprintln(os.Stderr, err)
			fmt.Fprintln(os.Stderr, usageMessage)
			return nil, err
		}
	}

	// Append the network type to the data directory so it is "namespaced"
	// per network. In addition to the block database, there are other
	// pieces of data that are saved to disk such as address manager state.
	// All data is specific to a network, so namespacing the data directory
	// means each individual piece of serialized data does not have to
	// worry about changing names per network and such.
	cfg.DataDir = cleanAndExpandPath(cfg.DataDir)
	cfg.DataDir = filepath.Join(cfg.DataDir, activeNetParams.Name)

	// Append the network type to the log directory so it is "namespaced"
	// per network in the same fashion as the data directory.
	cfg.LogDir = cleanAndExpandPath(cfg.LogDir)
	cfg.LogDir = filepath.Join(cfg.LogDir, activeNetParams.Name)

	// Initialize logging at the default logging level.
	initSeelogLogger(filepath.Join(cfg.LogDir, defaultLogFilename))
	setLogLevels(defaultLogLevel)

	// Parse, validate, and set debug log level(s).
	if err := parseAndSetDebugLevels(cfg.DebugLevel); err != nil {
		err := fmt.Errorf("%s: %v", funcName, err.Error())
		fmt.Fprintln(os.Stderr, err)
		fmt.Fprintln(os.Stderr, usageMessage)
		return nil, err
	}

	return &cfg, nil
}

// cleanAndExpandPath expands environment variables and leading ~ in the
// passed path, cleans the result, and returns it.
func cleanAndExpandPath(path string) string {
	// Expand initial ~ to OS specific home directory.
	if strings.HasPrefix(path, "~") {
		homeDir := filepath.Dir(lndHomeDir)
		path = strings.Replace(path, "~", homeDir, 1)
	}

	// NOTE: The os.ExpandEnv doesn't work with Windows-style %VARIABLE%,
	// but they variables can still be expanded via POSIX-style $VARIABLE.
	return filepath.Clean(os.ExpandEnv(path))
}

// parseAndSetDebugLevels attempts to parse the specified debug level and set
// the levels accordingly. An appropriate error is returned if anything is
// invalid.
func parseAndSetDebugLevels(debugLevel string) error {
	// When the specified string doesn't have any delimters, treat it as
	// the log level for all subsystems.
	if !strings.Contains(debugLevel, ",") && !strings.Contains(debugLevel, "=") {
		// Validate debug log level.
		if !validLogLevel(debugLevel) {
			str := "The specified debug level [%v] is invalid"
			return fmt.Errorf(str, debugLevel)
		}

		// Change the logging level for all subsystems.
		setLogLevels(debugLevel)

		return nil
	}

	// Split the specified string into subsystem/level pairs while detecting
	// issues and update the log levels accordingly.
	for _, logLevelPair := range strings.Split(debugLevel, ",") {
		if !strings.Contains(logLevelPair, "=") {
			str := "The specified debug level contains an invalid " +
				"subsystem/level pair [%v]"
			return fmt.Errorf(str, logLevelPair)
		}

		// Extract the specified subsystem and log level.
		fields := strings.Split(logLevelPair, "=")
		subsysID, logLevel := fields[0], fields[1]

		// Validate subsystem.
		if _, exists := subsystemLoggers[subsysID]; !exists {
			str := "The specified subsystem [%v] is invalid -- " +
				"supported subsytems %v"
			return fmt.Errorf(str, subsysID, supportedSubsystems())
		}

		// Validate log level.
		if !validLogLevel(logLevel) {
			str := "The specified debug level [%v] is invalid"
			return fmt.Errorf(str, logLevel)
		}

		setLogLevel(subsysID, logLevel)
	}

	return nil
}

// validLogLevel returns whether or not logLevel is a valid debug log level.
func validLogLevel(logLevel string) bool {
	switch logLevel {
	case "trace":
		fallthrough
	case "debug":
		fallthrough
	case "info":
		fallthrough
	case "warn":
		fallthrough
	case "error":
		fallthrough
	case "critical":
		return true
	}
	return false
}

// supportedSubsystems returns a sorted slice of the supported subsystems for
// logging purposes.
func supportedSubsystems() []string {
	// Convert the subsystemLoggers map keys to a slice.
	subsystems := make([]string, 0, len(subsystemLoggers))
	for subsysID := range subsystemLoggers {
		subsystems = append(subsystems, subsysID)
	}

	// Sort the subsystems for stable display.
	sort.Strings(subsystems)
	return subsystems
}
<commit_msg>lnd: drop the port from the default rpc host<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	flags "github.com/btcsuite/go-flags"
	"github.com/roasbeef/btcutil"
)

const (
	defaultConfigFilename = "lnd.conf"
	defaultDataDirname    = "data"
	defaultLogLevel       = "info"
	defaultLogDirname     = "logs"
	defaultLogFilename    = "lnd.log"
	defaultRPCPort        = 10009
	defaultSPVMode        = false
	defaultPeerPort       = 10011
	defaultRPCHost        = "localhost"
	defaultRPCUser        = "user"
	defaultRPCPass        = "passwd"
	defaultSPVHostAdr     = "localhost:18333"
)

var (
	lndHomeDir        = btcutil.AppDataDir("lnd", false)
	defaultConfigFile = filepath.Join(lndHomeDir, defaultConfigFilename)
	defaultDataDir    = filepath.Join(lndHomeDir, defaultDataDirname)
	defaultLogDir     = filepath.Join(lndHomeDir, defaultLogDirname)

	btcdHomeDir        = btcutil.AppDataDir("btcd", false)
	defaultRPCKeyFile  = filepath.Join(btcdHomeDir, "rpc.key")
	defaultRPCCertFile = filepath.Join(btcdHomeDir, "rpc.cert")
)

// config defines the configuration options for lnd.
//
// See loadConfig for further details regarding the configuration
// loading+parsing process.
type config struct {
	ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"`

	ConfigFile string `long:"C" long:"configfile" description:"Path to configuration file"`
	DataDir    string `short:"b" long:"datadir" description:"The directory to store lnd's data within"`
	LogDir     string `long:"logdir" description:"Directory to log output."`

	Listeners   []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 10011)"`
	ExternalIPs []string `long:"externalip" description:"Add an ip to the list of local addresses we claim to listen on to peers"`

	DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify <subsystem>=<level>,<subsystem2>=<level>,... to set the log level for individual subsystems -- Use show to list available subsystems"`

	Profile string `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`

	PeerPort int    `long:"peerport" description:"The port to listen on for incoming p2p connections"`
	RPCPort  int    `long:"rpcport" description:"The port for the rpc server"`
	SPVMode  bool   `long:"spv" description:"assert to enter spv wallet mode"`
	RPCHost  string `long:"btcdhost" description:"The btcd rpc listening address."`
	RPCUser  string `short:"u" long:"rpcuser" description:"Username for RPC connections"`
	RPCPass  string `short:"P" long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`

	RPCCert    string `long:"rpccert" description:"File containing btcd's certificate file"`
	RPCKey     string `long:"rpckey" description:"File containing btcd's certificate key"`
	SPVHostAdr string `long:"spvhostadr" description:"Address of full bitcoin node. It is used in SPV mode."`
	TestNet3   bool   `long:"testnet" description:"Use the test network"`
	SimNet     bool   `long:"simnet" description:"Use the simulation test network"`
	SegNet     bool   `long:"segnet" description:"Use the segragated witness test network"`
}

// loadConfig initializes and parses the config using a config file and command
// line options.
//
// The configuration proceeds as follows:
// 	1) Start with a default config with sane settings
// 	2) Pre-parse the command line to check for an alternative config file
// 	3) Load configuration file overwriting defaults with any specified options
// 	4) Parse CLI options and overwrite/add any specified options
func loadConfig() (*config, error) {
	defaultCfg := config{
		ConfigFile: defaultConfigFile,
		DataDir:    defaultDataDir,
		DebugLevel: defaultLogLevel,
		LogDir:     defaultLogDir,
		PeerPort:   defaultPeerPort,
		RPCPort:    defaultRPCPort,
		SPVMode:    defaultSPVMode,
		RPCHost:    defaultRPCHost,
		RPCUser:    defaultRPCUser,
		RPCPass:    defaultRPCPass,
		RPCCert:    defaultRPCCertFile,
		RPCKey:     defaultRPCKeyFile,
		SPVHostAdr: defaultSPVHostAdr,
	}

	// Pre-parse the command line options to pick up an alternative config
	// file.
	preCfg := defaultCfg
	if _, err := flags.Parse(&preCfg); err != nil {
		return nil, err
	}

	// Show the version and exit if the version flag was specified.
	appName := filepath.Base(os.Args[0])
	appName = strings.TrimSuffix(appName, filepath.Ext(appName))
	usageMessage := fmt.Sprintf("Use %s -h to show usage", appName)
	if preCfg.ShowVersion {
		fmt.Println(appName, "version", version())
		os.Exit(0)
	}

	// Create the home directory if it doesn't already exist.
	funcName := "loadConfig"
	if err := os.MkdirAll(lndHomeDir, 0700); err != nil {
		// Show a nicer error message if it's because a symlink is
		// linked to a directory that does not exist (probably because
		// it's not mounted).
		if e, ok := err.(*os.PathError); ok && os.IsExist(err) {
			if link, lerr := os.Readlink(e.Path); lerr == nil {
				str := "is symlink %s -> %s mounted?"
				err = fmt.Errorf(str, e.Path, link)
			}
		}

		str := "%s: Failed to create home directory: %v"
		err := fmt.Errorf(str, funcName, err)
		fmt.Fprintln(os.Stderr, err)
		return nil, err
	}

	// Next, load any additional configuration options from the file.
	cfg := defaultCfg
	if err := flags.IniParse(preCfg.ConfigFile, &cfg); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}

	// Finally, parse the remaining command line options again to ensure
	// they take precedence.
	if _, err := flags.Parse(&cfg); err != nil {
		return nil, err
	}

	// Multiple networks can't be selected simultaneously.
	// Count number of network flags passed; assign active network params
	// while we're at it
	numNets := 0
	if cfg.TestNet3 {
		numNets++
		activeNetParams = testNetParams
	}
	if cfg.SimNet {
		numNets++
		activeNetParams = simNetParams
	}
	if numNets > 1 {
		str := "%s: The testnet, segnet, and simnet params can't be " +
			"used together -- choose one of the three"
		err := fmt.Errorf(str, funcName)
		return nil, err
	}

	// Validate profile port number
	if cfg.Profile != "" {
		profilePort, err := strconv.Atoi(cfg.Profile)
		if err != nil || profilePort < 1024 || profilePort > 65535 {
			str := "%s: The profile port must be between 1024 and 65535"
			err := fmt.Errorf(str, funcName)
			fmt.Fprintln(os.Stderr, err)
			fmt.Fprintln(os.Stderr, usageMessage)
			return nil, err
		}
	}

	// Append the network type to the data directory so it is "namespaced"
	// per network. In addition to the block database, there are other
	// pieces of data that are saved to disk such as address manager state.
	// All data is specific to a network, so namespacing the data directory
	// means each individual piece of serialized data does not have to
	// worry about changing names per network and such.
	cfg.DataDir = cleanAndExpandPath(cfg.DataDir)
	cfg.DataDir = filepath.Join(cfg.DataDir, activeNetParams.Name)

	// Append the network type to the log directory so it is "namespaced"
	// per network in the same fashion as the data directory.
	cfg.LogDir = cleanAndExpandPath(cfg.LogDir)
	cfg.LogDir = filepath.Join(cfg.LogDir, activeNetParams.Name)

	// Initialize logging at the default logging level.
	initSeelogLogger(filepath.Join(cfg.LogDir, defaultLogFilename))
	setLogLevels(defaultLogLevel)

	// Parse, validate, and set debug log level(s).
	if err := parseAndSetDebugLevels(cfg.DebugLevel); err != nil {
		err := fmt.Errorf("%s: %v", funcName, err.Error())
		fmt.Fprintln(os.Stderr, err)
		fmt.Fprintln(os.Stderr, usageMessage)
		return nil, err
	}

	return &cfg, nil
}

// cleanAndExpandPath expands environment variables and leading ~ in the
// passed path, cleans the result, and returns it.
func cleanAndExpandPath(path string) string {
	// Expand initial ~ to OS specific home directory.
	if strings.HasPrefix(path, "~") {
		homeDir := filepath.Dir(lndHomeDir)
		path = strings.Replace(path, "~", homeDir, 1)
	}

	// NOTE: The os.ExpandEnv doesn't work with Windows-style %VARIABLE%,
	// but they variables can still be expanded via POSIX-style $VARIABLE.
	return filepath.Clean(os.ExpandEnv(path))
}

// parseAndSetDebugLevels attempts to parse the specified debug level and set
// the levels accordingly. An appropriate error is returned if anything is
// invalid.
func parseAndSetDebugLevels(debugLevel string) error {
	// When the specified string doesn't have any delimters, treat it as
	// the log level for all subsystems.
	if !strings.Contains(debugLevel, ",") && !strings.Contains(debugLevel, "=") {
		// Validate debug log level.
		if !validLogLevel(debugLevel) {
			str := "The specified debug level [%v] is invalid"
			return fmt.Errorf(str, debugLevel)
		}

		// Change the logging level for all subsystems.
		setLogLevels(debugLevel)

		return nil
	}

	// Split the specified string into subsystem/level pairs while detecting
	// issues and update the log levels accordingly.
	for _, logLevelPair := range strings.Split(debugLevel, ",") {
		if !strings.Contains(logLevelPair, "=") {
			str := "The specified debug level contains an invalid " +
				"subsystem/level pair [%v]"
			return fmt.Errorf(str, logLevelPair)
		}

		// Extract the specified subsystem and log level.
		fields := strings.Split(logLevelPair, "=")
		subsysID, logLevel := fields[0], fields[1]

		// Validate subsystem.
		if _, exists := subsystemLoggers[subsysID]; !exists {
			str := "The specified subsystem [%v] is invalid -- " +
				"supported subsytems %v"
			return fmt.Errorf(str, subsysID, supportedSubsystems())
		}

		// Validate log level.
		if !validLogLevel(logLevel) {
			str := "The specified debug level [%v] is invalid"
			return fmt.Errorf(str, logLevel)
		}

		setLogLevel(subsysID, logLevel)
	}

	return nil
}

// validLogLevel returns whether or not logLevel is a valid debug log level.
func validLogLevel(logLevel string) bool {
	switch logLevel {
	case "trace":
		fallthrough
	case "debug":
		fallthrough
	case "info":
		fallthrough
	case "warn":
		fallthrough
	case "error":
		fallthrough
	case "critical":
		return true
	}
	return false
}

// supportedSubsystems returns a sorted slice of the supported subsystems for
// logging purposes.
func supportedSubsystems() []string {
	// Convert the subsystemLoggers map keys to a slice.
	subsystems := make([]string, 0, len(subsystemLoggers))
	for subsysID := range subsystemLoggers {
		subsystems = append(subsystems, subsysID)
	}

	// Sort the subsystems for stable display.
	sort.Strings(subsystems)
	return subsystems
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"path/filepath"
)

// Config interacts with the config.json
type Config struct {
	SkipAnalytics *bool `json:"skip_analytics"`
	Color         *bool `json:"color"`
}

var config *Config

func configPath() string {
	return filepath.Join(ConfigHome, "config.json")
}

func init() {
	err := readJSON(&config, configPath())
	if config == nil {
		config = &Config{}
	}
	if err != nil && !os.IsNotExist(err) {
		WarnIfError(err)
	}
}
<commit_msg>write out config.json<commit_after>package main

import (
	"os"
	"path/filepath"
)

// Config interacts with the config.json
type Config struct {
	SkipAnalytics *bool `json:"skip_analytics"`
	Color         *bool `json:"color"`
}

var config *Config

func configPath() string {
	return filepath.Join(ConfigHome, "config.json")
}

func init() {
	err := readJSON(&config, configPath())
	if config == nil {
		config = &Config{
			SkipAnalytics: pbool(false),
			Color:         pbool(true),
		}
	}
	if err != nil {
		if os.IsNotExist(err) {
			saveJSON(&config, configPath())
		} else {
			WarnIfError(err)
		}
	}
}

func pbool(b bool) *bool {
	a := b
	return &a
}
<|endoftext|>


<commit_before>// Copyright 2012 GAEGo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package config provides persisted configuration maps that can be set
and retrieved using a unique key.

Each Config is stored in memory, memcache, and the datastore. This allows
Config settings to be shared between instances, while at the same time
elimating the cost associated with retrieving directly form the datastore.

Example Usage:

Set the config

	c := appengine.Context(r)
	appMap := map[string]string{
		"Title": "Storeski"
	}
	appConfig, err := config.GetOrInsert(c, "app", appMap)

Get the config in another package

	appConfig, err := config.Get(c, "app")
	appTitle := appConfig.Values"Title"] // "Storeski"

Edit and save the config

	appConfig.Values["Title"] = "Changed"
  err := appConfig.Put(c)
	appConfig2, err := config.Get(c, "app")
  appTitle := appConfig2.Values["Title"] // "Changed"

*/
package config

// TODO(kylefinley) move the JSON encoding of invalid datastore types
// to ds.

import (
	"appengine"
	"appengine/datastore"
	"encoding/json"
	"github.com/gaego/ds"
)

func init() {
	// Store Config entities in datastore, memcache and memory.
	ds.Register("Config", true, true, true)
}

// Config is the struct that is stored. The map[string]string{}
// is gob encoded before being passed to ds.
type Config struct {
	Key        *datastore.Key `datastore:"-"`
	ValuesJSON []byte
	Values     map[string]string `datastore:"-"`
}

// SetValues sets the map to the Values property and sets ValuesJSON
// to a JSON encoded []byte
func (cnfg *Config) SetValues(m map[string]string) (err error) {

	val, err := json.Marshal(m)
	cnfg.Values = m
	cnfg.ValuesJSON = val
	return
}

// SetKey set the Key property to a datastore.Key using the passed key
// as the StringID
func (cnfg *Config) SetKey(c appengine.Context, key string) {
	cnfg.Key = datastore.NewKey(c, "Config", key, 0, nil)
	return
}

// Put encodes the Values and saves the Config to the store.
func (cnfg *Config) Put(c appengine.Context) (err error) {

	val, err := json.Marshal(cnfg.Values)
	if err != nil {
		return
	}
	cnfg.ValuesJSON = val

	key, err := ds.Put(c, cnfg.Key, cnfg)
	cnfg.Key = key
	return
}

// Get retieves a Config using the string key. If Config is not found
// it returns ds/error.ErrNoSuchEntity.
func Get(c appengine.Context, key string) (cnfg *Config, err error) {

	cnfg = new(Config)
	cnfg.SetKey(c, key)
	err = ds.Get(c, cnfg.Key, cnfg)
	if err != nil {
		return
	}

	m := make(map[string]string)
	err = json.Unmarshal(cnfg.ValuesJSON, &m)
	cnfg.Values = m

	return
}

// GetOrInsert takes takes a key and map. If the key belongs to a previously
// saved Config it is returned; Otherwise the map is saved.
func GetOrInsert(c appengine.Context, key string,
	m map[string]string) (cnfg *Config, err error) {

	cnfg, err = Get(c, key)
	if err == nil {
		return
	}
	cnfg = new(Config)
	err = cnfg.SetValues(m)
	if err != nil {
		return
	}
	cnfg.SetKey(c, key)
	cnfg.Put(c)
	return
}
<commit_msg>Converted storage from JSON to Gob. Added Decode() Encode() methods Removed SetValues() method -- to make api more consistent with existing code.<commit_after>// Copyright 2012 GAEGo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package config provides persisted configuration maps that can be set
and retrieved using a unique key.

Each Config is stored in memory, memcache, and the datastore. This allows
Config settings to be shared between instances, while at the same time
elimating the cost associated with retrieving directly form the datastore.

Example Usage:

Set the config

	c := appengine.Context(r)
	appMap := map[string]string{
		"Title": "Storeski"
	}
	appConfig, err := config.GetOrInsert(c, "app", appMap)

Get the config in another package

	appConfig, err := config.Get(c, "app")
	appTitle := appConfig.Values"Title"] // "Storeski"

Edit and save the config

	appConfig.Values["Title"] = "Changed"
  err := appConfig.Put(c)
	appConfig2, err := config.Get(c, "app")
  appTitle := appConfig2.Values["Title"] // "Changed"

*/
package config

// TODO(kylefinley) move the JSON encoding of invalid datastore types
// to ds.

import (
	"appengine"
	"appengine/datastore"
	"bytes"
	"encoding/gob"
	"github.com/gaego/ds"
)

func init() {
	// Store Config entities in datastore, memcache and memory.
	ds.Register("Config", true, true, true)
}

// Config is the struct that is stored. The map[string]string{}
// is gob encoded before being passed to ds.
type Config struct {
	Key       *datastore.Key `datastore:"-"`
	ValuesGob []byte
	Values    map[string]string `datastore:"-"`
}

// SetKey set the Key property to a datastore.Key using the passed key
// as the StringID
func (cnfg *Config) SetKey(c appengine.Context, key string) {
	cnfg.Key = datastore.NewKey(c, "Config", key, 0, nil)
	return
}

// Encode is called prior to save. Any fields that need to be updated
// prior to save are updated here.
func (cnfg *Config) Encode() error {
	var b bytes.Buffer
	enc := gob.NewEncoder(&b)
	err := enc.Encode(cnfg.Values)
	cnfg.ValuesGob = b.Bytes()
	return err
}

// Decode is called after the entity has been retrieved from the the ds.
func (cnfg *Config) Decode() error {
	b := bytes.NewBuffer(cnfg.ValuesGob)
	dec := gob.NewDecoder(b)
	err := dec.Decode(&cnfg.Values)
	return err
}

// Put encodes the Values and saves the Config to the store.
func (cnfg *Config) Put(c appengine.Context) (err error) {
	if err = cnfg.Encode(); err != nil {
		return
	}
	key, err := ds.Put(c, cnfg.Key, cnfg)
	cnfg.Key = key
	return
}

// Get retieves a Config using the string key. If Config is not found
// it returns ds/error.ErrNoSuchEntity.
func Get(c appengine.Context, key string) (cnfg *Config, err error) {

	cnfg = new(Config)
	cnfg.SetKey(c, key)
	err = ds.Get(c, cnfg.Key, cnfg)
	if err != nil {
		return
	}
	err = cnfg.Decode()
	return
}

// GetOrInsert takes takes a key and map. If the key belongs to a previously
// saved Config it is returned; Otherwise the map is saved.
func GetOrInsert(c appengine.Context, key string,
	m map[string]string) (cnfg *Config, err error) {

	cnfg, err = Get(c, key)
	if err == nil {
		return
	}
	cnfg = new(Config)
	cnfg.Values = m
	cnfg.SetKey(c, key)
	err = cnfg.Put(c)
	return
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path"
)

type Config struct {
	Nick      string
	Host      string
	RealName  string
	User      string
	Networks  []string
	Servers   map[string][]string
	Channels  map[string][]string
	Passwords map[string]string
	Plugins   []string
	Ignore    []string
	Logpath   string
	ConfigDir string
}

type Context struct {
	Network string
	Target  string
	Source  string
}

var C Config

func lookupvar(key, path string) interface{} {
	var f interface{}
	_, err := os.Stat(path)
	if err != nil {
		return ""
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return ""
	}

	err = json.Unmarshal(data, &f)

	return f.(map[string]interface{})[key]
}

func (c *Config) Lookup(context Context, key string) interface{} {
	var fpath string
	var value interface{}
	if context.Network != "" {
		if context.Source != "" {
			if context.Target != "" {
				fpath = path.Join(c.ConfigDir, context.Network, context.Source, context.Target+".json")

				log.Println("Context:", context, "Looking up", key, "in", fpath)
				value = lookupvar(key, fpath)
				if value != nil {
					return value
				}
			}
			return lookupvar(key, path.Join(c.ConfigDir, context.Network, context.Source+".json"))
		}
		return lookupvar(key, path.Join(c.ConfigDir, context.Network+".json"))
	}
	return lookupvar(key, path.Join(c.ConfigDir, "common.json"))
}
<commit_msg>It /mostly/ works now.<commit_after>package config

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path"
)

type Config struct {
	Nick      string
	Host      string
	RealName  string
	User      string
	Networks  []string
	Servers   map[string][]string
	Channels  map[string][]string
	Passwords map[string]string
	Plugins   []string
	Ignore    []string
	Logpath   string
	ConfigDir string
}

type Context struct {
	Network string
	Target  string
	Source  string
}

var C Config

func lookupvar(key, path string) interface{} {
	var f interface{}
	_, err := os.Stat(path)
	if err != nil {
		return nil
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil
	}

	err = json.Unmarshal(data, &f)

	return f.(map[string]interface{})[key]
}

func (c *Config) Lookup(context Context, key string) interface{} {
	var fpath string
	var value interface{}
	if context.Network != "" {
		if context.Source != "" {
			if context.Target != "" {
				fpath = path.Join(c.ConfigDir, context.Network, context.Source, context.Target+".json")

				log.Println("Context:", context, "Looking up", key, "in", fpath)
				value = lookupvar(key, fpath)
				if value != nil {
                    log.Println("Context:", context, "Found", key, value)
					return value
				}
			}

            fpath = path.Join(c.ConfigDir, context.Network, context.Source+".json")

            log.Println("Context:", context, "Looking up", key, "in", fpath)
            value = lookupvar(key, fpath)
            if value != nil {
                log.Println("Context:", context, "Found", key, value)
                return value
            }
		}

        fpath = path.Join(c.ConfigDir, context.Network+".json")

        log.Println("Context:", context, "Looking up", key, "in", fpath)
        value = lookupvar(key, fpath)
        if value != nil {
            log.Println("Context:", context, "Found", key, value)
            return value
        }
	}

    fpath = path.Join(c.ConfigDir, "common.json")

    log.Println("Context:", context, "Looking up", key, "in", fpath)
    value = lookupvar(key, fpath)
    if value != nil {
        log.Println("Context:", context, "Found", key, value)
        return value
    } else {
        log.Println("Key", key, "not found")
        return nil
    }
}
<|endoftext|>


<commit_before>package ParquetReader

import (
	"encoding/binary"
	"reflect"
	"sync"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/Marshal"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
)

type ParquetReader struct {
	SchemaHandler *SchemaHandler.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         ParquetFile.ParquetFile

	ColumnBuffers map[string]*ColumnBufferType
}

//Create a parquet reader
func NewParquetReader(pFile ParquetFile.ParquetFile, obj interface{}, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	if err = res.ReadFooter(); err != nil {
		return nil, err
	}
	res.ColumnBuffers = make(map[string]*ColumnBufferType)

	if obj != nil {
		if res.SchemaHandler, err = SchemaHandler.NewSchemaHandlerFromStruct(obj); err != nil {
			return res, err
		}
		res.RenameSchema()

		for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
			schema := res.SchemaHandler.SchemaElements[i]
			if schema.GetNumChildren() == 0 {
				pathStr := res.SchemaHandler.IndexMap[int32(i)]
				if res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr); err != nil {
					return res, err
				}
			}
		}
	}
	return res, nil
}

func (self *ParquetReader) SetSchemaHandlerFromJSON(jsonSchema string) error {
	var err error
	if self.SchemaHandler, err = SchemaHandler.NewSchemaHandlerFromJSON(jsonSchema); err != nil {
		return err
	}
	self.RenameSchema()
	for i := 0; i < len(self.SchemaHandler.SchemaElements); i++ {
		schema := self.SchemaHandler.SchemaElements[i]
		if schema.GetNumChildren() == 0 {
			pathStr := self.SchemaHandler.IndexMap[int32(i)]
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}
	return nil
}

//Rename schema name to inname
func (self *ParquetReader) RenameSchema() {
	for i := 0; i < len(self.Footer.Schema); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i].InName
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			exPath := make([]string, 0)
			exPath = append(exPath, self.SchemaHandler.GetRootName())
			exPath = append(exPath, chunk.MetaData.GetPathInSchema()...)
			exPathStr := Common.PathToStr(exPath)

			inPathStr := self.SchemaHandler.ExPathToInPath[exPathStr]
			inPath := Common.StrToPath(inPathStr)[1:]
			chunk.MetaData.PathInSchema = inPath
		}
	}
}

func (self *ParquetReader) GetNumRows() int64 {
	return self.Footer.GetNumRows()
}

//Get the footer size
func (self *ParquetReader) GetFooterSize() (uint32, error) {
	var err error
	buf := make([]byte, 4)
	if _, err = self.PFile.Seek(-8, 2); err != nil {
		return 0, err
	}
	if _, err = self.PFile.Read(buf); err != nil {
		return 0, err
	}
	size := binary.LittleEndian.Uint32(buf)
	return size, err
}

//Read footer from parquet file
func (self *ParquetReader) ReadFooter() error {
	size, err := self.GetFooterSize()
	if err != nil {
		return err
	}
	if _, err = self.PFile.Seek(-(int64)(8+size), 2); err != nil {
		return err
	}
	self.Footer = parquet.NewFileMetaData()
	pf := thrift.NewTCompactProtocolFactory()
	protocol := pf.GetProtocol(thrift.NewStreamTransportR(self.PFile))
	return self.Footer.Read(protocol)
}

//Skip rows of parquet file
func (self *ParquetReader) SkipRows(num int64) error {
	var err error
	if num <= 0 {
		return nil
	}
	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.SchemaHandler.ValueColumns))
	stopChan := make(chan int)

	for _, pathStr := range self.SchemaHandler.ValueColumns {
		if _, ok := self.ColumnBuffers[pathStr]; !ok {
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					cb.SkipRows(int64(num))
					doneChan <- 0
				}
			}
		}()
	}

	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}

	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}
	return err
}

//Read rows of parquet file
func (self *ParquetReader) Read(dstInterface interface{}) error {
	var err error
	tmap := make(map[string]*Layout.Table)
	locker := new(sync.Mutex)
	ot := reflect.TypeOf(dstInterface).Elem().Elem()
	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return nil
	}

	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.ColumnBuffers))
	stopChan := make(chan int)

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					table, _ := cb.ReadRows(int64(num))
					locker.Lock()
					if _, ok := tmap[pathStr]; ok {
						tmap[pathStr].Merge(table)
					} else {
						tmap[pathStr] = Layout.NewTableFromTable(table)
						tmap[pathStr].Merge(table)
					}
					locker.Unlock()
					doneChan <- 0
				}
			}
		}()
	}
	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}
	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}

	dstList := make([]interface{}, self.NP)
	delta := (int64(num) + self.NP - 1) / self.NP

	doneChan = make(chan int)
	for c := int64(0); c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > int64(num) {
			end = int64(num)
		}
		if bgn >= int64(num) {
			bgn, end = int64(num), int64(num)
		}
		go func(b, e, index int) {
			dstList[index] = reflect.New(reflect.SliceOf(ot)).Interface()
			if err2 := Marshal.Unmarshal(&tmap, b, e, dstList[index], self.SchemaHandler); err2 != nil {
				err = err2
			}
			doneChan <- 0
		}(int(bgn), int(end), int(c))
	}
	for c := int64(0); c < self.NP; c++ {
		<-doneChan
	}

	resTmp := reflect.MakeSlice(reflect.SliceOf(ot), 0, num)
	for _, dst := range dstList {
		resTmp = reflect.AppendSlice(resTmp, reflect.ValueOf(dst).Elem())
	}
	reflect.ValueOf(dstInterface).Elem().Set(resTmp)
	return err
}

//Stop Read
func (self *ParquetReader) ReadStop() {
	for _, cb := range self.ColumnBuffers {
		if cb != nil {
			cb.PFile.Close()
		}
	}
}
<commit_msg>ading<commit_after>package ParquetReader

import (
	"encoding/binary"
	"reflect"
	"sync"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/Marshal"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
)

type ParquetReader struct {
	SchemaHandler *SchemaHandler.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         ParquetFile.ParquetFile

	ColumnBuffers map[string]*ColumnBufferType
}

//Create a parquet reader
func NewParquetReader(pFile ParquetFile.ParquetFile, obj interface{}, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	if err = res.ReadFooter(); err != nil {
		return nil, err
	}
	res.ColumnBuffers = make(map[string]*ColumnBufferType)

	if obj != nil {
		if res.SchemaHandler, err = SchemaHandler.NewSchemaHandlerFromStruct(obj); err != nil {
			return res, err
		}
		res.RenameSchema()

		for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
			schema := res.SchemaHandler.SchemaElements[i]
			if schema.GetNumChildren() == 0 {
				pathStr := res.SchemaHandler.IndexMap[int32(i)]
				if res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr); err != nil {
					return res, err
				}
			}
		}
	}
	return res, nil
}

func (self *ParquetReader) SetSchemaHandlerFromJSON(jsonSchema string) error {
	var err error
	if self.SchemaHandler, err = SchemaHandler.NewSchemaHandlerFromJSON(jsonSchema); err != nil {
		return err
	}
	self.RenameSchema()
	for i := 0; i < len(self.SchemaHandler.SchemaElements); i++ {
		schema := self.SchemaHandler.SchemaElements[i]
		if schema.GetNumChildren() == 0 {
			pathStr := self.SchemaHandler.IndexMap[int32(i)]
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}
	return nil
}

//Rename schema name to inname
func (self *ParquetReader) RenameSchema() {
	for i := 0; i < len(self.SchemaHandler.Infos); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i].InName
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			exPath := make([]string, 0)
			exPath = append(exPath, self.SchemaHandler.GetRootName())
			exPath = append(exPath, chunk.MetaData.GetPathInSchema()...)
			exPathStr := Common.PathToStr(exPath)

			inPathStr := self.SchemaHandler.ExPathToInPath[exPathStr]
			inPath := Common.StrToPath(inPathStr)[1:]
			chunk.MetaData.PathInSchema = inPath
		}
	}
}

func (self *ParquetReader) GetNumRows() int64 {
	return self.Footer.GetNumRows()
}

//Get the footer size
func (self *ParquetReader) GetFooterSize() (uint32, error) {
	var err error
	buf := make([]byte, 4)
	if _, err = self.PFile.Seek(-8, 2); err != nil {
		return 0, err
	}
	if _, err = self.PFile.Read(buf); err != nil {
		return 0, err
	}
	size := binary.LittleEndian.Uint32(buf)
	return size, err
}

//Read footer from parquet file
func (self *ParquetReader) ReadFooter() error {
	size, err := self.GetFooterSize()
	if err != nil {
		return err
	}
	if _, err = self.PFile.Seek(-(int64)(8+size), 2); err != nil {
		return err
	}
	self.Footer = parquet.NewFileMetaData()
	pf := thrift.NewTCompactProtocolFactory()
	protocol := pf.GetProtocol(thrift.NewStreamTransportR(self.PFile))
	return self.Footer.Read(protocol)
}

//Skip rows of parquet file
func (self *ParquetReader) SkipRows(num int64) error {
	var err error
	if num <= 0 {
		return nil
	}
	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.SchemaHandler.ValueColumns))
	stopChan := make(chan int)

	for _, pathStr := range self.SchemaHandler.ValueColumns {
		if _, ok := self.ColumnBuffers[pathStr]; !ok {
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					cb.SkipRows(int64(num))
					doneChan <- 0
				}
			}
		}()
	}

	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}

	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}
	return err
}

//Read rows of parquet file
func (self *ParquetReader) Read(dstInterface interface{}) error {
	var err error
	tmap := make(map[string]*Layout.Table)
	locker := new(sync.Mutex)
	ot := reflect.TypeOf(dstInterface).Elem().Elem()
	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return nil
	}

	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.ColumnBuffers))
	stopChan := make(chan int)

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					table, _ := cb.ReadRows(int64(num))
					locker.Lock()
					if _, ok := tmap[pathStr]; ok {
						tmap[pathStr].Merge(table)
					} else {
						tmap[pathStr] = Layout.NewTableFromTable(table)
						tmap[pathStr].Merge(table)
					}
					locker.Unlock()
					doneChan <- 0
				}
			}
		}()
	}
	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}
	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}

	dstList := make([]interface{}, self.NP)
	delta := (int64(num) + self.NP - 1) / self.NP

	doneChan = make(chan int)
	for c := int64(0); c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > int64(num) {
			end = int64(num)
		}
		if bgn >= int64(num) {
			bgn, end = int64(num), int64(num)
		}
		go func(b, e, index int) {
			dstList[index] = reflect.New(reflect.SliceOf(ot)).Interface()
			if err2 := Marshal.Unmarshal(&tmap, b, e, dstList[index], self.SchemaHandler); err2 != nil {
				err = err2
			}
			doneChan <- 0
		}(int(bgn), int(end), int(c))
	}
	for c := int64(0); c < self.NP; c++ {
		<-doneChan
	}

	resTmp := reflect.MakeSlice(reflect.SliceOf(ot), 0, num)
	for _, dst := range dstList {
		resTmp = reflect.AppendSlice(resTmp, reflect.ValueOf(dst).Elem())
	}
	reflect.ValueOf(dstInterface).Elem().Set(resTmp)
	return err
}

//Stop Read
func (self *ParquetReader) ReadStop() {
	for _, cb := range self.ColumnBuffers {
		if cb != nil {
			cb.PFile.Close()
		}
	}
}
<|endoftext|>


<commit_before>package qingcloud

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	qc "github.com/yunify/qingcloud-sdk-go/service"
)

func modifyInstanceAttributes(d *schema.ResourceData, meta interface{}, create bool) error {
	clt := meta.(*QingCloudClient).instance
	input := new(qc.ModifyInstanceAttributesInput)
	input.Instance = qc.String(d.Id())
	if create {
		if description := d.Get("description").(string); description == "" {
			return nil
		}
		input.Description = qc.String(d.Get("description").(string))
	} else {
		if !d.HasChange("description") && !d.HasChange("name") {
			return nil
		}
		if d.HasChange("description") {
			input.Description = qc.String(d.Get("description").(string))
		}
		if d.HasChange("name") {
			input.InstanceName = qc.String(d.Get("name").(string))
		}
	}
	_, err := clt.ModifyInstanceAttributes(input)
	if err != nil {
		return fmt.Errorf("Error modify instance attributes: %s", err)
	}
	return nil
}

func instanceUpdateChangeVxNet(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("vxnet_id") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	vxnetClt := meta.(*QingCloudClient).vxnet
	oldV, newV := d.GetChange("vxnet_id")
	// leave old vxnet
	if oldV.(string) != "" {
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
		leaveVxnetInput := new(qc.LeaveVxNetInput)
		leaveVxnetInput.Instances = []*string{qc.String(d.Id())}
		leaveVxnetInput.VxNet = qc.String(oldV.(string))

		_, err := vxnetClt.LeaveVxNet(leaveVxnetInput)
		if err != nil {
			return fmt.Errorf("Error leave vxnet: %s", err)
		}
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
	}
	// join new vxnet
	if newV.(string) != "" {
		joinVxnetInput := new(qc.JoinVxNetInput)
		joinVxnetInput.Instances = []*string{qc.String(d.Id())}
		joinVxnetInput.VxNet = qc.String(newV.(string))
		_, err := vxnetClt.JoinVxNet(joinVxnetInput)
		if err != nil {
			return fmt.Errorf("Error leave vxnet: %s", err)
		}
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
	}
	return nil
}

func instanceUpdateChangeSecurityGroup(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("security_group_id") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	sgClt := meta.(*QingCloudClient).securitygroup
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	input := new(qc.ApplySecurityGroupInput)
	input.SecurityGroup = qc.String(d.Get("security_group_id").(string))
	input.Instances = []*string{qc.String(d.Id())}
	_, err := sgClt.ApplySecurityGroup(input)
	if err != nil {
		return fmt.Errorf("Error apply security group: %s", err)
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	return nil
}

func instanceUpdateChangeEip(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("eip_id") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	eipClt := meta.(*QingCloudClient).eip
	describeEIPInput := new(qc.DescribeEIPsInput)
	describeEIPInput.EIPs = []*string{qc.String(d.Get("eip_id").(string))}
	describeEIPOutput, err := eipClt.DescribeEIPs(describeEIPInput)
	if err != nil {
		return fmt.Errorf("Error describe eip: %s", err)
	}
	if qc.StringValue(describeEIPOutput.EIPSet[0].Status) != "available" {
		return fmt.Errorf("Error eip %s state is %s", d.Get("eip_id").(string), qc.StringValue(describeEIPOutput.EIPSet[0].Status))
	}
	if _, err := EIPTransitionStateRefresh(eipClt, d.Get("eip_id").(string)); err != nil {
		return err
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	oldV, newV := d.GetChange("eip_id")
	// dissociate old eip
	if oldV.(string) != "" {
		dissociateEIPInput := new(qc.DissociateEIPsInput)
		dissociateEIPInput.EIPs = []*string{qc.String(oldV.(string))}
		_, err := eipClt.DissociateEIPs(dissociateEIPInput)
		if err != nil {
			return fmt.Errorf("Error dissociate eip: %s", err)
		}
	}

	if _, err := EIPTransitionStateRefresh(eipClt, d.Get("eip_id").(string)); err != nil {
		return err
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	// associate new eip
	if newV.(string) != "" {
		assoicateEIPInput := new(qc.AssociateEIPInput)
		assoicateEIPInput.EIP = qc.String(newV.(string))
		assoicateEIPInput.Instance = qc.String(d.Id())
		_, err := eipClt.AssociateEIP(assoicateEIPInput)
		if err != nil {
			return fmt.Errorf("Error assoicate eip: %s", err)
		}
	}
	if _, err := EIPTransitionStateRefresh(eipClt, d.Get("eip_id").(string)); err != nil {
		return err
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	return nil
}

func instanceUpdateChangeKeyPairs(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("keypair_ids") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	kpClt := meta.(*QingCloudClient).keypair

	oldV, newV := d.GetChange("keypair_ids")
	var nkps []string
	var okps []string
	for _, v := range oldV.(*schema.Set).List() {
		okps = append(okps, v.(string))
	}
	for _, v := range newV.(*schema.Set).List() {
		nkps = append(nkps, v.(string))
	}
	additions, deletions := stringSliceDiff(nkps, okps)
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	// attach new key_pair
	if len(additions) > 0 {
		attachInput := new(qc.AttachKeyPairsInput)
		attachInput.Instances = []*string{qc.String(d.Id())}
		attachInput.KeyPairs = qc.StringSlice(additions)
		output, err := kpClt.AttachKeyPairs(attachInput)
		if err != nil {
			return fmt.Errorf("Error attach keypairs: %s", err)
		} else if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
			return fmt.Errorf("Error attach keypair: %s", *output.Message)
		}
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	// dettach old key_pair
	if len(deletions) > 0 {
		detachInput := new(qc.DetachKeyPairsInput)
		detachInput.Instances = []*string{qc.String(d.Id())}
		detachInput.KeyPairs = qc.StringSlice(deletions)
		output, err := kpClt.DetachKeyPairs(detachInput)
		if err != nil {
			return fmt.Errorf("Errorr detach keypairs: %s", err)
		} else if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
			return fmt.Errorf("Error detach keypair: %s", *output.Message)
		}
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
	}
	return nil
}

func instanceUpdateResize(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("cpu") && !d.HasChange("memory") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	// check instance state
	describeInstanceOutput, err := describeInstance(d, meta)
	if err != nil {
		return err
	}
	instance := describeInstanceOutput.InstanceSet[0]
	// stop instance
	if *instance.Status == "running" {
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			log.Printf("InstanceTransitionStateRefresh err: %s", err)
			return err
		}
		_, err := stopInstance(d, meta)
		if err != nil {
			return err
		}
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
	}
	//  resize instance
	input := new(qc.ResizeInstancesInput)
	input.Instances = []*string{qc.String(d.Id())}
	input.CPU = qc.Int(d.Get("cpu").(int))
	input.Memory = qc.Int(d.Get("memory").(int))
	_, err = clt.ResizeInstances(input)
	if err != nil {
		return fmt.Errorf("Error resize instance: %s", err)
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	// start instance
	_, err = startInstance(d, meta)
	if err != nil {
		return err
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	return nil
}

func describeInstance(d *schema.ResourceData, meta interface{}) (*qc.DescribeInstancesOutput, error) {
	clt := meta.(*QingCloudClient).instance
	input := new(qc.DescribeInstancesInput)
	input.Instances = []*string{qc.String(d.Id())}
	input.Verbose = qc.Int(1)
	output, err := clt.DescribeInstances(input)
	if err != nil {
		return nil, fmt.Errorf("Error describe instance: %s", err)
	}
	return output, nil
}

func stopInstance(d *schema.ResourceData, meta interface{}) (*qc.StopInstancesOutput, error) {
	clt := meta.(*QingCloudClient).instance
	input := new(qc.StopInstancesInput)
	input.Instances = []*string{qc.String(d.Id())}
	output, err := clt.StopInstances(input)
	if err != nil {
		return nil, fmt.Errorf("Error stop instance: %s", err)
	}
	return output, nil
}

func startInstance(d *schema.ResourceData, meta interface{}) (*qc.StartInstancesOutput, error) {
	clt := meta.(*QingCloudClient).instance
	input := new(qc.StartInstancesInput)
	input.Instances = []*string{qc.String(d.Id())}
	output, err := clt.StartInstances(input)
	if err != nil {
		return nil, fmt.Errorf("Error start instance: %s", err)
	}
	return output, nil
}

func deleteInstanceLeaveVxnet(d *schema.ResourceData, meta interface{}) (*qc.LeaveVxNetOutput, error) {
	vxnetID := d.Get("vxnet_id").(string)
	if vxnetID != "" {
		clt := meta.(*QingCloudClient).vxnet
		input := new(qc.LeaveVxNetInput)
		input.Instances = []*string{qc.String(d.Id())}
		input.VxNet = qc.String(vxnetID)
		_, err := clt.LeaveVxNet(input)
		if err != nil {
			return nil, fmt.Errorf("Error instance leave vxnet: %s", err)
		}
	}
	return nil, nil
}

func deleteInstanceDissociateEip(d *schema.ResourceData, meta interface{}) (*qc.LeaveVxNetOutput, error) {
	eipID := d.Get("eip_id").(string)
	if eipID != "" {
		clt := meta.(*QingCloudClient).eip
		if _, err := EIPTransitionStateRefresh(clt, eipID); err != nil {
			return nil, err
		}
		input := new(qc.DissociateEIPsInput)
		input.EIPs = []*string{qc.String(eipID)}
		_, err := clt.DissociateEIPs(input)
		if err != nil {
			return nil, fmt.Errorf("Error dissciate eip: %s", err)
		}
		if _, err := EIPTransitionStateRefresh(clt, eipID); err != nil {
			return nil, err
		}
	}
	return nil, nil
}

func updateInstanceVolume(d *schema.ResourceData, meta interface{}, create bool) error {
	// clt := meta.(*QingCloudClient).instance
	return nil
}
<commit_msg>Follow the go language error handling habit<commit_after>package qingcloud

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	qc "github.com/yunify/qingcloud-sdk-go/service"
)

func modifyInstanceAttributes(d *schema.ResourceData, meta interface{}, create bool) error {
	clt := meta.(*QingCloudClient).instance
	input := new(qc.ModifyInstanceAttributesInput)
	input.Instance = qc.String(d.Id())
	if create {
		if description := d.Get("description").(string); description == "" {
			return nil
		}
		input.Description = qc.String(d.Get("description").(string))
	} else {
		if !d.HasChange("description") && !d.HasChange("name") {
			return nil
		}
		if d.HasChange("description") {
			input.Description = qc.String(d.Get("description").(string))
		}
		if d.HasChange("name") {
			input.InstanceName = qc.String(d.Get("name").(string))
		}
	}
	_, err := clt.ModifyInstanceAttributes(input)
	if err != nil {
		return fmt.Errorf("Error modify instance attributes: %s", err)
	}
	return nil
}

func instanceUpdateChangeVxNet(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("vxnet_id") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	vxnetClt := meta.(*QingCloudClient).vxnet
	oldV, newV := d.GetChange("vxnet_id")
	// leave old vxnet
	if oldV.(string) != "" {
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
		leaveVxnetInput := new(qc.LeaveVxNetInput)
		leaveVxnetInput.Instances = []*string{qc.String(d.Id())}
		leaveVxnetInput.VxNet = qc.String(oldV.(string))

		_, err := vxnetClt.LeaveVxNet(leaveVxnetInput)
		if err != nil {
			return fmt.Errorf("Error leave vxnet: %s", err)
		}
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
	}
	// join new vxnet
	if newV.(string) != "" {
		joinVxnetInput := new(qc.JoinVxNetInput)
		joinVxnetInput.Instances = []*string{qc.String(d.Id())}
		joinVxnetInput.VxNet = qc.String(newV.(string))
		_, err := vxnetClt.JoinVxNet(joinVxnetInput)
		if err != nil {
			return fmt.Errorf("Error leave vxnet: %s", err)
		}
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
	}
	return nil
}

func instanceUpdateChangeSecurityGroup(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("security_group_id") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	sgClt := meta.(*QingCloudClient).securitygroup
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	input := new(qc.ApplySecurityGroupInput)
	input.SecurityGroup = qc.String(d.Get("security_group_id").(string))
	input.Instances = []*string{qc.String(d.Id())}
	_, err := sgClt.ApplySecurityGroup(input)
	if err != nil {
		return fmt.Errorf("Error apply security group: %s", err)
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	return nil
}

func instanceUpdateChangeEip(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("eip_id") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	eipClt := meta.(*QingCloudClient).eip
	describeEIPInput := new(qc.DescribeEIPsInput)
	describeEIPInput.EIPs = []*string{qc.String(d.Get("eip_id").(string))}
	describeEIPOutput, err := eipClt.DescribeEIPs(describeEIPInput)
	if err != nil {
		return fmt.Errorf("Error describe eip: %s", err)
	}
	if qc.StringValue(describeEIPOutput.EIPSet[0].Status) != "available" {
		return fmt.Errorf("Error eip %s state is %s", d.Get("eip_id").(string), qc.StringValue(describeEIPOutput.EIPSet[0].Status))
	}
	if _, err := EIPTransitionStateRefresh(eipClt, d.Get("eip_id").(string)); err != nil {
		return err
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	oldV, newV := d.GetChange("eip_id")
	// dissociate old eip
	if oldV.(string) != "" {
		dissociateEIPInput := new(qc.DissociateEIPsInput)
		dissociateEIPInput.EIPs = []*string{qc.String(oldV.(string))}
		_, err := eipClt.DissociateEIPs(dissociateEIPInput)
		if err != nil {
			return fmt.Errorf("Error dissociate eip: %s", err)
		}
	}

	if _, err := EIPTransitionStateRefresh(eipClt, d.Get("eip_id").(string)); err != nil {
		return err
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	// associate new eip
	if newV.(string) != "" {
		assoicateEIPInput := new(qc.AssociateEIPInput)
		assoicateEIPInput.EIP = qc.String(newV.(string))
		assoicateEIPInput.Instance = qc.String(d.Id())
		_, err := eipClt.AssociateEIP(assoicateEIPInput)
		if err != nil {
			return fmt.Errorf("Error assoicate eip: %s", err)
		}
	}
	if _, err := EIPTransitionStateRefresh(eipClt, d.Get("eip_id").(string)); err != nil {
		return err
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	return nil
}

func instanceUpdateChangeKeyPairs(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("keypair_ids") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	kpClt := meta.(*QingCloudClient).keypair

	oldV, newV := d.GetChange("keypair_ids")
	var nkps []string
	var okps []string
	for _, v := range oldV.(*schema.Set).List() {
		okps = append(okps, v.(string))
	}
	for _, v := range newV.(*schema.Set).List() {
		nkps = append(nkps, v.(string))
	}
	additions, deletions := stringSliceDiff(nkps, okps)
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	// attach new key_pair
	if len(additions) > 0 {
		attachInput := new(qc.AttachKeyPairsInput)
		attachInput.Instances = []*string{qc.String(d.Id())}
		attachInput.KeyPairs = qc.StringSlice(additions)
		output, err := kpClt.AttachKeyPairs(attachInput)
		if err != nil {
			return fmt.Errorf("Error attach keypairs: %s", err)
		}
		if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
			return fmt.Errorf("Error attach keypair: %s", *output.Message)
		}
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	// dettach old key_pair
	if len(deletions) > 0 {
		detachInput := new(qc.DetachKeyPairsInput)
		detachInput.Instances = []*string{qc.String(d.Id())}
		detachInput.KeyPairs = qc.StringSlice(deletions)
		output, err := kpClt.DetachKeyPairs(detachInput)
		if err != nil {
			return fmt.Errorf("Errorr detach keypairs: %s", err)
		}
		if output.RetCode != nil && qc.IntValue(output.RetCode) != 0 {
			return fmt.Errorf("Error detach keypair: %s", *output.Message)
		}
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
	}
	return nil
}

func instanceUpdateResize(d *schema.ResourceData, meta interface{}) error {
	if !d.HasChange("cpu") && !d.HasChange("memory") {
		return nil
	}
	clt := meta.(*QingCloudClient).instance
	// check instance state
	describeInstanceOutput, err := describeInstance(d, meta)
	if err != nil {
		return err
	}
	instance := describeInstanceOutput.InstanceSet[0]
	// stop instance
	if *instance.Status == "running" {
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			log.Printf("InstanceTransitionStateRefresh err: %s", err)
			return err
		}
		_, err := stopInstance(d, meta)
		if err != nil {
			return err
		}
		if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
			return err
		}
	}
	//  resize instance
	input := new(qc.ResizeInstancesInput)
	input.Instances = []*string{qc.String(d.Id())}
	input.CPU = qc.Int(d.Get("cpu").(int))
	input.Memory = qc.Int(d.Get("memory").(int))
	_, err = clt.ResizeInstances(input)
	if err != nil {
		return fmt.Errorf("Error resize instance: %s", err)
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	// start instance
	_, err = startInstance(d, meta)
	if err != nil {
		return err
	}
	if _, err := InstanceTransitionStateRefresh(clt, d.Id()); err != nil {
		return err
	}
	return nil
}

func describeInstance(d *schema.ResourceData, meta interface{}) (*qc.DescribeInstancesOutput, error) {
	clt := meta.(*QingCloudClient).instance
	input := new(qc.DescribeInstancesInput)
	input.Instances = []*string{qc.String(d.Id())}
	input.Verbose = qc.Int(1)
	output, err := clt.DescribeInstances(input)
	if err != nil {
		return nil, fmt.Errorf("Error describe instance: %s", err)
	}
	return output, nil
}

func stopInstance(d *schema.ResourceData, meta interface{}) (*qc.StopInstancesOutput, error) {
	clt := meta.(*QingCloudClient).instance
	input := new(qc.StopInstancesInput)
	input.Instances = []*string{qc.String(d.Id())}
	output, err := clt.StopInstances(input)
	if err != nil {
		return nil, fmt.Errorf("Error stop instance: %s", err)
	}
	return output, nil
}

func startInstance(d *schema.ResourceData, meta interface{}) (*qc.StartInstancesOutput, error) {
	clt := meta.(*QingCloudClient).instance
	input := new(qc.StartInstancesInput)
	input.Instances = []*string{qc.String(d.Id())}
	output, err := clt.StartInstances(input)
	if err != nil {
		return nil, fmt.Errorf("Error start instance: %s", err)
	}
	return output, nil
}

func deleteInstanceLeaveVxnet(d *schema.ResourceData, meta interface{}) (*qc.LeaveVxNetOutput, error) {
	vxnetID := d.Get("vxnet_id").(string)
	if vxnetID != "" {
		clt := meta.(*QingCloudClient).vxnet
		input := new(qc.LeaveVxNetInput)
		input.Instances = []*string{qc.String(d.Id())}
		input.VxNet = qc.String(vxnetID)
		_, err := clt.LeaveVxNet(input)
		if err != nil {
			return nil, fmt.Errorf("Error instance leave vxnet: %s", err)
		}
	}
	return nil, nil
}

func deleteInstanceDissociateEip(d *schema.ResourceData, meta interface{}) (*qc.LeaveVxNetOutput, error) {
	eipID := d.Get("eip_id").(string)
	if eipID != "" {
		clt := meta.(*QingCloudClient).eip
		if _, err := EIPTransitionStateRefresh(clt, eipID); err != nil {
			return nil, err
		}
		input := new(qc.DissociateEIPsInput)
		input.EIPs = []*string{qc.String(eipID)}
		_, err := clt.DissociateEIPs(input)
		if err != nil {
			return nil, fmt.Errorf("Error dissciate eip: %s", err)
		}
		if _, err := EIPTransitionStateRefresh(clt, eipID); err != nil {
			return nil, err
		}
	}
	return nil, nil
}

func updateInstanceVolume(d *schema.ResourceData, meta interface{}, create bool) error {
	// clt := meta.(*QingCloudClient).instance
	return nil
}
<|endoftext|>


<commit_before>// Package store provides a distributed SQLite instance.
//
// Distributed consensus is provided via the Raft algorithm.
package store

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	sql "github.com/otoolep/rqlite/db"
)

const (
	retainSnapshotCount = 2
	raftTimeout         = 10 * time.Second
	sqliteFile          = "db.sqlite"
	leaderWaitDelay     = 100 * time.Millisecond
	appliedWaitDelay    = 100 * time.Millisecond
)

var (
	// ErrFieldsRequired is returned when a node attempts to execute a leader-only
	// operation.
	ErrNotLeader = errors.New("not leader")
)

type command struct {
	Tx      bool     `json:"tx,omitempty"`
	Queries []string `json:"queries,omitempty"`
}

// Store is a SQLite database, where all changes are made via Raft consensus.
type Store struct {
	raftDir  string
	raftBind string

	mu sync.RWMutex // Sync access between queries and snapshots.

	ln     *networkLayer // Raft network between nodes.
	raft   *raft.Raft    // The consensus mechanism.
	dbConf *sql.Config   // SQLite database config.
	dbPath string        // Path to database file.
	db     *sql.DB       // The underlying SQLite store.

	logger *log.Logger
}

// New returns a new Store.
func New(dbConf *sql.Config, dir, bind string) *Store {
	dbPath := filepath.Join(dir, sqliteFile)
	if dbConf.Memory {
		dbPath = ":memory:"
	}

	return &Store{
		raftDir:  dir,
		raftBind: bind,
		dbConf:   dbConf,
		dbPath:   dbPath,
		logger:   log.New(os.Stderr, "[store] ", log.LstdFlags),
	}
}

// Open opens the store. If enableSingle is set, and there are no existing peers,
// then this node becomes the first node, and therefore leader, of the cluster.
func (s *Store) Open(enableSingle bool) error {
	if err := os.MkdirAll(s.raftDir, 0755); err != nil {
		return err
	}

	// Create the database. Unless it's a memory-based database, tt must be deleted
	if !s.dbConf.Memory {
		// as it will be rebuilt from (possibly) a snapshot and committed log entries.
		if err := os.Remove(s.dbPath); err != nil && !os.IsNotExist(err) {
			return err
		}
	}

	db, err := sql.OpenWithConfiguration(s.dbPath, s.dbConf)
	if err != nil {
		return err
	}
	s.db = db
	s.logger.Println("SQLite database opened at", s.dbConf.FQDSN(s.dbPath))

	// Setup Raft configuration.
	config := raft.DefaultConfig()

	// Check for any existing peers.
	peers, err := readPeersJSON(filepath.Join(s.raftDir, "peers.json"))
	if err != nil {
		return err
	}

	// Allow the node to entry single-mode, potentially electing itself, if
	// explicitly enabled and there is only 1 node in the cluster already.
	if enableSingle && len(peers) <= 1 {
		s.logger.Println("enabling single-node mode")
		config.EnableSingleNode = true
		config.DisableBootstrapAfterElect = false
	}

	// Setup Raft communication.
	ln, err := net.Listen("tcp", s.raftBind)
	if err != nil {
		return err
	}
	s.ln = newnetworkLayer(ln)
	transport := raft.NewNetworkTransport(s.ln, 3, 10*time.Second, os.Stderr)

	// Create peer storage.
	peerStore := raft.NewJSONPeers(s.raftDir, transport)

	// Create the snapshot store. This allows Raft to truncate the log.
	snapshots, err := raft.NewFileSnapshotStore(s.raftDir, retainSnapshotCount, os.Stderr)
	if err != nil {
		return fmt.Errorf("file snapshot store: %s", err)
	}

	// Create the log store and stable store.
	logStore, err := raftboltdb.NewBoltStore(filepath.Join(s.raftDir, "raft.db"))
	if err != nil {
		return fmt.Errorf("new bolt store: %s", err)
	}

	// Instantiate the Raft system.
	ra, err := raft.NewRaft(config, s, logStore, logStore, snapshots, peerStore, transport)
	if err != nil {
		return fmt.Errorf("new raft: %s", err)
	}
	s.raft = ra

	return nil
}

// Close closes the store.
func (s *Store) Close() error {
	if err := s.db.Close(); err != nil {
		return err
	}
	f := s.raft.Shutdown()
	if e := f.(raft.Future); e.Error() != nil {
		return e.Error()
	}
	return nil
}

// Path returns the path to the store's storage directory.
func (s *Store) Path() string {
	return s.raftDir
}

func (s *Store) Addr() net.Addr {
	return s.ln.Addr()
}

// Leader returns the current leader. Returns a blank string if there is
// no leader.
func (s *Store) Leader() string {
	return s.raft.Leader()
}

// WaitForLeader blocks until a leader is detected, or the timeout expires.
func (s *Store) WaitForLeader(timeout time.Duration) (string, error) {
	tck := time.NewTicker(leaderWaitDelay)
	defer tck.Stop()
	tmr := time.NewTimer(timeout)
	defer tmr.Stop()

	for {
		select {
		case <-tck.C:
			l := s.Leader()
			if l != "" {
				return l, nil
			}
		case <-tmr.C:
			return "", fmt.Errorf("timeout expired")
		}
	}
}

// WaitForAppliedIndex blocks until a given log index has been applied,
// or the timeout expires.
func (s *Store) WaitForAppliedIndex(idx uint64, timeout time.Duration) error {
	tck := time.NewTicker(appliedWaitDelay)
	defer tck.Stop()
	tmr := time.NewTimer(timeout)
	defer tmr.Stop()

	for {
		select {
		case <-tck.C:
			if s.raft.AppliedIndex() >= idx {
				return nil
			}
		case <-tmr.C:
			return fmt.Errorf("timeout expired")
		}
	}
}

// Stats returns stats for the store.
func (s *Store) Stats() (map[string]interface{}, error) {
	dbStatus := map[string]interface{}{
		"path":    s.dbPath,
		"dns":     s.dbConf.FQDSN(s.dbPath),
		"version": sql.DBVersion,
	}
	status := map[string]interface{}{
		"raft":    s.raft.Stats(),
		"addr":    s.Addr().String(),
		"leader":  s.Leader(),
		"sqlite3": dbStatus,
	}
	return status, nil
}

// Execute executes queries that return no rows, but do modify the database.
func (s *Store) Execute(queries []string, tx bool) ([]*sql.Result, error) {
	if s.raft.State() != raft.Leader {
		return nil, ErrNotLeader
	}

	c := &command{
		Tx:      tx,
		Queries: queries,
	}
	b, err := json.Marshal(c)
	if err != nil {
		return nil, err
	}

	f := s.raft.Apply(b, raftTimeout)
	if e := f.(raft.Future); e.Error() != nil {
		return nil, e.Error()
	}

	r := f.Response().(*fsmResponse)
	return r.results, r.error
}

// Backup return a consistent snapshot of the underlying database.
func (s *Store) Backup(leader bool) ([]byte, error) {
	if leader && s.raft.State() != raft.Leader {
		return nil, fmt.Errorf("not leader")
	}

	f, err := ioutil.TempFile("", "rqlilte-bak-")
	if err != nil {
		return nil, err
	}
	f.Close()
	defer os.Remove(f.Name())

	if err := s.db.Backup(f.Name()); err != nil {
		return nil, err
	}

	b, err := ioutil.ReadFile(f.Name())
	if err != nil {
		return nil, err
	}
	return b, nil
}

// Query executes queries that return rows, and do not modify the database.
func (s *Store) Query(queries []string, tx, leader, verify bool) ([]*sql.Rows, error) {
	// Allow concurrent queries.
	s.mu.RLock()
	defer s.mu.RUnlock()

	if leader && s.raft.State() != raft.Leader {
		return nil, ErrNotLeader
	}

	if verify {
		f := s.raft.VerifyLeader()
		if e := f.(raft.Future); e.Error() != nil {
			return nil, e.Error()
		}
	}

	r, err := s.db.Query(queries, tx, true)
	return r, err
}

// Join joins a node, located at addr, to this store. The node must be ready to
// respond to Raft communications at that address.
func (s *Store) Join(addr string) error {
	s.logger.Printf("received request to join node at %s", addr)

	f := s.raft.AddPeer(addr)
	if f.Error() != nil {
		return f.Error()
	}
	s.logger.Printf("node at %s joined successfully", addr)
	return nil
}

type fsmResponse struct {
	results []*sql.Result
	error   error
}

// Apply applies a Raft log entry to the database.
func (s *Store) Apply(l *raft.Log) interface{} {
	var c command
	if err := json.Unmarshal(l.Data, &c); err != nil {
		panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error()))
	}

	r, err := s.db.Execute(c.Queries, c.Tx, true)

	return &fsmResponse{results: r, error: err}
}

// Snapshot returns a snapshot of the database. The caller must ensure that
// no transaction is taking place during this call. Hashsicorp Raft guarantees
// that this function will not be called concurrently with Apply.
//
// http://sqlite.org/howtocorrupt.html states it is safe to do this
// as long as no transaction is in progress.
func (s *Store) Snapshot() (raft.FSMSnapshot, error) {
	// Ensure only one snapshot can take place at once, and block all queries.
	s.mu.Lock()
	defer s.mu.Unlock()

	f, err := ioutil.TempFile("", "rqlilte-snap-")
	if err != nil {
		return nil, err
	}
	f.Close()
	defer os.Remove(f.Name())

	if err := s.db.Backup(f.Name()); err != nil {
		return nil, err
	}

	b, err := ioutil.ReadFile(f.Name())
	if err != nil {
		log.Printf("Failed to generate snapshot: %s", err.Error())
		return nil, err
	}
	return &fsmSnapshot{data: b}, nil
}

// Restore restores the database to a previous state.
func (s *Store) Restore(rc io.ReadCloser) error {
	if err := os.Remove(s.dbPath); err != nil && !os.IsNotExist(err) {
		return err
	}

	b, err := ioutil.ReadAll(rc)
	if err != nil {
		return err
	}

	if err := ioutil.WriteFile(s.dbPath, b, 0660); err != nil {
		return err
	}

	db, err := sql.OpenWithConfiguration(s.dbPath, s.dbConf)
	if err != nil {
		return err
	}
	s.db = db

	return nil
}

type fsmSnapshot struct {
	data []byte
}

// Persist writes the snapshot to the give sink.
func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error {
	err := func() error {
		// Write data to sink.
		if _, err := sink.Write(f.data); err != nil {
			return err
		}

		// Close the sink.
		if err := sink.Close(); err != nil {
			return err
		}

		return nil
	}()

	if err != nil {
		sink.Cancel()
		return err
	}

	return nil
}

// Release is a no-op.
func (f *fsmSnapshot) Release() {}

// readPeersJSON reads the peers from the path.
func readPeersJSON(path string) ([]string, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if len(b) == 0 {
		return nil, nil
	}

	var peers []string
	dec := json.NewDecoder(bytes.NewReader(b))
	if err := dec.Decode(&peers); err != nil {
		return nil, err
	}

	return peers, nil
}
<commit_msg>Refactor Raft command to support queries<commit_after>// Package store provides a distributed SQLite instance.
//
// Distributed consensus is provided via the Raft algorithm.
package store

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	sql "github.com/otoolep/rqlite/db"
)

const (
	retainSnapshotCount = 2
	raftTimeout         = 10 * time.Second
	sqliteFile          = "db.sqlite"
	leaderWaitDelay     = 100 * time.Millisecond
	appliedWaitDelay    = 100 * time.Millisecond
)

var (
	// ErrFieldsRequired is returned when a node attempts to execute a leader-only
	// operation.
	ErrNotLeader = errors.New("not leader")
)

type commandType int

const (
	execute = iota
	query
)

type command struct {
	typ     commandType `json:"typ,omitempty"`
	Tx      bool        `json:"tx,omitempty"`
	Queries []string    `json:"queries,omitempty"`
	Timings bool        `json:"timings",omitempty"`
}

// Store is a SQLite database, where all changes are made via Raft consensus.
type Store struct {
	raftDir  string
	raftBind string

	mu sync.RWMutex // Sync access between queries and snapshots.

	ln     *networkLayer // Raft network between nodes.
	raft   *raft.Raft    // The consensus mechanism.
	dbConf *sql.Config   // SQLite database config.
	dbPath string        // Path to database file.
	db     *sql.DB       // The underlying SQLite store.

	logger *log.Logger
}

// New returns a new Store.
func New(dbConf *sql.Config, dir, bind string) *Store {
	dbPath := filepath.Join(dir, sqliteFile)
	if dbConf.Memory {
		dbPath = ":memory:"
	}

	return &Store{
		raftDir:  dir,
		raftBind: bind,
		dbConf:   dbConf,
		dbPath:   dbPath,
		logger:   log.New(os.Stderr, "[store] ", log.LstdFlags),
	}
}

// Open opens the store. If enableSingle is set, and there are no existing peers,
// then this node becomes the first node, and therefore leader, of the cluster.
func (s *Store) Open(enableSingle bool) error {
	if err := os.MkdirAll(s.raftDir, 0755); err != nil {
		return err
	}

	// Create the database. Unless it's a memory-based database, tt must be deleted
	if !s.dbConf.Memory {
		// as it will be rebuilt from (possibly) a snapshot and committed log entries.
		if err := os.Remove(s.dbPath); err != nil && !os.IsNotExist(err) {
			return err
		}
	}

	db, err := sql.OpenWithConfiguration(s.dbPath, s.dbConf)
	if err != nil {
		return err
	}
	s.db = db
	s.logger.Println("SQLite database opened at", s.dbConf.FQDSN(s.dbPath))

	// Setup Raft configuration.
	config := raft.DefaultConfig()

	// Check for any existing peers.
	peers, err := readPeersJSON(filepath.Join(s.raftDir, "peers.json"))
	if err != nil {
		return err
	}

	// Allow the node to entry single-mode, potentially electing itself, if
	// explicitly enabled and there is only 1 node in the cluster already.
	if enableSingle && len(peers) <= 1 {
		s.logger.Println("enabling single-node mode")
		config.EnableSingleNode = true
		config.DisableBootstrapAfterElect = false
	}

	// Setup Raft communication.
	ln, err := net.Listen("tcp", s.raftBind)
	if err != nil {
		return err
	}
	s.ln = newnetworkLayer(ln)
	transport := raft.NewNetworkTransport(s.ln, 3, 10*time.Second, os.Stderr)

	// Create peer storage.
	peerStore := raft.NewJSONPeers(s.raftDir, transport)

	// Create the snapshot store. This allows Raft to truncate the log.
	snapshots, err := raft.NewFileSnapshotStore(s.raftDir, retainSnapshotCount, os.Stderr)
	if err != nil {
		return fmt.Errorf("file snapshot store: %s", err)
	}

	// Create the log store and stable store.
	logStore, err := raftboltdb.NewBoltStore(filepath.Join(s.raftDir, "raft.db"))
	if err != nil {
		return fmt.Errorf("new bolt store: %s", err)
	}

	// Instantiate the Raft system.
	ra, err := raft.NewRaft(config, s, logStore, logStore, snapshots, peerStore, transport)
	if err != nil {
		return fmt.Errorf("new raft: %s", err)
	}
	s.raft = ra

	return nil
}

// Close closes the store.
func (s *Store) Close() error {
	if err := s.db.Close(); err != nil {
		return err
	}
	f := s.raft.Shutdown()
	if e := f.(raft.Future); e.Error() != nil {
		return e.Error()
	}
	return nil
}

// Path returns the path to the store's storage directory.
func (s *Store) Path() string {
	return s.raftDir
}

func (s *Store) Addr() net.Addr {
	return s.ln.Addr()
}

// Leader returns the current leader. Returns a blank string if there is
// no leader.
func (s *Store) Leader() string {
	return s.raft.Leader()
}

// WaitForLeader blocks until a leader is detected, or the timeout expires.
func (s *Store) WaitForLeader(timeout time.Duration) (string, error) {
	tck := time.NewTicker(leaderWaitDelay)
	defer tck.Stop()
	tmr := time.NewTimer(timeout)
	defer tmr.Stop()

	for {
		select {
		case <-tck.C:
			l := s.Leader()
			if l != "" {
				return l, nil
			}
		case <-tmr.C:
			return "", fmt.Errorf("timeout expired")
		}
	}
}

// WaitForAppliedIndex blocks until a given log index has been applied,
// or the timeout expires.
func (s *Store) WaitForAppliedIndex(idx uint64, timeout time.Duration) error {
	tck := time.NewTicker(appliedWaitDelay)
	defer tck.Stop()
	tmr := time.NewTimer(timeout)
	defer tmr.Stop()

	for {
		select {
		case <-tck.C:
			if s.raft.AppliedIndex() >= idx {
				return nil
			}
		case <-tmr.C:
			return fmt.Errorf("timeout expired")
		}
	}
}

// Stats returns stats for the store.
func (s *Store) Stats() (map[string]interface{}, error) {
	dbStatus := map[string]interface{}{
		"path":    s.dbPath,
		"dns":     s.dbConf.FQDSN(s.dbPath),
		"version": sql.DBVersion,
	}
	status := map[string]interface{}{
		"raft":    s.raft.Stats(),
		"addr":    s.Addr().String(),
		"leader":  s.Leader(),
		"sqlite3": dbStatus,
	}
	return status, nil
}

// Execute executes queries that return no rows, but do modify the database.
func (s *Store) Execute(queries []string, tx bool) ([]*sql.Result, error) {
	if s.raft.State() != raft.Leader {
		return nil, ErrNotLeader
	}

	c := &command{
		typ:     execute,
		Tx:      tx,
		Queries: queries,
	}
	b, err := json.Marshal(c)
	if err != nil {
		return nil, err
	}

	f := s.raft.Apply(b, raftTimeout)
	if e := f.(raft.Future); e.Error() != nil {
		return nil, e.Error()
	}

	r := f.Response().(*fsmExecuteResponse)
	return r.results, r.error
}

// Backup return a consistent snapshot of the underlying database.
func (s *Store) Backup(leader bool) ([]byte, error) {
	if leader && s.raft.State() != raft.Leader {
		return nil, fmt.Errorf("not leader")
	}

	f, err := ioutil.TempFile("", "rqlilte-bak-")
	if err != nil {
		return nil, err
	}
	f.Close()
	defer os.Remove(f.Name())

	if err := s.db.Backup(f.Name()); err != nil {
		return nil, err
	}

	b, err := ioutil.ReadFile(f.Name())
	if err != nil {
		return nil, err
	}
	return b, nil
}

// Query executes queries that return rows, and do not modify the database.
func (s *Store) Query(queries []string, tx, leader, verify bool) ([]*sql.Rows, error) {
	// Allow concurrent queries.
	s.mu.RLock()
	defer s.mu.RUnlock()

	if leader && s.raft.State() != raft.Leader {
		return nil, ErrNotLeader
	}

	if verify {
		f := s.raft.VerifyLeader()
		if e := f.(raft.Future); e.Error() != nil {
			return nil, e.Error()
		}
	}

	r, err := s.db.Query(queries, tx, true)
	return r, err
}

// Join joins a node, located at addr, to this store. The node must be ready to
// respond to Raft communications at that address.
func (s *Store) Join(addr string) error {
	s.logger.Printf("received request to join node at %s", addr)

	f := s.raft.AddPeer(addr)
	if f.Error() != nil {
		return f.Error()
	}
	s.logger.Printf("node at %s joined successfully", addr)
	return nil
}

type fsmExecuteResponse struct {
	results []*sql.Result
	error   error
}

type fsmQueryResponse struct {
	rows  []*sql.Rows
	error error
}

// Apply applies a Raft log entry to the database.
func (s *Store) Apply(l *raft.Log) interface{} {
	var c command
	if err := json.Unmarshal(l.Data, &c); err != nil {
		panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error()))
	}

	if c.typ == execute {
		r, err := s.db.Execute(c.Queries, c.Tx, c.Timings)
		return &fsmExecuteResponse{results: r, error: err}
	}
	r, err := s.db.Query(c.Queries, c.Tx, c.Timings)
	return &fsmQueryResponse{rows: r, error: err}
}

// Snapshot returns a snapshot of the database. The caller must ensure that
// no transaction is taking place during this call. Hashsicorp Raft guarantees
// that this function will not be called concurrently with Apply.
//
// http://sqlite.org/howtocorrupt.html states it is safe to do this
// as long as no transaction is in progress.
func (s *Store) Snapshot() (raft.FSMSnapshot, error) {
	// Ensure only one snapshot can take place at once, and block all queries.
	s.mu.Lock()
	defer s.mu.Unlock()

	f, err := ioutil.TempFile("", "rqlilte-snap-")
	if err != nil {
		return nil, err
	}
	f.Close()
	defer os.Remove(f.Name())

	if err := s.db.Backup(f.Name()); err != nil {
		return nil, err
	}

	b, err := ioutil.ReadFile(f.Name())
	if err != nil {
		log.Printf("Failed to generate snapshot: %s", err.Error())
		return nil, err
	}
	return &fsmSnapshot{data: b}, nil
}

// Restore restores the database to a previous state.
func (s *Store) Restore(rc io.ReadCloser) error {
	if err := os.Remove(s.dbPath); err != nil && !os.IsNotExist(err) {
		return err
	}

	b, err := ioutil.ReadAll(rc)
	if err != nil {
		return err
	}

	if err := ioutil.WriteFile(s.dbPath, b, 0660); err != nil {
		return err
	}

	db, err := sql.OpenWithConfiguration(s.dbPath, s.dbConf)
	if err != nil {
		return err
	}
	s.db = db

	return nil
}

type fsmSnapshot struct {
	data []byte
}

// Persist writes the snapshot to the give sink.
func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error {
	err := func() error {
		// Write data to sink.
		if _, err := sink.Write(f.data); err != nil {
			return err
		}

		// Close the sink.
		if err := sink.Close(); err != nil {
			return err
		}

		return nil
	}()

	if err != nil {
		sink.Cancel()
		return err
	}

	return nil
}

// Release is a no-op.
func (f *fsmSnapshot) Release() {}

// readPeersJSON reads the peers from the path.
func readPeersJSON(path string) ([]string, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if len(b) == 0 {
		return nil, nil
	}

	var peers []string
	dec := json.NewDecoder(bytes.NewReader(b))
	if err := dec.Decode(&peers); err != nil {
		return nil, err
	}

	return peers, nil
}
<|endoftext|>