<commit_before>// Copyright (c) 2013-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package connmgr

import (
	"encoding/binary"
	"errors"
	"net"
)

const (
	torSucceeded         = 0x00
	torGeneralError      = 0x01
	torNotAllowed        = 0x02
	torNetUnreachable    = 0x03
	torHostUnreachable   = 0x04
	torConnectionRefused = 0x05
	torTTLExpired        = 0x06
	torCmdNotSupported   = 0x07
	torAddrNotSupported  = 0x08
)

var (
	// ErrTorInvalidAddressResponse indicates an invalid address was
	// returned by the Tor DNS resolver.
	ErrTorInvalidAddressResponse = errors.New("invalid address response")

	// ErrTorInvalidProxyResponse indicates the Tor proxy returned a
	// response in an unexpected format.
	ErrTorInvalidProxyResponse = errors.New("invalid proxy response")

	// ErrTorUnrecognizedAuthMethod indicates the authentication method
	// provided is not recognized.
	ErrTorUnrecognizedAuthMethod = errors.New("invalid proxy authentication method")

	torStatusErrors = map[byte]error{
		torSucceeded:         errors.New("tor succeeded"),
		torGeneralError:      errors.New("tor general error"),
		torNotAllowed:        errors.New("tor not allowed"),
		torNetUnreachable:    errors.New("tor network is unreachable"),
		torHostUnreachable:   errors.New("tor host is unreachable"),
		torConnectionRefused: errors.New("tor connection refused"),
		torTTLExpired:        errors.New("tor TTL expired"),
		torCmdNotSupported:   errors.New("tor command not supported"),
		torAddrNotSupported:  errors.New("tor address type not supported"),
	}
)

// TorLookupIP uses Tor to resolve DNS via the SOCKS extension they provide for
// resolution over the Tor network. Tor itself doesn't support ipv6 so this
// doesn't either.
func TorLookupIP(host, proxy string) ([]net.IP, error) {
	conn, err := net.Dial("tcp", proxy)
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	buf := []byte{'\x05', '\x01', '\x00'}
	_, err = conn.Write(buf)
	if err != nil {
		return nil, err
	}

	buf = make([]byte, 2)
	_, err = conn.Read(buf)
	if err != nil {
		return nil, err
	}
	if buf[0] != '\x05' {
		return nil, ErrTorInvalidProxyResponse
	}
	if buf[1] != '\x00' {
		return nil, ErrTorUnrecognizedAuthMethod
	}

	buf = make([]byte, 7+len(host))
	buf[0] = 5      // protocol version
	buf[1] = '\xF0' // Tor Resolve
	buf[2] = 0      // reserved
	buf[3] = 3      // Tor Resolve
	buf[4] = byte(len(host))
	copy(buf[5:], host)
	buf[5+len(host)] = 0 // Port 0

	_, err = conn.Write(buf)
	if err != nil {
		return nil, err
	}

	buf = make([]byte, 4)
	_, err = conn.Read(buf)
	if err != nil {
		return nil, err
	}
	if buf[0] != 5 {
		return nil, ErrTorInvalidProxyResponse
	}
	if buf[1] != 0 {
		if int(buf[1]) > len(torStatusErrors) {
			return nil, ErrTorInvalidProxyResponse
		} else if err := torStatusErrors[buf[1]]; err != nil {
			return nil, err
		}
		return nil, ErrTorInvalidProxyResponse
	}
	if buf[3] != 1 {
		err := torStatusErrors[torGeneralError]
		return nil, err
	}

	buf = make([]byte, 4)
	bytes, err := conn.Read(buf)
	if err != nil {
		return nil, err
	}
	if bytes != 4 {
		return nil, ErrTorInvalidAddressResponse
	}

	r := binary.BigEndian.Uint32(buf)

	addr := make([]net.IP, 1)
	addr[0] = net.IPv4(byte(r>>24), byte(r>>16), byte(r>>8), byte(r))

	return addr, nil
}
<commit_msg>Fix comparator that could cause a panic<commit_after>// Copyright (c) 2013-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package connmgr

import (
	"encoding/binary"
	"errors"
	"net"
)

const (
	torSucceeded         = 0x00
	torGeneralError      = 0x01
	torNotAllowed        = 0x02
	torNetUnreachable    = 0x03
	torHostUnreachable   = 0x04
	torConnectionRefused = 0x05
	torTTLExpired        = 0x06
	torCmdNotSupported   = 0x07
	torAddrNotSupported  = 0x08
)

var (
	// ErrTorInvalidAddressResponse indicates an invalid address was
	// returned by the Tor DNS resolver.
	ErrTorInvalidAddressResponse = errors.New("invalid address response")

	// ErrTorInvalidProxyResponse indicates the Tor proxy returned a
	// response in an unexpected format.
	ErrTorInvalidProxyResponse = errors.New("invalid proxy response")

	// ErrTorUnrecognizedAuthMethod indicates the authentication method
	// provided is not recognized.
	ErrTorUnrecognizedAuthMethod = errors.New("invalid proxy authentication method")

	torStatusErrors = map[byte]error{
		torSucceeded:         errors.New("tor succeeded"),
		torGeneralError:      errors.New("tor general error"),
		torNotAllowed:        errors.New("tor not allowed"),
		torNetUnreachable:    errors.New("tor network is unreachable"),
		torHostUnreachable:   errors.New("tor host is unreachable"),
		torConnectionRefused: errors.New("tor connection refused"),
		torTTLExpired:        errors.New("tor TTL expired"),
		torCmdNotSupported:   errors.New("tor command not supported"),
		torAddrNotSupported:  errors.New("tor address type not supported"),
	}
)

// TorLookupIP uses Tor to resolve DNS via the SOCKS extension they provide for
// resolution over the Tor network. Tor itself doesn't support ipv6 so this
// doesn't either.
func TorLookupIP(host, proxy string) ([]net.IP, error) {
	conn, err := net.Dial("tcp", proxy)
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	buf := []byte{'\x05', '\x01', '\x00'}
	_, err = conn.Write(buf)
	if err != nil {
		return nil, err
	}

	buf = make([]byte, 2)
	_, err = conn.Read(buf)
	if err != nil {
		return nil, err
	}
	if buf[0] != '\x05' {
		return nil, ErrTorInvalidProxyResponse
	}
	if buf[1] != '\x00' {
		return nil, ErrTorUnrecognizedAuthMethod
	}

	buf = make([]byte, 7+len(host))
	buf[0] = 5      // protocol version
	buf[1] = '\xF0' // Tor Resolve
	buf[2] = 0      // reserved
	buf[3] = 3      // Tor Resolve
	buf[4] = byte(len(host))
	copy(buf[5:], host)
	buf[5+len(host)] = 0 // Port 0

	_, err = conn.Write(buf)
	if err != nil {
		return nil, err
	}

	buf = make([]byte, 4)
	_, err = conn.Read(buf)
	if err != nil {
		return nil, err
	}
	if buf[0] != 5 {
		return nil, ErrTorInvalidProxyResponse
	}
	if buf[1] != 0 {
		if int(buf[1]) >= len(torStatusErrors) {
			return nil, ErrTorInvalidProxyResponse
		} else if err := torStatusErrors[buf[1]]; err != nil {
			return nil, err
		}
		return nil, ErrTorInvalidProxyResponse
	}
	if buf[3] != 1 {
		err := torStatusErrors[torGeneralError]
		return nil, err
	}

	buf = make([]byte, 4)
	bytes, err := conn.Read(buf)
	if err != nil {
		return nil, err
	}
	if bytes != 4 {
		return nil, ErrTorInvalidAddressResponse
	}

	r := binary.BigEndian.Uint32(buf)

	addr := make([]net.IP, 1)
	addr[0] = net.IPv4(byte(r>>24), byte(r>>16), byte(r>>8), byte(r))

	return addr, nil
}
<|endoftext|>


<commit_before>/*
   PulseHA - HA Cluster Daemon
   Copyright (C) 2017-2019  Andrew Zak <andrew@linux.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
package config

import (
	"encoding/json"
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/Syleron/PulseHA/src/utils"
	"io/ioutil"
	"os"
	"sync"
)

type Config struct {
	Pulse   Local               `json:"pulseha"`
	Groups  map[string][]string `json:"floating_ip_groups"`
	Nodes   map[string]Node     `json:"nodes"`
	Logging Logging             `json:"logging"`
	sync.Mutex
}

type Local struct {
	TLS                 bool   `json:"tls"`
	HealthCheckInterval int    `json:"hcs_interval"`
	FailOverInterval    int    `json:"fos_interval"`
	FailOverLimit       int    `json:"fo_limit"`
	LocalNode           string `json:"local_node"`
}

type Nodes struct {
	Nodes map[string]Node
}

type Node struct {
	IP       string              `json:"bind_address"`
	Port     string              `json:"bind_port"`
	IPGroups map[string][]string `json:"group_assignments"`
}

type Logging struct {
	Level     string `json:"level"`
	ToLogFile bool   `json:"to_logfile"`
	LogFile   string `json:"logfile"`
}

/**
 * Returns a copy of the config
 */
func (c *Config) GetConfig() Config {
	return *c
}

/**
 * Sets the local node name
 */
func (c *Config) SetLocalNode() error {
	hostname, err := utils.GetHostname()
	if err != nil {
		return errors.New("cannot set local node because unable to get local hostname")
	}
	log.Debugf("Config:setLocalNode Hostname is: %s", hostname)
	c.Pulse.LocalNode = hostname
	return nil
}

/**

 */
func (c *Config) NodeCount() int {
	return len(c.Nodes)
}

/**
 * Return the local node name
 */
func (c *Config) GetLocalNode() string {
	return c.Pulse.LocalNode
}

/**
 * Function used to load the config
 */
func (c *Config) Load() {
	c.Lock()
	defer c.Unlock()
	b, err := ioutil.ReadFile("/etc/pulseha/config.json")
	if err != nil {
		log.Fatalf("Error reading config file: %s", err)
	}
	err = json.Unmarshal([]byte(b), &c)
	if err != nil {
		log.Fatalf("Unable to unmarshal config: %s", err)
	}
	if err != nil {
		log.Fatal("Unable to load config.json. Either it doesn't exist or there may be a permissions issue")
	}
	err = c.SetLocalNode()
	if err != nil {
		log.Fatalf("The local Hostname does not match the configuration")
	}
}

/**
 * Function used to save the config
 */
func (c *Config) Save() {
	log.Debug("Config:Save() Saving config..")
	c.Lock()
	defer c.Unlock()
	// Validate before we save
	c.Validate()
	// Convert struct back to JSON format
	configJSON, _ := json.MarshalIndent(c, "", "    ")
	// Save back to file
	err := ioutil.WriteFile("/etc/pulseha/config.json", configJSON, 0644)
	// Check for errors
	if err != nil {
		log.Error("Unable to save config.json. Either it doesn't exist or there may be a permissions issue")
		os.Exit(1)
	}
}

/**
 * Reload the config file into memory.
 * Note: Need to clear memory value before calling Load()
 */
func (c *Config) Reload() {
	log.Info("Reloading PulseHA config")
	c.Load()
}

/**
 *
 */
func (c *Config) Validate() {
	var success bool = true

	// Make sure our groups section is valid
	if c.Groups == nil {
		log.Fatal("Unable to load Groups section of the config.")
		success = false
	}

	// Make sure our nodes section is valid
	if c.Nodes == nil {
		log.Fatal("Unable to load Nodes section of the config.")
		success = false
	}

	// if we are in a cluster.. does our hostname exist?
	if c.ClusterCheck() {
		for name, _ := range c.Nodes {
			if _, ok := c.Nodes[name]; !ok {
				log.Fatal()"Hostname mistmatch. Localhost does not exist in cluster config.")
				success = false
			}
		}
	}

	if c.Pulse.FailOverInterval < 1000 || c.Pulse.FailOverLimit < 1000 || c.Pulse.HealthCheckInterval < 1000 {
		log.Fatal("Please make sure the interval and limit values in your config are valid millisecond values of at least 1 second")
		success = false
	}

	if c.Pulse.FailOverLimit < c.Pulse.FailOverInterval {
		log.Fatal("The fos_interval value must be a smaller value then your fo_limit")
		success = false
	}

	// TODO: Check if our hostname exists in the cluster config
	// TODO: Check if we have valid network interface names

	// Handles if shit hits the roof
	if success == false {
		// log why we exited?
		os.Exit(1)
	}
}

/**
 *
 */
func (c *Config) LocalNode() Node {
	hostname, err := utils.GetHostname()
	if err != nil {
		return Node{}
	}
	return c.Nodes[hostname]
}

/**
 * Private - Check to see if we are in a configured cluster or not.
 */
func (c *Config) ClusterCheck() bool {
	total := len(c.Nodes)
	if total > 0 {
		// if there is only one node we can assume it's ours
		if total == 1 {
			// make sure we have a bind IP/Port or we are not in a cluster
			hostname, err := utils.GetHostname()
			if err != nil {
				return false
			}
			if c.Nodes[hostname].IP == "" && c.Nodes[hostname].Port == "" {
				return false
			}
		}
		return true
	}
	return false
}

/**
Returns the interface the group is assigned to
*/
func (c *Config) GetGroupIface(node string, groupName string) string {
	for nodeName, n := range c.Nodes {
		if nodeName == node {
			for iface, groups := range n.IPGroups {
				for _, group := range groups {
					if group == groupName {
						return iface
					}
				}
			}
		}
	}
	return ""
}

/**
Instantiate, setup and return our Config
 */
func GetConfig() *Config {
	cfg := Config{}
	cfg.Load()
	cfg.Validate()
	return &cfg
}

/**
Returns the hostname for a node based of it's IP address
*/
func (c *Config) GetNodeHostnameByAddress(address string) (string, error) {
	for name, node := range c.Nodes {
		if node.IP == address {
			return name, nil
		}
	}
	return "", errors.New("unable to find node with IP address " + address)
}
<commit_msg>Fixed fatal message<commit_after>/*
   PulseHA - HA Cluster Daemon
   Copyright (C) 2017-2019  Andrew Zak <andrew@linux.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
package config

import (
	"encoding/json"
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/Syleron/PulseHA/src/utils"
	"io/ioutil"
	"os"
	"sync"
)

type Config struct {
	Pulse   Local               `json:"pulseha"`
	Groups  map[string][]string `json:"floating_ip_groups"`
	Nodes   map[string]Node     `json:"nodes"`
	Logging Logging             `json:"logging"`
	sync.Mutex
}

type Local struct {
	TLS                 bool   `json:"tls"`
	HealthCheckInterval int    `json:"hcs_interval"`
	FailOverInterval    int    `json:"fos_interval"`
	FailOverLimit       int    `json:"fo_limit"`
	LocalNode           string `json:"local_node"`
}

type Nodes struct {
	Nodes map[string]Node
}

type Node struct {
	IP       string              `json:"bind_address"`
	Port     string              `json:"bind_port"`
	IPGroups map[string][]string `json:"group_assignments"`
}

type Logging struct {
	Level     string `json:"level"`
	ToLogFile bool   `json:"to_logfile"`
	LogFile   string `json:"logfile"`
}

/**
 * Returns a copy of the config
 */
func (c *Config) GetConfig() Config {
	return *c
}

/**
 * Sets the local node name
 */
func (c *Config) SetLocalNode() error {
	hostname, err := utils.GetHostname()
	if err != nil {
		return errors.New("cannot set local node because unable to get local hostname")
	}
	log.Debugf("Config:setLocalNode Hostname is: %s", hostname)
	c.Pulse.LocalNode = hostname
	return nil
}

/**

 */
func (c *Config) NodeCount() int {
	return len(c.Nodes)
}

/**
 * Return the local node name
 */
func (c *Config) GetLocalNode() string {
	return c.Pulse.LocalNode
}

/**
 * Function used to load the config
 */
func (c *Config) Load() {
	c.Lock()
	defer c.Unlock()
	b, err := ioutil.ReadFile("/etc/pulseha/config.json")
	if err != nil {
		log.Fatalf("Error reading config file: %s", err)
	}
	err = json.Unmarshal([]byte(b), &c)
	if err != nil {
		log.Fatalf("Unable to unmarshal config: %s", err)
	}
	if err != nil {
		log.Fatal("Unable to load config.json. Either it doesn't exist or there may be a permissions issue")
	}
	err = c.SetLocalNode()
	if err != nil {
		log.Fatalf("The local Hostname does not match the configuration")
	}
}

/**
 * Function used to save the config
 */
func (c *Config) Save() {
	log.Debug("Config:Save() Saving config..")
	c.Lock()
	defer c.Unlock()
	// Validate before we save
	c.Validate()
	// Convert struct back to JSON format
	configJSON, _ := json.MarshalIndent(c, "", "    ")
	// Save back to file
	err := ioutil.WriteFile("/etc/pulseha/config.json", configJSON, 0644)
	// Check for errors
	if err != nil {
		log.Error("Unable to save config.json. Either it doesn't exist or there may be a permissions issue")
		os.Exit(1)
	}
}

/**
 * Reload the config file into memory.
 * Note: Need to clear memory value before calling Load()
 */
func (c *Config) Reload() {
	log.Info("Reloading PulseHA config")
	c.Load()
}

/**
 *
 */
func (c *Config) Validate() {
	var success bool = true

	// Make sure our groups section is valid
	if c.Groups == nil {
		log.Fatal("Unable to load Groups section of the config.")
		success = false
	}

	// Make sure our nodes section is valid
	if c.Nodes == nil {
		log.Fatal("Unable to load Nodes section of the config.")
		success = false
	}

	// if we are in a cluster.. does our hostname exist?
	if c.ClusterCheck() {
		for name, _ := range c.Nodes {
			if _, ok := c.Nodes[name]; !ok {
				log.Fatal("Hostname mistmatch. Localhost does not exist in cluster config.")
				success = false
			}
		}
	}

	if c.Pulse.FailOverInterval < 1000 || c.Pulse.FailOverLimit < 1000 || c.Pulse.HealthCheckInterval < 1000 {
		log.Fatal("Please make sure the interval and limit values in your config are valid millisecond values of at least 1 second")
		success = false
	}

	if c.Pulse.FailOverLimit < c.Pulse.FailOverInterval {
		log.Fatal("The fos_interval value must be a smaller value then your fo_limit")
		success = false
	}

	// TODO: Check if our hostname exists in the cluster config
	// TODO: Check if we have valid network interface names

	// Handles if shit hits the roof
	if success == false {
		// log why we exited?
		os.Exit(1)
	}
}

/**
 *
 */
func (c *Config) LocalNode() Node {
	hostname, err := utils.GetHostname()
	if err != nil {
		return Node{}
	}
	return c.Nodes[hostname]
}

/**
 * Private - Check to see if we are in a configured cluster or not.
 */
func (c *Config) ClusterCheck() bool {
	total := len(c.Nodes)
	if total > 0 {
		// if there is only one node we can assume it's ours
		if total == 1 {
			// make sure we have a bind IP/Port or we are not in a cluster
			hostname, err := utils.GetHostname()
			if err != nil {
				return false
			}
			if c.Nodes[hostname].IP == "" && c.Nodes[hostname].Port == "" {
				return false
			}
		}
		return true
	}
	return false
}

/**
Returns the interface the group is assigned to
*/
func (c *Config) GetGroupIface(node string, groupName string) string {
	for nodeName, n := range c.Nodes {
		if nodeName == node {
			for iface, groups := range n.IPGroups {
				for _, group := range groups {
					if group == groupName {
						return iface
					}
				}
			}
		}
	}
	return ""
}

/**
Instantiate, setup and return our Config
 */
func GetConfig() *Config {
	cfg := Config{}
	cfg.Load()
	cfg.Validate()
	return &cfg
}

/**
Returns the hostname for a node based of it's IP address
*/
func (c *Config) GetNodeHostnameByAddress(address string) (string, error) {
	for name, node := range c.Nodes {
		if node.IP == address {
			return name, nil
		}
	}
	return "", errors.New("unable to find node with IP address " + address)
}
<|endoftext|>


<commit_before>package gopdf

import (
	"fmt"
	"io"
)

type cacheContentLine struct {
	pageHeight float64
	x1         float64
	y1         float64
	x2         float64
	y2         float64
}

func (c *cacheContentLine) write(w io.Writer, protection *PDFProtection) error {
	fmt.Fprintf(w, "%0.2f %0.2f m %0.2f %0.2f l s\n", c.x1, c.pageHeight-c.y1, c.x2, c.pageHeight-c.y2)
	return nil
}
<commit_msg>When Draw Line, No Need Close Path, so use S NOT s<commit_after>package gopdf

import (
	"fmt"
	"io"
)

type cacheContentLine struct {
	pageHeight float64
	x1         float64
	y1         float64
	x2         float64
	y2         float64
}

func (c *cacheContentLine) write(w io.Writer, protection *PDFProtection) error {
	fmt.Fprintf(w, "%0.2f %0.2f m %0.2f %0.2f l S\n", c.x1, c.pageHeight-c.y1, c.x2, c.pageHeight-c.y2)
	return nil
}
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2017 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// Package templatehelper provides a func-map of common template functions
package templatehelper

import (
	"regexp"
	"strings"
	"text/template"
)

// FuncMap contains all the common string helpers
var (
	FuncMap = template.FuncMap{
		"Left": func(values ...interface{}) string {
			return values[0].(string)[:values[1].(int)]
		},
		"Matches": func(values ...interface{}) bool {
			ok, _ := regexp.MatchString(values[1].(string), values[0].(string))
			return ok
		},
		"Mid": func(values ...interface{}) string {
			if len(values) > 2 {
				return values[0].(string)[values[1].(int):values[2].(int)]
			}
			return values[0].(string)[values[1].(int):]
		},
		"Right": func(values ...interface{}) string {
			return values[0].(string)[len(values[0].(string))-values[1].(int):]
		},
		"Last": func(values ...interface{}) string {
			return values[0].([]string)[len(values[0].([]string))-1]
		},
		// strings functions
		"Compare":      strings.Compare, // 1.5+ only
		"Contains":     strings.Contains,
		"ContainsAny":  strings.ContainsAny,
		"Count":        strings.Count,
		"EqualFold":    strings.EqualFold,
		"HasPrefix":    strings.HasPrefix,
		"HasSuffix":    strings.HasSuffix,
		"Index":        strings.Index,
		"IndexAny":     strings.IndexAny,
		"Join":         strings.Join,
		"LastIndex":    strings.LastIndex,
		"LastIndexAny": strings.LastIndexAny,
		"Repeat":       strings.Repeat,
		"Replace":      strings.Replace,
		"Split":        strings.Split,
		"SplitAfter":   strings.SplitAfter,
		"SplitAfterN":  strings.SplitAfterN,
		"SplitN":       strings.SplitN,
		"Title":        strings.Title,
		"ToLower":      strings.ToLower,
		"ToTitle":      strings.ToTitle,
		"ToUpper":      strings.ToUpper,
		"Trim":         strings.Trim,
		"TrimLeft":     strings.TrimLeft,
		"TrimPrefix":   strings.TrimPrefix,
		"TrimRight":    strings.TrimRight,
		"TrimSpace":    strings.TrimSpace,
		"TrimSuffix":   strings.TrimSuffix,
	}
)
<commit_msg>Add JSON-Function for Templates (#242)<commit_after>/*
 *    Copyright (C) 2017 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// Package templatehelper provides a func-map of common template functions
package templatehelper

import (
	"encoding/json"
	htmlTemplate "html/template"
	"regexp"
	"strings"
	"text/template"
)

// FuncMap contains all the common string helpers
var (
	FuncMap = template.FuncMap{
		"JSON": func(values ...interface{}) htmlTemplate.JS {
			json, _ := json.Marshal(values)
			return htmlTemplate.JS(json)
		},
		"Left": func(values ...interface{}) string {
			return values[0].(string)[:values[1].(int)]
		},
		"Matches": func(values ...interface{}) bool {
			ok, _ := regexp.MatchString(values[1].(string), values[0].(string))
			return ok
		},
		"Mid": func(values ...interface{}) string {
			if len(values) > 2 {
				return values[0].(string)[values[1].(int):values[2].(int)]
			}
			return values[0].(string)[values[1].(int):]
		},
		"Right": func(values ...interface{}) string {
			return values[0].(string)[len(values[0].(string))-values[1].(int):]
		},
		"Last": func(values ...interface{}) string {
			return values[0].([]string)[len(values[0].([]string))-1]
		},
		// strings functions
		"Compare":      strings.Compare, // 1.5+ only
		"Contains":     strings.Contains,
		"ContainsAny":  strings.ContainsAny,
		"Count":        strings.Count,
		"EqualFold":    strings.EqualFold,
		"HasPrefix":    strings.HasPrefix,
		"HasSuffix":    strings.HasSuffix,
		"Index":        strings.Index,
		"IndexAny":     strings.IndexAny,
		"Join":         strings.Join,
		"LastIndex":    strings.LastIndex,
		"LastIndexAny": strings.LastIndexAny,
		"Repeat":       strings.Repeat,
		"Replace":      strings.Replace,
		"Split":        strings.Split,
		"SplitAfter":   strings.SplitAfter,
		"SplitAfterN":  strings.SplitAfterN,
		"SplitN":       strings.SplitN,
		"Title":        strings.Title,
		"ToLower":      strings.ToLower,
		"ToTitle":      strings.ToTitle,
		"ToUpper":      strings.ToUpper,
		"Trim":         strings.Trim,
		"TrimLeft":     strings.TrimLeft,
		"TrimPrefix":   strings.TrimPrefix,
		"TrimRight":    strings.TrimRight,
		"TrimSpace":    strings.TrimSpace,
		"TrimSuffix":   strings.TrimSuffix,
	}
)
<|endoftext|>


<commit_before>package application

import (
	"time"

	"github.com/marcusolsson/goddd/domain/cargo"
	"github.com/marcusolsson/goddd/domain/location"
	"github.com/marcusolsson/goddd/domain/voyage"
	"github.com/marcusolsson/goddd/infrastructure"
	. "gopkg.in/check.v1"
)

type stubEventHandler struct {
	handledEvents []interface{}
}

func (h *stubEventHandler) CargoWasHandled(event cargo.HandlingEvent) {
	h.handledEvents = append(h.handledEvents, event)
}

func (h *stubEventHandler) CargoWasMisdirected(c cargo.Cargo) {
	h.handledEvents = append(h.handledEvents, c)
}

func (h *stubEventHandler) CargoHasArrived(c cargo.Cargo) {
	h.handledEvents = append(h.handledEvents, c)
}

func (s *S) TestRegisterHandlingEvent(c *C) {

	var (
		eventHandler            = &stubEventHandler{make([]interface{}, 0)}
		cargoRepository         = infrastructure.NewInMemCargoRepository()
		handlingEventRepository = infrastructure.NewInMemHandlingEventRepository()
		handlingEventFactory    = cargo.HandlingEventFactory{
			CargoRepository: cargoRepository,
		}
	)

	var service HandlingEventService = &handlingEventService{
		handlingEventRepository: handlingEventRepository,
		handlingEventFactory:    handlingEventFactory,
		eventHandler:            eventHandler,
	}

	var (
		completionTime = time.Date(2015, time.November, 10, 23, 0, 0, 0, time.UTC)
		trackingId     = cargo.TrackingId("ABC123")
		voyageNumber   = voyage.VoyageNumber("CM001")
		unLocode       = location.Stockholm.UNLocode
		eventType      = cargo.Load
	)

	service.RegisterHandlingEvent(completionTime, trackingId, voyageNumber, unLocode, eventType)

	c.Check(len(eventHandler.handledEvents), Equals, 1)
}
<commit_msg>Fix failing test.<commit_after>package application

import (
	"time"

	"github.com/marcusolsson/goddd/domain/cargo"
	"github.com/marcusolsson/goddd/domain/location"
	"github.com/marcusolsson/goddd/domain/voyage"
	"github.com/marcusolsson/goddd/infrastructure"
	. "gopkg.in/check.v1"
)

type stubEventHandler struct {
	handledEvents []interface{}
}

func (h *stubEventHandler) CargoWasHandled(event cargo.HandlingEvent) {
	h.handledEvents = append(h.handledEvents, event)
}

func (h *stubEventHandler) CargoWasMisdirected(c cargo.Cargo) {
	h.handledEvents = append(h.handledEvents, c)
}

func (h *stubEventHandler) CargoHasArrived(c cargo.Cargo) {
	h.handledEvents = append(h.handledEvents, c)
}

func (s *S) TestRegisterHandlingEvent(c *C) {

	var (
		eventHandler            = &stubEventHandler{make([]interface{}, 0)}
		cargoRepository         = infrastructure.NewInMemCargoRepository()
		handlingEventRepository = infrastructure.NewInMemHandlingEventRepository()
		handlingEventFactory    = cargo.HandlingEventFactory{
			CargoRepository: cargoRepository,
		}
	)

	var service HandlingEventService = &handlingEventService{
		handlingEventRepository: handlingEventRepository,
		handlingEventFactory:    handlingEventFactory,
		eventHandler:            eventHandler,
	}

	var (
		completionTime = time.Date(2015, time.November, 10, 23, 0, 0, 0, time.UTC)
		trackingId     = cargo.TrackingId("ABC123")
		voyageNumber   = voyage.VoyageNumber("CM001")
		unLocode       = location.Stockholm.UNLocode
		eventType      = cargo.Load
	)

	cargoRepository.Store(*cargo.NewCargo(trackingId, cargo.RouteSpecification{}))

	err := service.RegisterHandlingEvent(completionTime, trackingId, voyageNumber, unLocode, eventType)

	c.Check(err, IsNil)
	c.Check(len(eventHandler.handledEvents), Equals, 1)
}
<|endoftext|>


<commit_before>// (C) 2017 Julian Andres Klode <jak@jak-linux.org>
// Licensed under the 2-Clause BSD license, see LICENSE for more information.

package capabilities

import (
	"fmt"
	"go/ast"
	"go/token"
	"strings"

	"github.com/julian-klode/lingolang/permission"
	"github.com/julian-klode/lingolang/permission/parser"
)

// Checker is a type that keeps shared state from multiple check
// passes. It must be created by NewChecker().
type Checker struct {
	path   string
	fset   *token.FileSet
	conf   *Config
	info   *Info
	pmap   map[ast.Node]permission.Permission
	passes []pass
	// Errors occured during capability checking.
	Errors []error
}

// NewChecker returns a new checker with the specified settings.
func NewChecker(conf *Config, info *Info, path string, fset *token.FileSet) *Checker {
	checker := &Checker{
		path: path,
		fset: fset,
		conf: conf,
		info: info,
		pmap: make(map[ast.Node]permission.Permission),
	}
	// Configure all passes here.
	checker.passes = []pass{
		assignPass{checker: checker},
	}
	return checker
}

// Files performs the checks the checker was set up for.
//
// Returns the first error, if any. More errors can be found in the Errors
// field.
func (c *Checker) Files(files []*ast.File) (err error) {
	defer func() {
		r := recover()
		if _, ok := r.(bailout); r != nil && !ok {
			panic(r)
		}
		if len(c.Errors) > 0 {
			err = c.Errors[0]
		}
	}()
	// Perform the type check.
	_, err = c.conf.Types.Check(c.path, c.fset, files, &c.info.Types)
	if err != nil {
		c.Errors = append(c.Errors, err)
		return
	}

	// Run the individual capability checking passes.
	// TODO: Error handling.
	for _, p := range c.passes {
		for _, f := range files {
			ast.Walk(p, f)
		}
	}

	if len(c.Errors) > 0 {
		return c.Errors[0]
	}

	return
}

// bailout is a type to throw to end capability checking.
type bailout struct{}

// errorf inserts a new error into the error log.
func (c *Checker) errorf(format string, a ...interface{}) {
	c.Errors = append(c.Errors, fmt.Errorf(format, a...))
	if len(c.Errors) >= 10 {
		panic(bailout{})
	}
}

// pass is simply an interface that extends the ast.Visitor interface, and
// represents the individual passes of the type checker. Each pass my store
// its own data in its associated object. The state is shared between files
// of the same package.
type pass interface {
	ast.Visitor
}

// assignPass assigns annotations in comments to their associated nodes.
type assignPass struct {
	checker *Checker
	// Comment map for the active file
	commentMap ast.CommentMap
}

func (p assignPass) Visit(node ast.Node) (w ast.Visitor) {
	// We are entering a file, get its comment map.
	if f, ok := node.(*ast.File); ok {
		p.commentMap = ast.NewCommentMap(p.checker.fset, f, f.Comments)
	}

	// If we don't have comments for this node, no need to continue
	cmtGrps, ok := p.commentMap[node]
	if !ok {
		return p
	}

	// Iterate through the comment groups to find a comment in a group that
	// starts with @cap and parse the specification there.
	for _, cmtGrp := range cmtGrps {
		for _, cmt := range cmtGrp.List {
			text := cmt.Text[2:]
			if strings.HasPrefix(strings.TrimSpace(text), "@cap") {
				cap := text[strings.Index(text, "@cap")+len("@cap"):]

				perm, err := parser.NewParser(cap).Parse()
				if err != nil {
					p.checker.errorf("%s: Cannot parse permission: %s", p.checker.fset.Position(cmt.Slash), err)
				}
				p.checker.pmap[node] = perm
			}
		}
	}

	return p
}
<commit_msg>capabilities: Checker: Directly use parent checker<commit_after>// (C) 2017 Julian Andres Klode <jak@jak-linux.org>
// Licensed under the 2-Clause BSD license, see LICENSE for more information.

package capabilities

import (
	"fmt"
	"go/ast"
	"go/token"
	"go/types"
	"strings"

	"github.com/julian-klode/lingolang/permission"
	"github.com/julian-klode/lingolang/permission/parser"
)

// Checker is a type that keeps shared state from multiple check
// passes. It must be created by NewChecker().
type Checker struct {
	parent *types.Checker
	path   string
	fset   *token.FileSet
	conf   *Config
	info   *Info
	pmap   map[ast.Node]permission.Permission
	passes []pass
	// Errors occured during capability checking.
	Errors []error
}

// NewChecker returns a new checker with the specified settings.
func NewChecker(conf *Config, info *Info, path string, fset *token.FileSet) *Checker {
	pkg := types.NewPackage(path, "")
	checker := &Checker{
		parent: types.NewChecker(&conf.Types, fset, pkg, &info.Types),
		path:   path,
		fset:   fset,
		conf:   conf,
		info:   info,
		pmap:   make(map[ast.Node]permission.Permission),
	}
	// Configure all passes here.
	checker.passes = []pass{
		assignPass{checker: checker},
	}
	return checker
}

// Files performs the checks the checker was set up for.
//
// Returns the first error, if any. More errors can be found in the Errors
// field.
func (c *Checker) Files(files []*ast.File) (err error) {
	defer func() {
		r := recover()
		if _, ok := r.(bailout); r != nil && !ok {
			panic(r)
		}
		if len(c.Errors) > 0 {
			err = c.Errors[0]
		}
	}()
	// Perform the type check.
	err = c.parent.Files(files)
	if err != nil {
		c.Errors = append(c.Errors, err)
		return
	}

	// Run the individual capability checking passes.
	// TODO: Error handling.
	for _, p := range c.passes {
		for _, f := range files {
			ast.Walk(p, f)
		}
	}

	if len(c.Errors) > 0 {
		return c.Errors[0]
	}

	return
}

// bailout is a type to throw to end capability checking.
type bailout struct{}

// errorf inserts a new error into the error log.
func (c *Checker) errorf(format string, a ...interface{}) {
	c.Errors = append(c.Errors, fmt.Errorf(format, a...))
	if len(c.Errors) >= 10 {
		panic(bailout{})
	}
}

// pass is simply an interface that extends the ast.Visitor interface, and
// represents the individual passes of the type checker. Each pass my store
// its own data in its associated object. The state is shared between files
// of the same package.
type pass interface {
	ast.Visitor
}

// assignPass assigns annotations in comments to their associated nodes.
type assignPass struct {
	checker *Checker
	// Comment map for the active file
	commentMap ast.CommentMap
}

func (p assignPass) Visit(node ast.Node) (w ast.Visitor) {
	// We are entering a file, get its comment map.
	if f, ok := node.(*ast.File); ok {
		p.commentMap = ast.NewCommentMap(p.checker.fset, f, f.Comments)
	}

	// If we don't have comments for this node, no need to continue
	cmtGrps, ok := p.commentMap[node]
	if !ok {
		return p
	}

	// Iterate through the comment groups to find a comment in a group that
	// starts with @cap and parse the specification there.
	for _, cmtGrp := range cmtGrps {
		for _, cmt := range cmtGrp.List {
			text := cmt.Text[2:]
			if strings.HasPrefix(strings.TrimSpace(text), "@cap") {
				cap := text[strings.Index(text, "@cap")+len("@cap"):]

				perm, err := parser.NewParser(cap).Parse()
				if err != nil {
					p.checker.errorf("%s: Cannot parse permission: %s", p.checker.fset.Position(cmt.Slash), err)
				}
				p.checker.pmap[node] = perm
			}
		}
	}

	return p
}
<|endoftext|>


<commit_before>package core

// CommentsByNormedExt determines what parts of a file we should lint -- e.g.,
// we only want to lint // or /* comments in a C++ file. Multiple syntaxes are
// mapped to a single extension (e.g., .java -> .c) because many languages use
// the same comment delimiters.
var CommentsByNormedExt = map[string]map[string]string{
	".c": {
		"inline":     `(//.+)|(/\*.+\*/)`,
		"blockStart": `(/\*.*)`,
		"blockEnd":   `(.*\*/)`,
	},
	".css": {
		"inline":     `(/\*.+\*/)`,
		"blockStart": `(/\*.*)`,
		"blockEnd":   `(.*\*/)`,
	},
	".rs": {
		"inline":     `(//.+)`,
		"blockStart": `$^`,
		"blockEnd":   `$^`,
	},
	".r": {
		"inline":     `(#.+)`,
		"blockStart": `$^`,
		"blockEnd":   `$^`,
	},
	".py": {
		"inline":     `(#.*)|('{3}.+'{3})|("{3}.+"{3})`,
		"blockStart": `(?m)^((?:\s{4,})?[r]?["']{3}.*)$`,
		"blockEnd":   `(.*["']{3})`,
	},
	".php": {
		"inline":     `(//.+)|(/\*.+\*/)|(#.+)`,
		"blockStart": `(/\*.*)`,
		"blockEnd":   `(.*\*/)`,
	},
	".lua": {
		"inline":     `(-- .+)`,
		"blockStart": `(-{2,3}\[\[.*)`,
		"blockEnd":   `(.*\]\])`,
	},
	".hs": {
		"inline":     `(-- .+)`,
		"blockStart": `(\{-.*)`,
		"blockEnd":   `(.*-\})`,
	},
	".rb": {
		"inline":     `(#.+)`,
		"blockStart": `(^=begin)`,
		"blockEnd":   `(^=end)`,
	},
}

// FormatByExtension associates a file extension with its "normed" extension
// and its format (markup, code or text).
var FormatByExtension = map[string][]string{
	`\.(?:[rc]?py[3w]?|[Ss][Cc]onstruct)$`:        {".py", "code"},
	`\.(?:adoc|asciidoc)$`:                        {".adoc", "markup"},
	`\.(?:cpp|cc|c|cp|cxx|c\+\+|h|hpp|h\+\+)$`:    {".c", "code"},
	`\.(?:cs|csx)$`:                               {".c", "code"},
	`\.(?:css)$`:                                  {".css", "code"},
	`\.(?:go)$`:                                   {".c", "code"},
	`\.(?:html|htm|shtml|xhtml)$`:                 {".html", "markup"},
	`\.(?:java|bsh)$`:                             {".c", "code"},
	`\.(?:js)$`:                                   {".c", "code"},
	`\.(?:lua)$`:                                  {".lua", "code"},
	`\.(?:md|mdown|markdown|markdn)$`:             {".md", "markup"},
	`\.(?:php)$`:                                  {".php", "code"},
	`\.(?:pl|pm|pod)$`:                            {".r", "code"},
	`\.(?:r|R)$`:                                  {".r", "code"},
	`\.(?:rs)$`:                                   {".rs", "code"},
	`\.(?:rst|rest)$`:                             {".rst", "markup"},
	`\.(?:swift)$`:                                {".c", "code"},
	`\.(?:txt)$`:                                  {".txt", "text"},
	`\.(?:rb|Gemfile|Rakefile|Brewfile|gemspec)$`: {".rb", "code"},
	`\.(?:sass|less)$`:                            {".c", "code"},
	`\.(?:scala|sbt)$`:                            {".c", "code"},
	`\.(?:hs)$`:                                   {".hs", "code"},
}
<commit_msg>feat: recognize `.asc` as AsciiDoc<commit_after>package core

// CommentsByNormedExt determines what parts of a file we should lint -- e.g.,
// we only want to lint // or /* comments in a C++ file. Multiple syntaxes are
// mapped to a single extension (e.g., .java -> .c) because many languages use
// the same comment delimiters.
var CommentsByNormedExt = map[string]map[string]string{
	".c": {
		"inline":     `(//.+)|(/\*.+\*/)`,
		"blockStart": `(/\*.*)`,
		"blockEnd":   `(.*\*/)`,
	},
	".css": {
		"inline":     `(/\*.+\*/)`,
		"blockStart": `(/\*.*)`,
		"blockEnd":   `(.*\*/)`,
	},
	".rs": {
		"inline":     `(//.+)`,
		"blockStart": `$^`,
		"blockEnd":   `$^`,
	},
	".r": {
		"inline":     `(#.+)`,
		"blockStart": `$^`,
		"blockEnd":   `$^`,
	},
	".py": {
		"inline":     `(#.*)|('{3}.+'{3})|("{3}.+"{3})`,
		"blockStart": `(?m)^((?:\s{4,})?[r]?["']{3}.*)$`,
		"blockEnd":   `(.*["']{3})`,
	},
	".php": {
		"inline":     `(//.+)|(/\*.+\*/)|(#.+)`,
		"blockStart": `(/\*.*)`,
		"blockEnd":   `(.*\*/)`,
	},
	".lua": {
		"inline":     `(-- .+)`,
		"blockStart": `(-{2,3}\[\[.*)`,
		"blockEnd":   `(.*\]\])`,
	},
	".hs": {
		"inline":     `(-- .+)`,
		"blockStart": `(\{-.*)`,
		"blockEnd":   `(.*-\})`,
	},
	".rb": {
		"inline":     `(#.+)`,
		"blockStart": `(^=begin)`,
		"blockEnd":   `(^=end)`,
	},
}

// FormatByExtension associates a file extension with its "normed" extension
// and its format (markup, code or text).
var FormatByExtension = map[string][]string{
	`\.(?:[rc]?py[3w]?|[Ss][Cc]onstruct)$`:        {".py", "code"},
	`\.(?:adoc|asciidoc|asc)$`:                    {".adoc", "markup"},
	`\.(?:cpp|cc|c|cp|cxx|c\+\+|h|hpp|h\+\+)$`:    {".c", "code"},
	`\.(?:cs|csx)$`:                               {".c", "code"},
	`\.(?:css)$`:                                  {".css", "code"},
	`\.(?:go)$`:                                   {".c", "code"},
	`\.(?:html|htm|shtml|xhtml)$`:                 {".html", "markup"},
	`\.(?:java|bsh)$`:                             {".c", "code"},
	`\.(?:js)$`:                                   {".c", "code"},
	`\.(?:lua)$`:                                  {".lua", "code"},
	`\.(?:md|mdown|markdown|markdn)$`:             {".md", "markup"},
	`\.(?:php)$`:                                  {".php", "code"},
	`\.(?:pl|pm|pod)$`:                            {".r", "code"},
	`\.(?:r|R)$`:                                  {".r", "code"},
	`\.(?:rs)$`:                                   {".rs", "code"},
	`\.(?:rst|rest)$`:                             {".rst", "markup"},
	`\.(?:swift)$`:                                {".c", "code"},
	`\.(?:txt)$`:                                  {".txt", "text"},
	`\.(?:rb|Gemfile|Rakefile|Brewfile|gemspec)$`: {".rb", "code"},
	`\.(?:sass|less)$`:                            {".c", "code"},
	`\.(?:scala|sbt)$`:                            {".c", "code"},
	`\.(?:hs)$`:                                   {".hs", "code"},
}
<|endoftext|>


<commit_before>// Package core implement the core interfaces and structs used by go-git
package core

import (
	"errors"
	"io"
)

var (
	ObjectNotFoundErr = errors.New("object not found")
)

// Object is a generic representation of any git object
type Object interface {
	Hash() Hash
	Type() ObjectType
	SetType(ObjectType)
	Size() int64
	SetSize(int64)
	Reader() io.Reader
	Writer() io.Writer
}

// ObjectStorage generic storage of objects
type ObjectStorage interface {
	New() (Object, error)
	Set(Object) (Hash, error)
	Get(Hash) (Object, error)
	Iter(ObjectType) ObjectIter
}

// ObjectType internal object type's
type ObjectType int8

const (
	CommitObject   ObjectType = 1
	TreeObject     ObjectType = 2
	BlobObject     ObjectType = 3
	TagObject      ObjectType = 4
	OFSDeltaObject ObjectType = 6
	REFDeltaObject ObjectType = 7
)

func (t ObjectType) String() string {
	switch t {
	case CommitObject:
		return "commit"
	case TreeObject:
		return "tree"
	case BlobObject:
		return "blob"
	case TagObject:
		return "tag"
	case OFSDeltaObject:
		return "ofs-delta"
	case REFDeltaObject:
		return "ref-delta"
	default:
		return "unknown"
	}
}

func (t ObjectType) Bytes() []byte {
	return []byte(t.String())
}

// ObjectIter is a generic closable interface for iterating over objects.
type ObjectIter interface {
	Next() (Object, error)
	Close()
}

// ObjectLookupIter implements ObjectIter. It iterates over a series of object
// hashes and yields their associated objects by retrieving each one from
// object storage. The retrievals are lazy and only occur when the iterator
// moves forward with a call to Next().
//
// The ObjectLookupIter must be closed with a call to Close() when it is no
// longer needed.
type ObjectLookupIter struct {
	storage ObjectStorage
	series  []Hash
	pos     int
}

// NewObjectLookupIter returns an object iterator given an object storage and
// a slice of object hashes.
func NewObjectLookupIter(storage ObjectStorage, series []Hash) *ObjectLookupIter {
	return &ObjectLookupIter{
		storage: storage,
		series:  series,
	}
}

// Next returns the next object from the iterator. If the iterator has reached
// the end it will return io.EOF as an error. If the object can't be found in
// the object storage, it will return ObjectNotFoundErr as an error. If the
// object is retreieved successfully error will be nil.
func (iter *ObjectLookupIter) Next() (Object, error) {
	if iter.pos >= len(iter.series) {
		return nil, io.EOF
	}
	hash := iter.series[iter.pos]
	obj, err := iter.storage.Get(hash)
	if err == nil {
		iter.pos++
	}
	return obj, err
}

// Close releases any resources used by the iterator.
func (iter *ObjectLookupIter) Close() {
	iter.pos = len(iter.series)
}

// ObjectSliceIter implements ObjectIter. It iterates over a series of objects
// stored in a slice and yields each one in turn when Next() is called.
//
// The ObjectSliceIter must be closed with a call to Close() when it is no
// longer needed.
type ObjectSliceIter struct {
	series []Object
	pos    int
}

// NewObjectSliceIter returns an object iterator for the given slice of objects.
func NewObjectSliceIter(series []Object) *ObjectSliceIter {
	return &ObjectSliceIter{
		series: series,
	}
}

// Next returns the next object from the iterator. If the iterator has reached
// the end it will return io.EOF as an error. If the object is retreieved
// successfully error will be nil.
func (iter *ObjectSliceIter) Next() (Object, error) {
	if iter.pos >= len(iter.series) {
		return nil, io.EOF
	}
	obj := iter.series[iter.pos]
	iter.pos++
	return obj, nil
}

// Close releases any resources used by the iterator.
func (iter *ObjectSliceIter) Close() {
	iter.pos = len(iter.series)
}
<commit_msg>Added ParseObjectType function to core<commit_after>// Package core implement the core interfaces and structs used by go-git
package core

import (
	"errors"
	"io"
)

var (
	ObjectNotFoundErr = errors.New("object not found")
)

// Object is a generic representation of any git object
type Object interface {
	Hash() Hash
	Type() ObjectType
	SetType(ObjectType)
	Size() int64
	SetSize(int64)
	Reader() io.Reader
	Writer() io.Writer
}

// ObjectStorage generic storage of objects
type ObjectStorage interface {
	New() (Object, error)
	Set(Object) (Hash, error)
	Get(Hash) (Object, error)
	Iter(ObjectType) ObjectIter
}

// ObjectType internal object type's
type ObjectType int8

const (
	CommitObject   ObjectType = 1
	TreeObject     ObjectType = 2
	BlobObject     ObjectType = 3
	TagObject      ObjectType = 4
	OFSDeltaObject ObjectType = 6
	REFDeltaObject ObjectType = 7
)

func (t ObjectType) String() string {
	switch t {
	case CommitObject:
		return "commit"
	case TreeObject:
		return "tree"
	case BlobObject:
		return "blob"
	case TagObject:
		return "tag"
	case OFSDeltaObject:
		return "ofs-delta"
	case REFDeltaObject:
		return "ref-delta"
	default:
		return "unknown"
	}
}

func (t ObjectType) Bytes() []byte {
	return []byte(t.String())
}

// ParseObjectType parses a string representation of ObjectType. It returns an
// error on parse failure.
func ParseObjectType(value string) (typ ObjectType, err error) {
	switch value {
	case "commit":
		typ = CommitObject
	case "tree":
		typ = TreeObject
	case "blob":
		typ = BlobObject
	case "tag":
		typ = TagObject
	case "ofs-delta":
		typ = OFSDeltaObject
	case "ref-delta":
		typ = REFDeltaObject
	default:
		err = errors.New("unable to parse object type")
	}
	return
}

// ObjectIter is a generic closable interface for iterating over objects.
type ObjectIter interface {
	Next() (Object, error)
	Close()
}

// ObjectLookupIter implements ObjectIter. It iterates over a series of object
// hashes and yields their associated objects by retrieving each one from
// object storage. The retrievals are lazy and only occur when the iterator
// moves forward with a call to Next().
//
// The ObjectLookupIter must be closed with a call to Close() when it is no
// longer needed.
type ObjectLookupIter struct {
	storage ObjectStorage
	series  []Hash
	pos     int
}

// NewObjectLookupIter returns an object iterator given an object storage and
// a slice of object hashes.
func NewObjectLookupIter(storage ObjectStorage, series []Hash) *ObjectLookupIter {
	return &ObjectLookupIter{
		storage: storage,
		series:  series,
	}
}

// Next returns the next object from the iterator. If the iterator has reached
// the end it will return io.EOF as an error. If the object can't be found in
// the object storage, it will return ObjectNotFoundErr as an error. If the
// object is retreieved successfully error will be nil.
func (iter *ObjectLookupIter) Next() (Object, error) {
	if iter.pos >= len(iter.series) {
		return nil, io.EOF
	}
	hash := iter.series[iter.pos]
	obj, err := iter.storage.Get(hash)
	if err == nil {
		iter.pos++
	}
	return obj, err
}

// Close releases any resources used by the iterator.
func (iter *ObjectLookupIter) Close() {
	iter.pos = len(iter.series)
}

// ObjectSliceIter implements ObjectIter. It iterates over a series of objects
// stored in a slice and yields each one in turn when Next() is called.
//
// The ObjectSliceIter must be closed with a call to Close() when it is no
// longer needed.
type ObjectSliceIter struct {
	series []Object
	pos    int
}

// NewObjectSliceIter returns an object iterator for the given slice of objects.
func NewObjectSliceIter(series []Object) *ObjectSliceIter {
	return &ObjectSliceIter{
		series: series,
	}
}

// Next returns the next object from the iterator. If the iterator has reached
// the end it will return io.EOF as an error. If the object is retreieved
// successfully error will be nil.
func (iter *ObjectSliceIter) Next() (Object, error) {
	if iter.pos >= len(iter.series) {
		return nil, io.EOF
	}
	obj := iter.series[iter.pos]
	iter.pos++
	return obj, nil
}

// Close releases any resources used by the iterator.
func (iter *ObjectSliceIter) Close() {
	iter.pos = len(iter.series)
}
<|endoftext|>


<commit_before>// Copyright 2013 Hui Chen
// Copyright 2016 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package core

import (
	// "fmt"
	"log"
	"sort"
	"sync"

	"github.com/go-ego/riot/types"
	"github.com/go-ego/riot/utils"
)

// Ranker ranker
type Ranker struct {
	idOnly bool

	lock struct {
		sync.RWMutex
		fields map[uint64]interface{}
		docs   map[uint64]bool
		// new
		content map[uint64]string
		attri   map[uint64]interface{}
	}

	initialized bool
}

// Init init ranker
func (ranker *Ranker) Init(onlyID ...bool) {
	if ranker.initialized == true {
		log.Fatal("The Ranker can not be initialized twice.")
	}
	ranker.initialized = true

	ranker.lock.fields = make(map[uint64]interface{})
	ranker.lock.docs = make(map[uint64]bool)

	if len(onlyID) > 0 {
		ranker.idOnly = onlyID[0]
	}

	if !ranker.idOnly {
		// new
		ranker.lock.content = make(map[uint64]string)
		ranker.lock.attri = make(map[uint64]interface{})
	}
}

// AddDoc add doc
// 给某个文档添加评分字段
func (ranker *Ranker) AddDoc(
	// docId uint64, fields interface{}, content string, attri interface{}) {
	docId uint64, fields interface{}, content ...interface{}) {
	if ranker.initialized == false {
		log.Fatal("The Ranker has not been initialized.")
	}

	ranker.lock.Lock()
	ranker.lock.fields[docId] = fields
	ranker.lock.docs[docId] = true

	if !ranker.idOnly {
		// new
		if len(content) > 0 {
			ranker.lock.content[docId] = content[0].(string)
		}

		if len(content) > 1 {
			ranker.lock.attri[docId] = content[1]
			// ranker.lock.attri[docId] = attri
		}
	}

	ranker.lock.Unlock()
}

// RemoveDoc 删除某个文档的评分字段
func (ranker *Ranker) RemoveDoc(docId uint64) {
	if ranker.initialized == false {
		log.Fatal("The Ranker has not been initialized.")
	}

	ranker.lock.Lock()
	delete(ranker.lock.fields, docId)
	delete(ranker.lock.docs, docId)

	if !ranker.idOnly {
		// new
		delete(ranker.lock.content, docId)
		delete(ranker.lock.attri, docId)
	}

	ranker.lock.Unlock()
}

// RankDocId rank docs by types.ScoredIDs
func (ranker *Ranker) RankDocId(docs []types.IndexedDoc,
	options types.RankOpts, countDocsOnly bool) (types.ScoredIDs, int) {
	var outputDocs types.ScoredIDs
	numDocs := 0

	for _, d := range docs {
		ranker.lock.RLock()
		// 判断 doc 是否存在
		if _, ok := ranker.lock.docs[d.DocId]; ok {

			fs := ranker.lock.fields[d.DocId]

			ranker.lock.RUnlock()
			// 计算评分并剔除没有分值的文档
			scores := options.ScoringCriteria.Score(d, fs)
			if len(scores) > 0 {
				if !countDocsOnly {
					outputDocs = append(outputDocs, types.ScoredID{
						DocId:            d.DocId,
						Scores:           scores,
						TokenSnippetLocs: d.TokenSnippetLocs,
						TokenLocs:        d.TokenLocs})
				}
				numDocs++
			}
		} else {
			ranker.lock.RUnlock()
		}
	}

	// 排序
	if !countDocsOnly {
		if options.ReverseOrder {
			sort.Sort(sort.Reverse(outputDocs))
		} else {
			sort.Sort(outputDocs)
		}
		// 当用户要求只返回部分结果时返回部分结果
		var start, end int
		if options.MaxOutputs != 0 {
			start = utils.MinInt(options.OutputOffset, len(outputDocs))
			end = utils.MinInt(options.OutputOffset+options.MaxOutputs, len(outputDocs))
		} else {
			start = utils.MinInt(options.OutputOffset, len(outputDocs))
			end = len(outputDocs)
		}
		return outputDocs[start:end], numDocs
	}

	return outputDocs, numDocs
}

// RankDocs rank docs by types.ScoredDocs
func (ranker *Ranker) RankDocs(docs []types.IndexedDoc,
	options types.RankOpts, countDocsOnly bool) (types.ScoredDocs, int) {
	var outputDocs types.ScoredDocs
	numDocs := 0

	for _, d := range docs {
		ranker.lock.RLock()
		// 判断 doc 是否存在
		if _, ok := ranker.lock.docs[d.DocId]; ok {

			fs := ranker.lock.fields[d.DocId]
			content := ranker.lock.content[d.DocId]
			attri := ranker.lock.attri[d.DocId]

			ranker.lock.RUnlock()
			// 计算评分并剔除没有分值的文档
			scores := options.ScoringCriteria.Score(d, fs)
			if len(scores) > 0 {
				if !countDocsOnly {
					outputDocs = append(outputDocs, types.ScoredDoc{
						DocId: d.DocId,
						// new
						Fields:  fs,
						Content: content,
						Attri:   attri,
						//
						Scores:           scores,
						TokenSnippetLocs: d.TokenSnippetLocs,
						TokenLocs:        d.TokenLocs})
				}
				numDocs++
			}
		} else {
			ranker.lock.RUnlock()
		}
	}

	// 排序
	if !countDocsOnly {
		if options.ReverseOrder {
			sort.Sort(sort.Reverse(outputDocs))
		} else {
			sort.Sort(outputDocs)
		}
		// 当用户要求只返回部分结果时返回部分结果
		var start, end int
		if options.MaxOutputs != 0 {
			start = utils.MinInt(options.OutputOffset, len(outputDocs))
			end = utils.MinInt(options.OutputOffset+options.MaxOutputs, len(outputDocs))
		} else {
			start = utils.MinInt(options.OutputOffset, len(outputDocs))
			end = len(outputDocs)
		}
		return outputDocs[start:end], numDocs
	}

	return outputDocs, numDocs
}

// Rank rank docs
// 给文档评分并排序
func (ranker *Ranker) Rank(docs []types.IndexedDoc,
	options types.RankOpts, countDocsOnly bool) (
	interface{}, int) {

	if ranker.initialized == false {
		log.Fatal("The Ranker has not been initialized.")
	}

	// 对每个文档评分
	if ranker.idOnly {
		outputDocs, numDocs := ranker.RankDocId(docs, options, countDocsOnly)
		return outputDocs, numDocs
	}

	outputDocs, numDocs := ranker.RankDocs(docs, options, countDocsOnly)
	return outputDocs, numDocs
}
<commit_msg>update ranker code optimize output offset [ci skip]<commit_after>// Copyright 2013 Hui Chen
// Copyright 2016 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package core

import (
	// "fmt"
	"log"
	"sort"
	"sync"

	"github.com/go-ego/riot/types"
	"github.com/go-ego/riot/utils"
)

// Ranker ranker
type Ranker struct {
	idOnly bool

	lock struct {
		sync.RWMutex
		fields map[uint64]interface{}
		docs   map[uint64]bool
		// new
		content map[uint64]string
		attri   map[uint64]interface{}
	}

	initialized bool
}

// Init init ranker
func (ranker *Ranker) Init(onlyID ...bool) {
	if ranker.initialized == true {
		log.Fatal("The Ranker can not be initialized twice.")
	}
	ranker.initialized = true

	ranker.lock.fields = make(map[uint64]interface{})
	ranker.lock.docs = make(map[uint64]bool)

	if len(onlyID) > 0 {
		ranker.idOnly = onlyID[0]
	}

	if !ranker.idOnly {
		// new
		ranker.lock.content = make(map[uint64]string)
		ranker.lock.attri = make(map[uint64]interface{})
	}
}

// AddDoc add doc
// 给某个文档添加评分字段
func (ranker *Ranker) AddDoc(
	// docId uint64, fields interface{}, content string, attri interface{}) {
	docId uint64, fields interface{}, content ...interface{}) {
	if ranker.initialized == false {
		log.Fatal("The Ranker has not been initialized.")
	}

	ranker.lock.Lock()
	ranker.lock.fields[docId] = fields
	ranker.lock.docs[docId] = true

	if !ranker.idOnly {
		// new
		if len(content) > 0 {
			ranker.lock.content[docId] = content[0].(string)
		}

		if len(content) > 1 {
			ranker.lock.attri[docId] = content[1]
			// ranker.lock.attri[docId] = attri
		}
	}

	ranker.lock.Unlock()
}

// RemoveDoc 删除某个文档的评分字段
func (ranker *Ranker) RemoveDoc(docId uint64) {
	if ranker.initialized == false {
		log.Fatal("The Ranker has not been initialized.")
	}

	ranker.lock.Lock()
	delete(ranker.lock.fields, docId)
	delete(ranker.lock.docs, docId)

	if !ranker.idOnly {
		// new
		delete(ranker.lock.content, docId)
		delete(ranker.lock.attri, docId)
	}

	ranker.lock.Unlock()
}

func maxOutput(options types.RankOpts, docsLen int) (int, int) {
	var start, end int
	if options.MaxOutputs != 0 {
		start = utils.MinInt(options.OutputOffset, docsLen)
		end = utils.MinInt(options.OutputOffset+options.MaxOutputs, docsLen)
	} else {
		start = utils.MinInt(options.OutputOffset, docsLen)
		end = docsLen
	}
	return start, end
}

// RankDocID rank docs by types.ScoredIDs
func (ranker *Ranker) RankDocID(docs []types.IndexedDoc,
	options types.RankOpts, countDocsOnly bool) (types.ScoredIDs, int) {

	var outputDocs types.ScoredIDs
	numDocs := 0

	for _, d := range docs {
		ranker.lock.RLock()
		// 判断 doc 是否存在
		if _, ok := ranker.lock.docs[d.DocId]; ok {

			fs := ranker.lock.fields[d.DocId]

			ranker.lock.RUnlock()
			// 计算评分并剔除没有分值的文档
			scores := options.ScoringCriteria.Score(d, fs)
			if len(scores) > 0 {
				if !countDocsOnly {
					outputDocs = append(outputDocs, types.ScoredID{
						DocId:            d.DocId,
						Scores:           scores,
						TokenSnippetLocs: d.TokenSnippetLocs,
						TokenLocs:        d.TokenLocs})
				}
				numDocs++
			}
		} else {
			ranker.lock.RUnlock()
		}
	}

	// 排序
	if !countDocsOnly {
		if options.ReverseOrder {
			sort.Sort(sort.Reverse(outputDocs))
		} else {
			sort.Sort(outputDocs)
		}
		// 当用户要求只返回部分结果时返回部分结果
		docsLen := len(outputDocs)
		start, end := maxOutput(options, docsLen)

		return outputDocs[start:end], numDocs
	}

	return outputDocs, numDocs
}

// RankDocs rank docs by types.ScoredDocs
func (ranker *Ranker) RankDocs(docs []types.IndexedDoc,
	options types.RankOpts, countDocsOnly bool) (types.ScoredDocs, int) {

	var outputDocs types.ScoredDocs
	numDocs := 0

	for _, d := range docs {
		ranker.lock.RLock()
		// 判断 doc 是否存在
		if _, ok := ranker.lock.docs[d.DocId]; ok {

			fs := ranker.lock.fields[d.DocId]
			content := ranker.lock.content[d.DocId]
			attri := ranker.lock.attri[d.DocId]

			ranker.lock.RUnlock()
			// 计算评分并剔除没有分值的文档
			scores := options.ScoringCriteria.Score(d, fs)
			if len(scores) > 0 {
				if !countDocsOnly {
					outputDocs = append(outputDocs, types.ScoredDoc{
						DocId: d.DocId,
						// new
						Fields:  fs,
						Content: content,
						Attri:   attri,
						//
						Scores:           scores,
						TokenSnippetLocs: d.TokenSnippetLocs,
						TokenLocs:        d.TokenLocs})
				}
				numDocs++
			}
		} else {
			ranker.lock.RUnlock()
		}
	}

	// 排序
	if !countDocsOnly {
		if options.ReverseOrder {
			sort.Sort(sort.Reverse(outputDocs))
		} else {
			sort.Sort(outputDocs)
		}
		// 当用户要求只返回部分结果时返回部分结果
		docsLen := len(outputDocs)
		start, end := maxOutput(options, docsLen)

		return outputDocs[start:end], numDocs
	}

	return outputDocs, numDocs
}

// Rank rank docs
// 给文档评分并排序
func (ranker *Ranker) Rank(docs []types.IndexedDoc,
	options types.RankOpts, countDocsOnly bool) (interface{}, int) {

	if ranker.initialized == false {
		log.Fatal("The Ranker has not been initialized.")
	}

	// 对每个文档评分
	if ranker.idOnly {
		outputDocs, numDocs := ranker.RankDocID(docs, options, countDocsOnly)
		return outputDocs, numDocs
	}

	outputDocs, numDocs := ranker.RankDocs(docs, options, countDocsOnly)
	return outputDocs, numDocs
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"time"
)

var restoreFlags = flag.NewFlagSet("restore", flag.ExitOnError)
var restoreForce = restoreFlags.Bool("f", false, "Overwrite existing")
var restoreVerbose = restoreFlags.Bool("v", false, "Verbose restore")
var restoreNoop = restoreFlags.Bool("n", false, "Noop")

func restoreFile(base, path string, data interface{}) error {
	u, err := url.Parse(base)
	if err != nil {
	log.Printf("Restoring %v", path)

	if *restoreNoop {
		return nil
	}

		log.Fatalf("Error parsing URL: %v", err)
	}

	fileMetaBytes, err := json.Marshal(data)
	if err != nil {
		return err
	}

	u.Path = fmt.Sprintf("/.cbfs/backup/restore/%v", path)
	res, err := http.Post(u.String(),
		"application/json",
		bytes.NewReader(fileMetaBytes))
	if err != nil {
		log.Fatalf("Error executing POST to %v - %v", u, err)
	}
	defer res.Body.Close()
	if res.StatusCode != 201 {
		log.Printf("restore error: %v", res.Status)
		io.Copy(os.Stderr, res.Body)
		return fmt.Errorf("HTTP Error restoring %v: %v", path, res.Status)
	}

	return nil
}

func restoreCommand(ustr string, args []string) {
	restoreFlags.Parse(args)

	if restoreFlags.NArg() < 1 {
		log.Fatalf("Filename is required")
	}
	fn := restoreFlags.Arg(0)

	start := time.Now()

	f, err := os.Open(fn)
	if err != nil {
		log.Fatalf("Error opening restore file: %v", err)
	}
	defer f.Close()
	gz, err := gzip.NewReader(f)
	if err != nil {
		log.Fatalf("Error uncompressing restore file: %v", err)
	}

	d := json.NewDecoder(gz)
	nfiles := 0
	done := false
	for !done {
		ob := struct {
			Path string
			Meta *json.RawMessage
		}{}

		err := d.Decode(&ob)
		switch err {
		case nil:
			nfiles++
			err := restoreFile(ustr, ob.Path, ob.Meta)
			if err != nil {
				log.Printf("Error restoring %v: %v",
					ob.Path, err)
			}
		case io.EOF:
			done = true
			break
		default:
			log.Fatalf("Error reading backup file: %v", err)
		}
	}

	log.Printf("Restored %v files in %v", nfiles, time.Since(start))
}
<commit_msg>Added regex partial restore.<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"time"
)

var restoreFlags = flag.NewFlagSet("restore", flag.ExitOnError)
var restoreForce = restoreFlags.Bool("f", false, "Overwrite existing")
var restoreNoop = restoreFlags.Bool("n", false, "Noop")
var restoreVerbose = restoreFlags.Bool("v", false, "Verbose restore")
var restorePat = restoreFlags.String("match", ".*", "Regex for paths to match")

func restoreFile(base, path string, data interface{}) error {
	log.Printf("Restoring %v", path)

	if *restoreNoop {
		return nil
	}

	u, err := url.Parse(base)
	if err != nil {
		log.Fatalf("Error parsing URL: %v", err)
	}

	fileMetaBytes, err := json.Marshal(data)
	if err != nil {
		return err
	}

	u.Path = fmt.Sprintf("/.cbfs/backup/restore/%v", path)
	res, err := http.Post(u.String(),
		"application/json",
		bytes.NewReader(fileMetaBytes))
	if err != nil {
		log.Fatalf("Error executing POST to %v - %v", u, err)
	}
	defer res.Body.Close()
	if res.StatusCode != 201 {
		log.Printf("restore error: %v", res.Status)
		io.Copy(os.Stderr, res.Body)
		return fmt.Errorf("HTTP Error restoring %v: %v", path, res.Status)
	}

	return nil
}

func restoreCommand(ustr string, args []string) {
	restoreFlags.Parse(args)

	regex, err := regexp.Compile(*restorePat)
	if err != nil {
		log.Fatalf("Error parsing match pattern: %v", err)
	}

	if restoreFlags.NArg() < 1 {
		log.Fatalf("Filename is required")
	}
	fn := restoreFlags.Arg(0)

	start := time.Now()

	f, err := os.Open(fn)
	if err != nil {
		log.Fatalf("Error opening restore file: %v", err)
	}
	defer f.Close()
	gz, err := gzip.NewReader(f)
	if err != nil {
		log.Fatalf("Error uncompressing restore file: %v", err)
	}

	d := json.NewDecoder(gz)
	nfiles := 0
	done := false
	for !done {
		ob := struct {
			Path string
			Meta *json.RawMessage
		}{}

		err := d.Decode(&ob)
		switch err {
		case nil:
			if !regex.MatchString(ob.Path) {
				// Skipping
				continue
			}
			nfiles++
			err := restoreFile(ustr, ob.Path, ob.Meta)
			if err != nil {
				log.Printf("Error restoring %v: %v",
					ob.Path, err)
			}
		case io.EOF:
			done = true
			break
		default:
			log.Fatalf("Error reading backup file: %v", err)
		}
	}

	log.Printf("Restored %v files in %v", nfiles, time.Since(start))
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"log"
	"reflect"
)

// NodeModuleRemoved represents a module that is no longer in the
// config.
type NodeModuleRemoved struct {
	PathValue []string
}

func (n *NodeModuleRemoved) Name() string {
	return fmt.Sprintf("%s (removed)", modulePrefixStr(n.PathValue))
}

// GraphNodeSubPath
func (n *NodeModuleRemoved) Path() []string {
	return n.PathValue
}

// GraphNodeEvalable
func (n *NodeModuleRemoved) EvalTree() EvalNode {
	return &EvalOpFilter{
		Ops: []walkOperation{walkRefresh, walkApply, walkDestroy},
		Node: &EvalDeleteModule{
			PathValue: n.PathValue,
		},
	}
}

// EvalDeleteModule is an EvalNode implementation that removes an empty module
// entry from the state.
type EvalDeleteModule struct {
	PathValue []string
}

func (n *EvalDeleteModule) Eval(ctx EvalContext) (interface{}, error) {
	state, lock := ctx.State()
	if state == nil {
		return nil, nil
	}

	// Get a write lock so we can access this instance
	lock.Lock()
	defer lock.Unlock()

	// Make sure we have a clean state
	// Destroyed resources aren't deleted, they're written with an ID of "".
	state.prune()

	// find the module and delete it
	for i, m := range state.Modules {
		if reflect.DeepEqual(m.Path, n.PathValue) {
			if !m.Empty() {
				// a targeted apply may leave module resources even without a config,
				// so just log this and return.
				log.Printf("[DEBUG] cannot remove module %s, not empty", modulePrefixStr(n.PathValue))
				break
			}
			state.Modules = append(state.Modules[:i], state.Modules[i+1:]...)
			break
		}
	}

	return nil, nil
}
<commit_msg>make NodeModuleRemoved a GraphNodeReferencer<commit_after>package terraform

import (
	"fmt"
	"log"
	"reflect"
)

// NodeModuleRemoved represents a module that is no longer in the
// config.
type NodeModuleRemoved struct {
	PathValue []string
}

func (n *NodeModuleRemoved) Name() string {
	return fmt.Sprintf("%s (removed)", modulePrefixStr(n.PathValue))
}

// GraphNodeSubPath
func (n *NodeModuleRemoved) Path() []string {
	return n.PathValue
}

// GraphNodeEvalable
func (n *NodeModuleRemoved) EvalTree() EvalNode {
	return &EvalOpFilter{
		Ops: []walkOperation{walkRefresh, walkApply, walkDestroy},
		Node: &EvalDeleteModule{
			PathValue: n.PathValue,
		},
	}
}

func (n *NodeModuleRemoved) ReferenceGlobal() bool {
	return true
}

func (n *NodeModuleRemoved) References() []string {
	return []string{modulePrefixStr(n.PathValue)}
}

// EvalDeleteModule is an EvalNode implementation that removes an empty module
// entry from the state.
type EvalDeleteModule struct {
	PathValue []string
}

func (n *EvalDeleteModule) Eval(ctx EvalContext) (interface{}, error) {
	state, lock := ctx.State()
	if state == nil {
		return nil, nil
	}

	// Get a write lock so we can access this instance
	lock.Lock()
	defer lock.Unlock()

	// Make sure we have a clean state
	// Destroyed resources aren't deleted, they're written with an ID of "".
	state.prune()

	// find the module and delete it
	for i, m := range state.Modules {
		if reflect.DeepEqual(m.Path, n.PathValue) {
			if !m.Empty() {
				// a targeted apply may leave module resources even without a config,
				// so just log this and return.
				log.Printf("[DEBUG] cannot remove module %s, not empty", modulePrefixStr(n.PathValue))
				break
			}
			state.Modules = append(state.Modules[:i], state.Modules[i+1:]...)
			break
		}
	}

	return nil, nil
}
<|endoftext|>


<commit_before>package sarama

import (
	"encoding/binary"

	"github.com/klauspost/crc32"
)

// crc32Field implements the pushEncoder and pushDecoder interfaces for calculating CRC32s.
type crc32Field struct {
	startOffset int
}

func (c *crc32Field) saveOffset(in int) {
	c.startOffset = in
}

func (c *crc32Field) reserveLength() int {
	return 4
}

func (c *crc32Field) run(curOffset int, buf []byte) error {
	crc := crc32.ChecksumIEEE(buf[c.startOffset+4 : curOffset])
	binary.BigEndian.PutUint32(buf[c.startOffset:], crc)
	return nil
}

func (c *crc32Field) check(curOffset int, buf []byte) error {
	crc := crc32.ChecksumIEEE(buf[c.startOffset+4 : curOffset])

	if crc != binary.BigEndian.Uint32(buf[c.startOffset:]) {
		return PacketDecodingError{"CRC didn't match"}
	}

	return nil
}
<commit_msg>Go back to hash/crc32<commit_after>package sarama

import (
	"encoding/binary"
	"hash/crc32"
)

// crc32Field implements the pushEncoder and pushDecoder interfaces for calculating CRC32s.
type crc32Field struct {
	startOffset int
}

func (c *crc32Field) saveOffset(in int) {
	c.startOffset = in
}

func (c *crc32Field) reserveLength() int {
	return 4
}

func (c *crc32Field) run(curOffset int, buf []byte) error {
	crc := crc32.ChecksumIEEE(buf[c.startOffset+4 : curOffset])
	binary.BigEndian.PutUint32(buf[c.startOffset:], crc)
	return nil
}

func (c *crc32Field) check(curOffset int, buf []byte) error {
	crc := crc32.ChecksumIEEE(buf[c.startOffset+4 : curOffset])

	if crc != binary.BigEndian.Uint32(buf[c.startOffset:]) {
		return PacketDecodingError{"CRC didn't match"}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// Generate a self-signed X.509 certificate for a TLS server. Outputs to
// 'cert.pem' and 'key.pem' and will overwrite existing files.

package main

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"flag"
	"fmt"
	"log"
	"math/big"
	"net"
	"os"
	"strings"
	"time"
)

var (
	host       = flag.String("host", "", "Comma-separated hostnames and IPs to generate a certificate for")
	validFrom  = flag.String("start-date", "", "Creation date formatted as Jan 1 15:04:05 2011")
	validFor   = flag.Duration("duration", 365*24*time.Hour, "Duration that certificate is valid for")
	isCA       = flag.Bool("ca", false, "whether this cert should be its own Certificate Authority")
	rsaBits    = flag.Int("rsa-bits", 2048, "Size of RSA key to generate. Ignored if --ecdsa-curve is set")
	ecdsaCurve = flag.String("ecdsa-curve", "", "ECDSA curve to use to generate a key. Valid values are P224, P256, P384, P521")
)

func publicKey(priv interface{}) interface{} {
	switch k := priv.(type) {
		case *rsa.PrivateKey:
				return &k.PublicKey
				case *ecdsa.PrivateKey:
						return &k.PublicKey
						default:
								return nil
								}
							}

						func pemBlockForKey(priv interface{}) *pem.Block {
							switch k := priv.(type) {
								case *rsa.PrivateKey:
										return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}
										case *ecdsa.PrivateKey:
												b, err := x509.MarshalECPrivateKey(k)
													if err != nil {
																fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err)
																		os.Exit(2)
																			}
																				return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
																				default:
																						return nil
																						}
																					}

																				func main() {
																					flag.Parse()

																					if len(*host) == 0 {
																							log.Fatalf("Missing required --host parameter")
																							}

																							var priv interface{}
																							var err error
																							switch *ecdsaCurve {
																								case "":
																										priv, err = rsa.GenerateKey(rand.Reader, *rsaBits)
																										case "P224":
																												priv, err = ecdsa.GenerateKey(elliptic.P224(), rand.Reader)
																												case "P256":
																														priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
																														case "P384":
																																priv, err = ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
																																case "P521":
																																		priv, err = ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
																																		default:
																																				fmt.Fprintf(os.Stderr, "Unrecognized elliptic curve: %q", *ecdsaCurve)
																																					os.Exit(1)
																																					}
																																					if err != nil {
																																							log.Fatalf("failed to generate private key: %s", err)
																																							}

																																							var notBefore time.Time
																																							if len(*validFrom) == 0 {
																																									notBefore = time.Now()
																																									} else {
																																											notBefore, err = time.Parse("Jan 2 15:04:05 2006", *validFrom)
																																												if err != nil {
																																															fmt.Fprintf(os.Stderr, "Failed to parse creation date: %s\n", err)
																																																	os.Exit(1)
																																																		}
																																																		}

																																																		notAfter := notBefore.Add(*validFor)

																																																		serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
																																																		serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
																																																		if err != nil {
																																																				log.Fatalf("failed to generate serial number: %s", err)
																																																				}

																																																				template := x509.Certificate{
																																																						SerialNumber: serialNumber,
																																																							Subject: pkix.Name{
																																																										Organization: []string{"Acme Co"},
																																																											},
																																																												NotBefore: notBefore,
																																																													NotAfter:  notAfter,

																																																														KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
																																																															ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
																																																																BasicConstraintsValid: true,
																																																																}

																																																																hosts := strings.Split(*host, ",")
																																																																for _, h := range hosts {
																																																																		if ip := net.ParseIP(h); ip != nil {
																																																																					template.IPAddresses = append(template.IPAddresses, ip)
																																																																						} else {
																																																																									template.DNSNames = append(template.DNSNames, h)
																																																																										}
																																																																										}

																																																																										if *isCA {
																																																																												template.IsCA = true
																																																																													template.KeyUsage |= x509.KeyUsageCertSign
																																																																													}

																																																																													derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv)
																																																																													if err != nil {
																																																																															log.Fatalf("Failed to create certificate: %s", err)
																																																																															}

																																																																															certOut, err := os.Create("cert.pem")
																																																																															if err != nil {
																																																																																log.Fatalf("failed to open cert.pem for writing: %s", err)
																																																																																}
																																																																																pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
																																																																																certOut.Close()
																																																																																log.Print("written cert.pem\n")

																																																																																keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
																																																																																if err != nil {
																																																																																log.Print("failed to open key.pem for writing:", err)
																																																																																return
																																																																																}
																																																																																pem.Encode(keyOut, pemBlockForKey(priv))
																																																																																keyOut.Close()
																																																																																log.Print("written key.pem\n")
																																																																																}
<commit_msg>fix indent of generate_cert.go<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// Generate a self-signed X.509 certificate for a TLS server. Outputs to
// 'cert.pem' and 'key.pem' and will overwrite existing files.

package main

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"flag"
	"fmt"
	"log"
	"math/big"
	"net"
	"os"
	"strings"
	"time"
)

var (
	host       = flag.String("host", "", "Comma-separated hostnames and IPs to generate a certificate for")
	validFrom  = flag.String("start-date", "", "Creation date formatted as Jan 1 15:04:05 2011")
	validFor   = flag.Duration("duration", 365*24*time.Hour, "Duration that certificate is valid for")
	isCA       = flag.Bool("ca", false, "whether this cert should be its own Certificate Authority")
	rsaBits    = flag.Int("rsa-bits", 2048, "Size of RSA key to generate. Ignored if --ecdsa-curve is set")
	ecdsaCurve = flag.String("ecdsa-curve", "", "ECDSA curve to use to generate a key. Valid values are P224, P256, P384, P521")
)

func publicKey(priv interface{}) interface{} {
	switch k := priv.(type) {
		case *rsa.PrivateKey:
			return &k.PublicKey
		case *ecdsa.PrivateKey:
			return &k.PublicKey
		default:
			return nil
	}
}

func pemBlockForKey(priv interface{}) *pem.Block {
	switch k := priv.(type) {
		case *rsa.PrivateKey:
			return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}
		case *ecdsa.PrivateKey:
			b, err := x509.MarshalECPrivateKey(k)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err)
				os.Exit(2)
			}
			return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
		default:
			return nil
	}
}

func main() {
	flag.Parse()
	if len(*host) == 0 {
		log.Fatalf("Missing required --host parameter")
	}

	var priv interface{}
	var err error
	switch *ecdsaCurve {
		case "":
			priv, err = rsa.GenerateKey(rand.Reader, *rsaBits)
		case "P224":
			priv, err = ecdsa.GenerateKey(elliptic.P224(), rand.Reader)
		case "P256":
			priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
		case "P384":
			priv, err = ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
		case "P521":
			priv, err = ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
		default:
			fmt.Fprintf(os.Stderr, "Unrecognized elliptic curve: %q", *ecdsaCurve)
			os.Exit(1)
	}
	if err != nil {
		log.Fatalf("failed to generate private key: %s", err)
	}
	var notBefore time.Time
	if len(*validFrom) == 0 {
		notBefore = time.Now()
	} else {
		notBefore, err = time.Parse("Jan 2 15:04:05 2006", *validFrom)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to parse creation date: %s\n", err)
			os.Exit(1)
		}
	}
	notAfter := notBefore.Add(*validFor)
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		log.Fatalf("failed to generate serial number: %s", err)
	}
	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{"Acme Co"},
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}
	hosts := strings.Split(*host, ",")
	for _, h := range hosts {
		if ip := net.ParseIP(h); ip != nil {
			template.IPAddresses = append(template.IPAddresses, ip)
		} else {
			template.DNSNames = append(template.DNSNames, h)
		}
	}

	if *isCA {
		template.IsCA = true
		template.KeyUsage |= x509.KeyUsageCertSign
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv)
	if err != nil {
		log.Fatalf("Failed to create certificate: %s", err)
	}

	certOut, err := os.Create("cert.pem")
	if err != nil {
		log.Fatalf("failed to open cert.pem for writing: %s", err)
	}
	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()
	log.Print("written cert.pem\n")

	keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		log.Print("failed to open key.pem for writing:", err)
		return
	}
	pem.Encode(keyOut, pemBlockForKey(priv))
	keyOut.Close()
	log.Print("written key.pem\n")
}
<|endoftext|>


<commit_before>package main

//
// Prepare PG Database:
//
// $ createdb testdb
// $ psql testdb
// testdb=#
// CREATE TABLE test
//     (id int, call_uuid text, dst text, callerid_name text, callerid_num text, duration int,
//      data jsonb, created timestamp );
//
// INSERT INTO cdr VALUES ("Outbound Call","123555555","123555555","default","2015-01-14 17:58:01","2015-01-14 17:58:01","2015-01-14 17:58:06",5,5,"NORMAL_CLEARING","2bbe83f7-5111-4b5b-9626-c5154608d4ee","","")
//

import (
	log "github.com/Sirupsen/logrus"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"
)

// Wait time for results in goroutine
const WAITTIME = 60

// RunFetcher fetchs non imported CDRs from the local datasource (SQLite)
func RunFetcher(config Config, chanRes chan map[int][]string, chanSync chan bool) {
	f := new(SQLFetcher)
	if config.StorageSource == "sqlite3" || config.StorageSource == "mysql" {
		f.Init(config.DBFile, config.DBTable, config.MaxFetchBatch, config.CDRFields, config.DBFlagField,
			config.StorageSource, config.DBDNS)
		for {
			log.Info("RunFetcher waiting on chanSync before fetching")
			<-chanSync
			// Fetch CDRs from SQLite
			err := f.Fetch()
			if err != nil {
				log.Error(err.Error())
			}
			if err == nil && f.results != nil {
				chanRes <- f.results
			}
			// Wait x seconds between each DB fetch | Heartbeat
			log.Info("RunFetcher sleeps for " + strconv.Itoa(config.Heartbeat) + " seconds!")
			time.Sleep(time.Second * time.Duration(config.Heartbeat))
		}
	}
}

// DispatchPush is a dispacher to push the results to the right storage
func DispatchPush(config Config, results map[int][]string) {
	if config.StorageDestination == "postgres" {
		// Push CDRs to PostgreSQL
		pc := new(PGPusher)
		pc.Init(config.PGDatasourcename, config.CDRFields, config.SwitchIP, config.CDRSourceType, config.TableDestination)
		err := pc.Push(results)
		if err != nil {
			log.Error(err.Error())
		}
	} else if config.StorageDestination == "riak" {
		// Push CDRs to Riak
		rc := new(RiakPusher)
		rc.Init(config.RiakConnect, config.CDRFields, config.SwitchIP, config.CDRSourceType, config.RiakBucket)
		err := rc.Push(results)
		if err != nil {
			log.Error(err.Error())
		}
	}
}

// PushResult is goroutine that will push CDRs to storage when receiving from results
// on channel chanRes
func PushResult(config Config, chanRes chan map[int][]string, chanSync chan bool) {
	for {
		log.Debug("PushResult sending chanSync to start Fetching")
		// Send signal to go_fetch to fetch
		chanSync <- true
		// waiting for CDRs on channel
		select {
		case results := <-chanRes:
			// Send results to storage engine
			DispatchPush(config, results)
		case <-time.After(time.Second * WAITTIME):
			log.Debug("Nothing received yet...")
		}
	}
}

// PopulateFakeCDR is provided for tests purpose, it takes care of populating the
// SQlite database with fake CDRs at interval of time.
func PopulateFakeCDR(config Config) error {
	if config.FakeCDR != "yes" {
		return nil
	}
	// Heartbeat time for goPopulateFakeCDRs
	intval_time := 1
	for {
		// Wait x seconds when inserting fake CDRs
		log.Info("goPopulateFakeCDRs sleeps for " + strconv.Itoa(intval_time) + " seconds!")
		time.Sleep(time.Second * time.Duration(intval_time))
		GenerateCDR(config.DBFile, config.FakeAmountCDR)
	}
}

// RunApp is the core function of the service it launchs the different goroutines
// that will fetch and push
func RunApp() (string, error) {
	// if err := LoadConfig(defaultConf); err != nil {
	if err := LoadConfig(prodConf); err != nil {
		log.Error(err.Error())
		return "", err
	}
	if err := ValidateConfig(config); err != nil {
		panic(err)
	}

	chanSync := make(chan bool, 1)
	chanRes := make(chan map[int][]string, 1)

	// Start the coroutines
	go RunFetcher(config, chanRes, chanSync)
	go PushResult(config, chanRes, chanSync)
	go PopulateFakeCDR(config)

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM)

	// loop work cycle which listen for command or interrupt
	// by system signal
	for {
		select {
		case killSignal := <-interrupt:
			log.Warn("Got signal:", killSignal)
			if killSignal == os.Interrupt {
				return "Service was interruped by system signal", nil
			}
			return "Service was killed", nil
		}
	}
}

func main() {
	// Log as JSON instead of the default ASCII formatter.
	// log.SetFormatter(&log.JSONFormatter{})

	// Use the Airbrake hook to report errors that have Error severity or above to
	// an exception tracker. You can create custom hooks, see the Hooks section.
	// log.AddHook(&logrus_airbrake.AirbrakeHook{})

	setlogfile := false
	if setlogfile {
		// backendlog := logging.NewLogBackend(os.Stderr, "", 0)
		f, err := os.OpenFile("cdr-pusher.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			panic(err.Error())
		}
		defer f.Close()
		// Output to stderr instead of stdout, could also be a file.
		log.SetOutput(f)
	} else {
		log.SetOutput(os.Stderr)
	}

	// Only log the warning severity or above.
	// log.SetLevel(log.WarnLevel)
	// log.SetLevel(log.InfoLevel)
	log.SetLevel(log.DebugLevel)

	log.Info("StartTime: " + time.Now().Format("Mon Jan _2 2006 15:04:05"))
	RunApp()
	log.Info("StopTime: " + time.Now().Format("Mon Jan _2 2006 15:04:05"))
}
<commit_msg>update loglevel<commit_after>package main

//
// Prepare PG Database:
//
// $ createdb testdb
// $ psql testdb
// testdb=#
// CREATE TABLE test
//     (id int, call_uuid text, dst text, callerid_name text, callerid_num text, duration int,
//      data jsonb, created timestamp );
//
// INSERT INTO cdr VALUES ("Outbound Call","123555555","123555555","default","2015-01-14 17:58:01","2015-01-14 17:58:01","2015-01-14 17:58:06",5,5,"NORMAL_CLEARING","2bbe83f7-5111-4b5b-9626-c5154608d4ee","","")
//

import (
	log "github.com/Sirupsen/logrus"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"
)

// Wait time for results in goroutine
const WAITTIME = 60

// RunFetcher fetchs non imported CDRs from the local datasource (SQLite)
func RunFetcher(config Config, chanRes chan map[int][]string, chanSync chan bool) {
	f := new(SQLFetcher)
	if config.StorageSource == "sqlite3" || config.StorageSource == "mysql" {
		f.Init(config.DBFile, config.DBTable, config.MaxFetchBatch, config.CDRFields, config.DBFlagField,
			config.StorageSource, config.DBDNS)
		for {
			log.Info("RunFetcher waiting on chanSync before fetching")
			<-chanSync
			// Fetch CDRs from SQLite
			err := f.Fetch()
			if err != nil {
				log.Error(err.Error())
			}
			if err == nil && f.results != nil {
				chanRes <- f.results
			}
			// Wait x seconds between each DB fetch | Heartbeat
			log.Info("RunFetcher sleeps for " + strconv.Itoa(config.Heartbeat) + " seconds!")
			time.Sleep(time.Second * time.Duration(config.Heartbeat))
		}
	}
}

// DispatchPush is a dispacher to push the results to the right storage
func DispatchPush(config Config, results map[int][]string) {
	if config.StorageDestination == "postgres" {
		// Push CDRs to PostgreSQL
		pc := new(PGPusher)
		pc.Init(config.PGDatasourcename, config.CDRFields, config.SwitchIP, config.CDRSourceType, config.TableDestination)
		err := pc.Push(results)
		if err != nil {
			log.Error(err.Error())
		}
	} else if config.StorageDestination == "riak" {
		// Push CDRs to Riak
		rc := new(RiakPusher)
		rc.Init(config.RiakConnect, config.CDRFields, config.SwitchIP, config.CDRSourceType, config.RiakBucket)
		err := rc.Push(results)
		if err != nil {
			log.Error(err.Error())
		}
	}
}

// PushResult is goroutine that will push CDRs to storage when receiving from results
// on channel chanRes
func PushResult(config Config, chanRes chan map[int][]string, chanSync chan bool) {
	for {
		log.Debug("PushResult sending chanSync to start Fetching")
		// Send signal to go_fetch to fetch
		chanSync <- true
		// waiting for CDRs on channel
		select {
		case results := <-chanRes:
			// Send results to storage engine
			DispatchPush(config, results)
		case <-time.After(time.Second * WAITTIME):
			log.Debug("Nothing received yet...")
		}
	}
}

// PopulateFakeCDR is provided for tests purpose, it takes care of populating the
// SQlite database with fake CDRs at interval of time.
func PopulateFakeCDR(config Config) error {
	if config.FakeCDR != "yes" {
		return nil
	}
	// Heartbeat time for goPopulateFakeCDRs
	intval_time := 1
	for {
		// Wait x seconds when inserting fake CDRs
		log.Info("goPopulateFakeCDRs sleeps for " + strconv.Itoa(intval_time) + " seconds!")
		time.Sleep(time.Second * time.Duration(intval_time))
		GenerateCDR(config.DBFile, config.FakeAmountCDR)
	}
}

// RunApp is the core function of the service it launchs the different goroutines
// that will fetch and push
func RunApp() (string, error) {
	// if err := LoadConfig(defaultConf); err != nil {
	if err := LoadConfig(prodConf); err != nil {
		log.Error(err.Error())
		return "", err
	}
	if err := ValidateConfig(config); err != nil {
		panic(err)
	}

	chanSync := make(chan bool, 1)
	chanRes := make(chan map[int][]string, 1)

	// Start the coroutines
	go RunFetcher(config, chanRes, chanSync)
	go PushResult(config, chanRes, chanSync)
	go PopulateFakeCDR(config)

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM)

	// loop work cycle which listen for command or interrupt
	// by system signal
	for {
		select {
		case killSignal := <-interrupt:
			log.Warn("Got signal:", killSignal)
			if killSignal == os.Interrupt {
				return "Service was interruped by system signal", nil
			}
			return "Service was killed", nil
		}
	}
}

func main() {
	// Log as JSON instead of the default ASCII formatter.
	// log.SetFormatter(&log.JSONFormatter{})

	// Use the Airbrake hook to report errors that have Error severity or above to
	// an exception tracker. You can create custom hooks, see the Hooks section.
	// log.AddHook(&logrus_airbrake.AirbrakeHook{})

	setlogfile := false
	if setlogfile {
		// backendlog := logging.NewLogBackend(os.Stderr, "", 0)
		f, err := os.OpenFile("cdr-pusher.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			panic(err.Error())
		}
		defer f.Close()
		// Output to stderr instead of stdout, could also be a file.
		log.SetOutput(f)
	} else {
		log.SetOutput(os.Stderr)
	}

	// Only log the warning severity or above.
	// log.SetLevel(log.WarnLevel)
	log.SetLevel(log.InfoLevel)
	// log.SetLevel(log.DebugLevel)

	log.Info("StartTime: " + time.Now().Format("Mon Jan _2 2006 15:04:05"))
	RunApp()
	log.Info("StopTime: " + time.Now().Format("Mon Jan _2 2006 15:04:05"))
}
<|endoftext|>


<commit_before>package here

// Version of here
const Version = "v0.1.1"
<commit_msg>version bump: v0.1.2<commit_after>package here

// Version of here
const Version = "v0.1.2"
<|endoftext|>


<commit_before>package main

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string
var GitDescribe string

// The main version number that is being run at the moment.
const Version = "0.4.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"
<commit_msg>bump version<commit_after>package main

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string
var GitDescribe string

// The main version number that is being run at the moment.
const Version = "0.4.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "rc1"
<|endoftext|>