<commit_before>package lockbox

import (
	"crypto/rand"

	"encoding/pem"
	"io"
	"testing"
)

var (
	enc *Encryptor
	dec *Decryptor

	chunks = map[string][]byte{
		"128B":  mustRandomBytes(128),
		"4KB":   mustRandomBytes(4096),
		"64KB":  mustRandomBytes(65536),
		"512KB": mustRandomBytes(524288),
		"1MB":   mustRandomBytes(1048576),
		"4MB":   mustRandomBytes(4194304),
	}
)

func init() {
	ekey, dkey, err := GenerateKey(rand.Reader)
	if err != nil {
		panic(err)
	}

	eb, _ := pem.Decode(ekey)
	db, _ := pem.Decode(dkey)

	enc, err = NewEncryptor(eb)
	if err != nil {
		panic(err)
	}

	dec, err = NewDecryptor(db)
	if err != nil {
		panic(err)
	}
}

func BenchmarkLockboxEncrypt128B(b *testing.B)  { lbEncrypt("128B", b) }
func BenchmarkLockboxEncrypt4KB(b *testing.B)   { lbEncrypt("4KB", b) }
func BenchmarkLockboxEncrypt64KB(b *testing.B)  { lbEncrypt("64KB", b) }
func BenchmarkLockboxEncrypt512KB(b *testing.B) { lbEncrypt("512KB", b) }
func BenchmarkLockboxEncrypt1MB(b *testing.B)   { lbEncrypt("1MB", b) }
func BenchmarkLockboxEncrypt4MB(b *testing.B)   { lbEncrypt("4MB", b) }

func lbEncrypt(key string, b *testing.B) {
	data, ok := chunks[key]
	if !ok {
		b.Fatalf("missing chunk %s", key)
	}
	b.SetBytes(int64(len(data)))

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		enc.Encrypt(data)
	}
}

func BenchmarkLockboxDecrypt128(b *testing.B)   { lbDecrypt("128B", b) }
func BenchmarkLockboxDecrypt4096(b *testing.B)  { lbDecrypt("4KB", b) }
func BenchmarkLockboxDecrypt64KB(b *testing.B)  { lbDecrypt("64KB", b) }
func BenchmarkLockboxDecrypt512KB(b *testing.B) { lbDecrypt("512KB", b) }
func BenchmarkLockboxDecrypt1MB(b *testing.B)   { lbDecrypt("1MB", b) }
func BenchmarkLockboxDecrypt4MB(b *testing.B)   { lbDecrypt("4MB", b) }

func lbDecrypt(key string, b *testing.B) {
	chunk, ok := chunks[key]
	if !ok {
		b.Fatalf("missing chunk %s", key)
	}
	data, err := enc.Encrypt(chunk)
	if err != nil {
		panic(err)
	}
	b.SetBytes(int64(len(data)))

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		dec.Decrypt(data)
	}
}

func mustRandomBytes(s int) []byte {
	buf := make([]byte, s)
	if _, err := io.ReadAtLeast(rand.Reader, buf, s); err != nil {
		panic(err)
	}
	return buf
}
<commit_msg>fix decrypt benchmark byte count<commit_after>package lockbox

import (
	"crypto/rand"

	"encoding/pem"
	"io"
	"testing"
)

var (
	enc *Encryptor
	dec *Decryptor

	chunks = map[string][]byte{
		"128B":  mustRandomBytes(128),
		"4KB":   mustRandomBytes(4096),
		"64KB":  mustRandomBytes(65536),
		"512KB": mustRandomBytes(524288),
		"1MB":   mustRandomBytes(1048576),
		"4MB":   mustRandomBytes(4194304),
	}
)

func init() {
	ekey, dkey, err := GenerateKey(rand.Reader)
	if err != nil {
		panic(err)
	}

	eb, _ := pem.Decode(ekey)
	db, _ := pem.Decode(dkey)

	enc, err = NewEncryptor(eb)
	if err != nil {
		panic(err)
	}

	dec, err = NewDecryptor(db)
	if err != nil {
		panic(err)
	}
}

func BenchmarkLockboxEncrypt128B(b *testing.B)  { lbEncrypt("128B", b) }
func BenchmarkLockboxEncrypt4KB(b *testing.B)   { lbEncrypt("4KB", b) }
func BenchmarkLockboxEncrypt64KB(b *testing.B)  { lbEncrypt("64KB", b) }
func BenchmarkLockboxEncrypt512KB(b *testing.B) { lbEncrypt("512KB", b) }
func BenchmarkLockboxEncrypt1MB(b *testing.B)   { lbEncrypt("1MB", b) }
func BenchmarkLockboxEncrypt4MB(b *testing.B)   { lbEncrypt("4MB", b) }

func lbEncrypt(key string, b *testing.B) {
	data, ok := chunks[key]
	if !ok {
		b.Fatalf("missing chunk %s", key)
	}

	b.SetBytes(int64(len(data)))
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		enc.Encrypt(data)
	}
}

func BenchmarkLockboxDecrypt128(b *testing.B)   { lbDecrypt("128B", b) }
func BenchmarkLockboxDecrypt4096(b *testing.B)  { lbDecrypt("4KB", b) }
func BenchmarkLockboxDecrypt64KB(b *testing.B)  { lbDecrypt("64KB", b) }
func BenchmarkLockboxDecrypt512KB(b *testing.B) { lbDecrypt("512KB", b) }
func BenchmarkLockboxDecrypt1MB(b *testing.B)   { lbDecrypt("1MB", b) }
func BenchmarkLockboxDecrypt4MB(b *testing.B)   { lbDecrypt("4MB", b) }

func lbDecrypt(key string, b *testing.B) {
	chunk, ok := chunks[key]
	if !ok {
		b.Fatalf("missing chunk %s", key)
	}
	data, err := enc.Encrypt(chunk)
	if err != nil {
		panic(err)
	}

	b.SetBytes(int64(len(chunk)))
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		dec.Decrypt(data)
	}
}

func mustRandomBytes(s int) []byte {
	buf := make([]byte, s)
	if _, err := io.ReadAtLeast(rand.Reader, buf, s); err != nil {
		panic(err)
	}
	return buf
}
<|endoftext|>


<commit_before>package iso

import (
	packerCommon "github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/packer"
	"github.com/jetbrains-infra/packer-builder-vsphere/common"
	"github.com/jetbrains-infra/packer-builder-vsphere/driver"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/helper/communicator"
)

type Builder struct {
	config *Config
	runner multistep.Runner
}

func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
	c, warnings, errs := NewConfig(raws...)
	if errs != nil {
		return warnings, errs
	}
	b.config = c

	return warnings, nil
}

func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	state := new(multistep.BasicStateBag)
	state.Put("comm", &b.config.Comm)
	state.Put("hook", hook)
	state.Put("ui", ui)

	var steps []multistep.Step

	steps = append(steps,
		&common.StepConnect{
			Config: &b.config.ConnectConfig,
		},
		&StepCreateVM{
			Config: &b.config.CreateConfig,
		},
		&StepAddCDRom{
			Config: &b.config.CDRomConfig,
		},
		&packerCommon.StepCreateFloppy{
			Files:       b.config.FloppyFiles,
			Directories: b.config.FloppyDirectories,
		},
		&StepAddFloppy{
			Config:    &b.config.FloppyConfig,
			Datastore: b.config.Datastore,
			Host:      b.config.Host,
		},
		&StepConfigParams{
			Config: &b.config.ConfigParamsConfig,
		},
	)

	if b.config.Comm.Type != "none" {
		steps = append(steps,
			&common.StepRun{
				Config: &b.config.RunConfig,
			},
			&StepBootCommand{
				Config: &b.config.BootConfig,
			},
			&common.StepWaitForIp{},
			&communicator.StepConnect{
				Config:    &b.config.Comm,
				Host:      common.CommHost,
				SSHConfig: common.SshConfig,
			},
			&packerCommon.StepProvision{},
			&common.StepShutdown{
				Config: &b.config.ShutdownConfig,
			},
		)
	}

	steps = append(steps,
		&StepRemoveCDRom{},
		&StepRemoveFloppy{
			Datastore: b.config.Datastore,
			Host:      b.config.Host,
		},
		&common.StepCreateSnapshot{
			CreateSnapshot: b.config.CreateSnapshot,
		},
		&common.StepConvertToTemplate{
			ConvertToTemplate: b.config.ConvertToTemplate,
		},
	)

	// Run!
	b.runner = packerCommon.NewRunner(steps, b.config.PackerConfig, ui)
	b.runner.Run(state)

	if err := common.CheckRunStatus(state); err != nil {
		return nil, err
	}

	artifact := &common.Artifact{
		Name: b.config.VMName,
		VM:   state.Get("vm").(*driver.VirtualMachine),
	}
	return artifact, nil
}

func (b *Builder) Cancel() {
	if b.runner != nil {
		b.runner.Cancel()
	}
}
<commit_msg>add/remove cdrom & floppy only if vm is started<commit_after>package iso

import (
	packerCommon "github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/packer"
	"github.com/jetbrains-infra/packer-builder-vsphere/common"
	"github.com/jetbrains-infra/packer-builder-vsphere/driver"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/helper/communicator"
)

type Builder struct {
	config *Config
	runner multistep.Runner
}

func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
	c, warnings, errs := NewConfig(raws...)
	if errs != nil {
		return warnings, errs
	}
	b.config = c

	return warnings, nil
}

func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	state := new(multistep.BasicStateBag)
	state.Put("comm", &b.config.Comm)
	state.Put("hook", hook)
	state.Put("ui", ui)

	var steps []multistep.Step

	steps = append(steps,
		&common.StepConnect{
			Config: &b.config.ConnectConfig,
		},
		&StepCreateVM{
			Config: &b.config.CreateConfig,
		},
		&StepConfigParams{
			Config: &b.config.ConfigParamsConfig,
		},
	)

	if b.config.Comm.Type != "none" {
		steps = append(steps,
			&StepAddCDRom{
				Config: &b.config.CDRomConfig,
			},
			&packerCommon.StepCreateFloppy{
				Files:       b.config.FloppyFiles,
				Directories: b.config.FloppyDirectories,
			},
			&StepAddFloppy{
				Config:    &b.config.FloppyConfig,
				Datastore: b.config.Datastore,
				Host:      b.config.Host,
			},
			&common.StepRun{
				Config: &b.config.RunConfig,
			},
			&StepBootCommand{
				Config: &b.config.BootConfig,
			},
			&common.StepWaitForIp{},
			&communicator.StepConnect{
				Config:    &b.config.Comm,
				Host:      common.CommHost,
				SSHConfig: common.SshConfig,
			},
			&packerCommon.StepProvision{},
			&common.StepShutdown{
				Config: &b.config.ShutdownConfig,
			},
			&StepRemoveCDRom{},
			&StepRemoveFloppy{
				Datastore: b.config.Datastore,
				Host:      b.config.Host,
			},
		)
	}

	steps = append(steps,
		&common.StepCreateSnapshot{
			CreateSnapshot: b.config.CreateSnapshot,
		},
		&common.StepConvertToTemplate{
			ConvertToTemplate: b.config.ConvertToTemplate,
		},
	)

	b.runner = packerCommon.NewRunner(steps, b.config.PackerConfig, ui)
	b.runner.Run(state)

	if err := common.CheckRunStatus(state); err != nil {
		return nil, err
	}

	artifact := &common.Artifact{
		Name: b.config.VMName,
		VM:   state.Get("vm").(*driver.VirtualMachine),
	}
	return artifact, nil
}

func (b *Builder) Cancel() {
	if b.runner != nil {
		b.runner.Cancel()
	}
}
<|endoftext|>


<commit_before><commit_msg>That was a bug<commit_after><|endoftext|>


<commit_before>package cloudpelican

// @author Robin Verlangen
// @todo Support bulk index requests
// Tool for logging data to CloudPelican directly from Go

// Imports
import (
    "net"
    "net/http"
    "net/url"
    "log"
    "sync"
    "strconv"
    "os"
    "fmt"
)

// Settings
var ENDPOINT string = "https://app.cloudpelican.com/api"
var TOKEN string = ""
var backendTimeout = time.Duration(5 * time.Second)
var debugMode bool = false
var maxBulkSize uint64 = uint64(100)
var hostname string

// Monitor drain status
var startCounter uint64 = uint64(0)
var startCounterMux sync.Mutex
var doneCounter uint64 = uint64(0)
var doneCounterMux sync.Mutex
var isDraining bool = false
var drained = make(chan bool);

// Log queue
var writeAheadBufferSize int = 1000
var writeAhead chan map[string]string = make(chan map[string]string, writeAheadBufferSize)
var writeAheadInit bool
var dropOnFullWriteAheadBuffer bool = true

// Set token
func SetToken(t string) {
    // Validate before setting
    validateToken(t)

    // Store
    TOKEN = t
}

// Set endpoint
func SetEndpoint(e string) {
    // Store
    ENDPOINT = e
}

// Set timeout
func SetBackendTimeout(to time.Duration) {
    backendTimeout = to
}

// Debug
func SetDebugMode(b bool) {
    debugMode = b
}

// Drain
func Drain() {
    isDraining = true
    if startCounter > doneCounter {
        // Wait for signal
        <- drained
    }
}

// Write a message
func LogMessage(msg string) bool {
    // Create fields map
    params := make(map[string]string)
    params["__token__"] = TOKEN
    params["msg"] = msg
    params["dt"] = getTimeString()

    // Push to channel
    return requestAsync(params)
}

// Current time
func getTimeString() string {
    return fmt.Sprintf("%d", int32(time.Now().Unix()) * int32(1000))
}

// Request async
func requestAsync(params map[string]string) bool {
    // Check amount of open items in the channel, if the channel is full, return false and drop this message
    if dropOnFullWriteAheadBuffer {
        var lwa int = len(writeAhead)
        if lwa == writeAheadBufferSize {
            log.Printf("Write ahead buffer is full and contains %d items. Dropping current log message", lwa)
        }
    }

    // Add counter
    startCounterMux.Lock()
    startCounter++
    startCounterMux.Unlock()

    // Do we have to start a writer?
    if writeAheadInit == false {
        writeAheadInit = true
        backendWriter()
    }

    // Insert into channel
    writeAhead <- params

    // OK
    return true
}

// Get hostname of this system
func getHostname() string {
    // Hostname
    name, err := os.Hostname()
    if err != nil {
        return ""
    }
    return name
}

// Backend writer
func backendWriter() {
    hostname = getHostname()
    go func() {
        // Client
        transport := &http.Transport{
            Dial: func(netw, addr string) (net.Conn, error) {
                    deadline := time.Now().Add(backendTimeout)
                    c, err := net.DialTimeout(netw, addr, time.Second)
                    if err != nil {
                            return nil, err
                    }
                    c.SetDeadline(deadline)
                    return c, nil
            }}
        httpclient := &http.Client{Transport: transport}

        // Wait for messages
        var urlParams url.Values
        var currentEventCount uint64 = uint64(0)
        for {
            // Read from channel
            var fields map[string]string
            fields = <- writeAhead

            // Populate url params
            if currentEventCount == 0 {
                urlParams = url.Values{}
            }
            for k, _ := range fields {
                if k == "__token__" {
                    // Token
                    urlParams.Add("t", fields[k]);
                } else {
                    // Field
                    urlParams.Add("f[" + strconv.FormatUint(currentEventCount, 10) + "][" + k + "]", fields[k]);
                }
            }

            // Host
            if len(hostname) > 0 {
                urlParams.Add("f[" + strconv.FormatUint(currentEventCount, 10) + "][host]", hostname);
            }

            // Increase current count
            currentEventCount++

            // Queue length
            var qLen = len(writeAhead)
            if qLen > 0 && currentEventCount < maxBulkSize {
                // There is more in the current queue, bulk request
                continue
            }

            // Assemble url
            var url string = ENDPOINT + "/push/bulk"

            // Make request
            if debugMode {
                log.Printf("Write ahead queue %d\n", qLen)
                log.Println(urlParams.Encode())
            }
            resp, err := httpclient.PostForm(url, urlParams)
            if err != nil {
                log.Printf("Error while forwarding data: %s\n", err)
            } else {
                defer resp.Body.Close()
            }

            // Done counter
            doneCounterMux.Lock()
            doneCounter += currentEventCount
            doneCounterMux.Unlock()

            // Reset event count
            currentEventCount = 0

            // Are we draining the system?
            if isDraining && doneCounter >= startCounter {
                // Flag the drained channel
                drained <- true
            }
        }
        log.Printf("Stopping backend writer")
    }()
}

// Timeout helper
func dialTimeout(network, addr string) (net.Conn, error) {
    return net.DialTimeout(network, addr, backendTimeout)
}

// Validate the token
func validateToken(t string) {
    if len(t) == 0 {
        log.Println("Please set a valid token with cloudpelican.SetToken(token string)")
    }
}<commit_msg>Time import<commit_after>package cloudpelican

// @author Robin Verlangen
// @todo Support bulk index requests
// Tool for logging data to CloudPelican directly from Go

// Imports
import (
    "net"
    "net/http"
    "net/url"
    "log"
    "sync"
    "strconv"
    "os"
    "fmt"
    "time"
)

// Settings
var ENDPOINT string = "https://app.cloudpelican.com/api"
var TOKEN string = ""
var backendTimeout = time.Duration(5 * time.Second)
var debugMode bool = false
var maxBulkSize uint64 = uint64(100)
var hostname string

// Monitor drain status
var startCounter uint64 = uint64(0)
var startCounterMux sync.Mutex
var doneCounter uint64 = uint64(0)
var doneCounterMux sync.Mutex
var isDraining bool = false
var drained = make(chan bool);

// Log queue
var writeAheadBufferSize int = 1000
var writeAhead chan map[string]string = make(chan map[string]string, writeAheadBufferSize)
var writeAheadInit bool
var dropOnFullWriteAheadBuffer bool = true

// Set token
func SetToken(t string) {
    // Validate before setting
    validateToken(t)

    // Store
    TOKEN = t
}

// Set endpoint
func SetEndpoint(e string) {
    // Store
    ENDPOINT = e
}

// Set timeout
func SetBackendTimeout(to time.Duration) {
    backendTimeout = to
}

// Debug
func SetDebugMode(b bool) {
    debugMode = b
}

// Drain
func Drain() {
    isDraining = true
    if startCounter > doneCounter {
        // Wait for signal
        <- drained
    }
}

// Write a message
func LogMessage(msg string) bool {
    // Create fields map
    params := make(map[string]string)
    params["__token__"] = TOKEN
    params["msg"] = msg
    params["dt"] = getTimeString()

    // Push to channel
    return requestAsync(params)
}

// Current time
func getTimeString() string {
    return fmt.Sprintf("%d", int32(time.Now().Unix()) * int32(1000))
}

// Request async
func requestAsync(params map[string]string) bool {
    // Check amount of open items in the channel, if the channel is full, return false and drop this message
    if dropOnFullWriteAheadBuffer {
        var lwa int = len(writeAhead)
        if lwa == writeAheadBufferSize {
            log.Printf("Write ahead buffer is full and contains %d items. Dropping current log message", lwa)
        }
    }

    // Add counter
    startCounterMux.Lock()
    startCounter++
    startCounterMux.Unlock()

    // Do we have to start a writer?
    if writeAheadInit == false {
        writeAheadInit = true
        backendWriter()
    }

    // Insert into channel
    writeAhead <- params

    // OK
    return true
}

// Get hostname of this system
func getHostname() string {
    // Hostname
    name, err := os.Hostname()
    if err != nil {
        return ""
    }
    return name
}

// Backend writer
func backendWriter() {
    hostname = getHostname()
    go func() {
        // Client
        transport := &http.Transport{
            Dial: func(netw, addr string) (net.Conn, error) {
                    deadline := time.Now().Add(backendTimeout)
                    c, err := net.DialTimeout(netw, addr, time.Second)
                    if err != nil {
                            return nil, err
                    }
                    c.SetDeadline(deadline)
                    return c, nil
            }}
        httpclient := &http.Client{Transport: transport}

        // Wait for messages
        var urlParams url.Values
        var currentEventCount uint64 = uint64(0)
        for {
            // Read from channel
            var fields map[string]string
            fields = <- writeAhead

            // Populate url params
            if currentEventCount == 0 {
                urlParams = url.Values{}
            }
            for k, _ := range fields {
                if k == "__token__" {
                    // Token
                    urlParams.Add("t", fields[k]);
                } else {
                    // Field
                    urlParams.Add("f[" + strconv.FormatUint(currentEventCount, 10) + "][" + k + "]", fields[k]);
                }
            }

            // Host
            if len(hostname) > 0 {
                urlParams.Add("f[" + strconv.FormatUint(currentEventCount, 10) + "][host]", hostname);
            }

            // Increase current count
            currentEventCount++

            // Queue length
            var qLen = len(writeAhead)
            if qLen > 0 && currentEventCount < maxBulkSize {
                // There is more in the current queue, bulk request
                continue
            }

            // Assemble url
            var url string = ENDPOINT + "/push/bulk"

            // Make request
            if debugMode {
                log.Printf("Write ahead queue %d\n", qLen)
                log.Println(urlParams.Encode())
            }
            resp, err := httpclient.PostForm(url, urlParams)
            if err != nil {
                log.Printf("Error while forwarding data: %s\n", err)
            } else {
                defer resp.Body.Close()
            }

            // Done counter
            doneCounterMux.Lock()
            doneCounter += currentEventCount
            doneCounterMux.Unlock()

            // Reset event count
            currentEventCount = 0

            // Are we draining the system?
            if isDraining && doneCounter >= startCounter {
                // Flag the drained channel
                drained <- true
            }
        }
        log.Printf("Stopping backend writer")
    }()
}

// Timeout helper
func dialTimeout(network, addr string) (net.Conn, error) {
    return net.DialTimeout(network, addr, backendTimeout)
}

// Validate the token
func validateToken(t string) {
    if len(t) == 0 {
        log.Println("Please set a valid token with cloudpelican.SetToken(token string)")
    }
}<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"cups-connector/cups"
	"cups-connector/gcp"
	"cups-connector/lib"
	"cups-connector/manager"
	"cups-connector/monitor"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/golang/glog"
)

func main() {
	config, err := lib.ConfigFromFile()
	if err != nil {
		glog.Fatal(err)
	}

	if _, err := os.Stat(config.MonitorSocketFilename); !os.IsNotExist(err) {
		if err != nil {
			glog.Fatal(err)
		}
		glog.Errorf(
			"A connector is already running, or the monitoring socket %s wasn't cleaned up properly",
			config.MonitorSocketFilename)
	}

	cups, err := cups.NewCUPS(config.CopyPrinterInfoToDisplayName, config.CUPSIgnoreRawPrinters,
		config.CUPSPrinterAttributes)
	if err != nil {
		glog.Fatal(err)
	}

	gcp, err := gcp.NewGoogleCloudPrint(config.XMPPJID, config.RobotRefreshToken,
		config.UserRefreshToken, config.ShareScope, config.ProxyName)
	if err != nil {
		glog.Fatal(err)
	}

	pm, err := manager.NewPrinterManager(cups, gcp, config.CUPSPrinterPollInterval,
		config.GCPMaxConcurrentDownloads, config.CUPSJobQueueSize, config.CUPSJobFullUsername)
	if err != nil {
		glog.Fatal(err)
	}

	m, err := monitor.NewMonitor(cups, gcp, pm, config.MonitorSocketFilename)
	if err != nil {
		glog.Fatal(err)
	}

	fmt.Printf("Google Cloud Print CUPS Connector ready to rock as proxy '%s'\n", config.ProxyName)

	waitIndefinitely()

	fmt.Println("")
	fmt.Println("shutting down normally")

	m.Quit()
	pm.Quit()
	cups.Quit()
	glog.Flush()
}

// Blocks until Ctrl-C or SIGTERM.
func waitIndefinitely() {
	// TODO(jacobmarble): Second signal forces quit.
	ch := make(chan os.Signal)
	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
	<-ch
}
<commit_msg>Fix a socket error<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"cups-connector/cups"
	"cups-connector/gcp"
	"cups-connector/lib"
	"cups-connector/manager"
	"cups-connector/monitor"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/golang/glog"
)

func main() {
	config, err := lib.ConfigFromFile()
	if err != nil {
		glog.Fatal(err)
	}

	if _, err := os.Stat(config.MonitorSocketFilename); !os.IsNotExist(err) {
		if err != nil {
			glog.Fatal(err)
		}
		glog.Fatalf(
			"A connector is already running, or the monitoring socket %s wasn't cleaned up properly",
			config.MonitorSocketFilename)
	}

	cups, err := cups.NewCUPS(config.CopyPrinterInfoToDisplayName, config.CUPSIgnoreRawPrinters,
		config.CUPSPrinterAttributes)
	if err != nil {
		glog.Fatal(err)
	}

	gcp, err := gcp.NewGoogleCloudPrint(config.XMPPJID, config.RobotRefreshToken,
		config.UserRefreshToken, config.ShareScope, config.ProxyName)
	if err != nil {
		glog.Fatal(err)
	}

	pm, err := manager.NewPrinterManager(cups, gcp, config.CUPSPrinterPollInterval,
		config.GCPMaxConcurrentDownloads, config.CUPSJobQueueSize, config.CUPSJobFullUsername)
	if err != nil {
		glog.Fatal(err)
	}

	m, err := monitor.NewMonitor(cups, gcp, pm, config.MonitorSocketFilename)
	if err != nil {
		glog.Fatal(err)
	}

	fmt.Printf("Google Cloud Print CUPS Connector ready to rock as proxy '%s'\n", config.ProxyName)

	waitIndefinitely()

	fmt.Println("")
	fmt.Println("shutting down normally")

	m.Quit()
	pm.Quit()
	cups.Quit()
	glog.Flush()
}

// Blocks until Ctrl-C or SIGTERM.
func waitIndefinitely() {
	// TODO(jacobmarble): Second signal forces quit.
	ch := make(chan os.Signal)
	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
	<-ch
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"
)

type repoTree struct {
	// repositories under this repository
	children map[string]*repoTree

	// files in this repository.
	entries []string
}

func newRepoTree(localRoot string) *repoTree {
	return &repoTree{
		children: make(map[string]*repoTree),
	}
}

// allChildren returns all the repositories (including the receiver)
// as a map keyed by relative path.
func (t *repoTree) allChildren() map[string]*repoTree {
	r := map[string]*repoTree{"": t}
	for nm, ch := range t.children {
		for sub, subCh := range ch.allChildren() {
			r[filepath.Join(nm, sub)] = subCh
		}
	}
	return r
}

// construct fills `parent` looking through `dir` subdir of `repoRoot`.
func construct(repoRoot, dir string, parent *repoTree) error {
	isRepo := false
	localRoot := filepath.Join(repoRoot, dir)
	if stat, err := os.Stat(filepath.Join(localRoot, ".git")); err == nil && stat.IsDir() {
		isRepo = true
	} else if stat, err := os.Stat(filepath.Join(localRoot, ".gitid")); err == nil && !stat.IsDir() {
		isRepo = true
	}

	if isRepo {
		sub := newRepoTree(localRoot)
		parent.children[dir] = sub
		parent = sub

		repoRoot = localRoot
		dir = ""
	}

	entries, err := ioutil.ReadDir(localRoot)
	if err != nil {
		return err
	}
	for _, e := range entries {
		if (e.IsDir() && e.Name() == ".git") || (!e.IsDir() && e.Name() == ".gitid") {
			continue
		}

		subName := filepath.Join(dir, e.Name())
		if e.IsDir() {
			construct(repoRoot, subName, parent)
		} else {
			parent.entries = append(parent.entries, subName)
		}
	}
	return nil
}

// symlinkRepo creates symlinks for all the files in `child`.
func symlinkRepo(name string, child *repoTree, roRoot, rwRoot string) error {
	fi, err := os.Stat(filepath.Join(rwRoot, name))
	if err == nil && fi.IsDir() {
		return nil
	}

	for _, e := range child.entries {
		dest := filepath.Join(rwRoot, name, e)

		if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
			return err
		}
		if err := os.Symlink(filepath.Join(roRoot, name, e), dest); err != nil {
			return err
		}
	}
	return nil
}

// createTreeLinks tries to short-cut symlinks for whole trees by
// symlinking to the root of a repository in the RO tree.
func createTreeLinks(ro, rw *repoTree, roRoot, rwRoot string) error {
	allRW := rw.allChildren()

outer:
	for nm, ch := range ro.children {
		foundCheckout := false
		foundRecurse := false
		for k := range allRW {
			if k == "" {
				continue
			}
			if nm == k {
				foundRecurse = true
				break
			}
			rel, err := filepath.Rel(nm, k)
			if err != nil {
				return err
			}

			if strings.HasPrefix(rel, "..") {
				continue
			}

			// we have a checkout below "nm".
			foundCheckout = true
			break
		}

		switch {
		case foundRecurse:
			if err := createTreeLinks(ch, rw.children[nm], filepath.Join(roRoot, nm), filepath.Join(rwRoot, nm)); err != nil {
				return err
			}
			continue outer
		case !foundCheckout:
			dest := filepath.Join(rwRoot, nm)
			if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
				return err
			}
			if err := os.Symlink(filepath.Join(roRoot, nm), dest); err != nil {
				return err
			}
		}
	}
	return nil
}

// createLinks will populate a RW tree with symlinks to the RO tree.
func createLinks(ro, rw *repoTree, roRoot, rwRoot string) error {
	if err := createTreeLinks(ro, rw, roRoot, rwRoot); err != nil {
		return err
	}

	rwc := rw.allChildren()
	for nm, ch := range ro.allChildren() {
		if _, ok := rwc[nm]; !ok {
			if err := symlinkRepo(nm, ch, roRoot, rwRoot); err != nil {
				return err
			}
		}
	}
	return nil
}

// clearLinks removes all symlinks to the RO tree. It returns the workspace name that was linked before.
func clearLinks(mount, dir string) (string, error) {
	mount = filepath.Clean(mount)

	var prefix string
	var dirs []string
	if err := filepath.Walk(dir, func(n string, fi os.FileInfo, err error) error {
		if fi.Mode()&os.ModeSymlink != 0 {
			target, err := os.Readlink(n)
			if err != nil {
				return err
			}
			if strings.HasPrefix(target, mount) {
				prefix = target
				if err := os.Remove(n); err != nil {
					return err
				}
			}
		}
		if fi.IsDir() {
			dirs = append(dirs, n)
		}
		return nil
	}); err != nil {
		return "", err
	}

	// Reverse the ordering, so we get the deepest subdirs first.
	sort.Strings(dirs)
	for i := range dirs {
		d := dirs[len(dirs)-1-i]
		// Ignore error: dir may still contain entries.
		os.Remove(d)
	}

	prefix = strings.TrimPrefix(prefix, mount+"/")
	if i := strings.Index(prefix, "/"); i != -1 {
		prefix = prefix[:i]
	}
	return prefix, nil
}

func getSHA1s(dir string) (map[string]string, error) {
	attr := "user.gitsha1"

	shamap := map[string]string{}

	data := make([]byte, 1024)

	if err := filepath.Walk(dir, func(n string, fi os.FileInfo, err error) error {
		if n == filepath.Join(dir, "manifest.xml") {
			return nil
		}
		if fi.Mode()&os.ModeType != 0 {
			return nil
		}
		if filepath.Base(n) == ".gitid" {
			return nil
		}

		sz, err := syscall.Getxattr(n, attr, data)
		if err != nil {
			return fmt.Errorf("Getxattr(%s, %s): %v", n, attr, err)
		}
		rel, err := filepath.Rel(dir, n)
		if err != nil {
			return err
		}
		shamap[rel] = string(data[:sz])
		return nil
	}); err != nil {
		return nil, err
	}

	return shamap, nil
}

// Returns the filenames (as relative paths) in newDir that have
// changed relative to the files in oldDir.
func changedFiles(oldDir, newDir string) ([]string, error) {
	// TODO(hanwen): could be parallel.
	oldSHA1s, err := getSHA1s(oldDir)
	if err != nil {
		return nil, err
	}
	newSHA1s, err := getSHA1s(newDir)
	if err != nil {
		return nil, err
	}

	var changed []string
	for k, v := range newSHA1s {
		old, ok := oldSHA1s[k]
		if !ok || old != v {
			changed = append(changed, k)
		}
	}
	sort.Strings(changed)
	return changed, nil
}

// populateCheckout updates a RW dir with new symlinks to the given RO dir.
func populateCheckout(ro, rw string) error {
	wsName, err := clearLinks(filepath.Dir(ro), rw)
	if err != nil {
		log.Fatal(err)
	}

	rwTree := newRepoTree(rw)
	if err := construct(rw, "", rwTree); err != nil {
		return err
	}

	roTree := newRepoTree(ro)
	if err := construct(ro, "", roTree); err != nil {
		return err
	}

	if err := createLinks(roTree, rwTree, ro, rw); err != nil {
		return err
	}

	// TODO(hanwen): can be done in parallel to the other processes.
	oldRoot := filepath.Join(filepath.Dir(ro), wsName)
	changed, err := changedFiles(oldRoot, ro)
	if err != nil {
		return fmt.Errorf("changedFiles: %v", err)
	}

	// TODO(hanwen): parallel?
	now := time.Now()
	for _, n := range changed {
		if err := os.Chtimes(filepath.Join(ro, n), now, now); err != nil {
			return err
		}
	}

	return nil
}

func main() {
	mount := flag.String("ro", "", "path to slothfs-multifs mount.")
	flag.Parse()

	dir := "."
	if len(flag.Args()) == 1 {
		dir = flag.Arg(0)
	} else if len(flag.Args()) > 1 {
		log.Fatal("too many arguments.")
	}

	if err := populateCheckout(*mount, dir); err != nil {
		log.Fatalf("populateCheckout: %v", err)
	}
}
<commit_msg>Clean the filepath of -ro option.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"
)

type repoTree struct {
	// repositories under this repository
	children map[string]*repoTree

	// files in this repository.
	entries []string
}

func newRepoTree(localRoot string) *repoTree {
	return &repoTree{
		children: make(map[string]*repoTree),
	}
}

// allChildren returns all the repositories (including the receiver)
// as a map keyed by relative path.
func (t *repoTree) allChildren() map[string]*repoTree {
	r := map[string]*repoTree{"": t}
	for nm, ch := range t.children {
		for sub, subCh := range ch.allChildren() {
			r[filepath.Join(nm, sub)] = subCh
		}
	}
	return r
}

// construct fills `parent` looking through `dir` subdir of `repoRoot`.
func construct(repoRoot, dir string, parent *repoTree) error {
	isRepo := false
	localRoot := filepath.Join(repoRoot, dir)
	if stat, err := os.Stat(filepath.Join(localRoot, ".git")); err == nil && stat.IsDir() {
		isRepo = true
	} else if stat, err := os.Stat(filepath.Join(localRoot, ".gitid")); err == nil && !stat.IsDir() {
		isRepo = true
	}

	if isRepo {
		sub := newRepoTree(localRoot)
		parent.children[dir] = sub
		parent = sub

		repoRoot = localRoot
		dir = ""
	}

	entries, err := ioutil.ReadDir(localRoot)
	if err != nil {
		return err
	}
	for _, e := range entries {
		if (e.IsDir() && e.Name() == ".git") || (!e.IsDir() && e.Name() == ".gitid") {
			continue
		}

		subName := filepath.Join(dir, e.Name())
		if e.IsDir() {
			construct(repoRoot, subName, parent)
		} else {
			parent.entries = append(parent.entries, subName)
		}
	}
	return nil
}

// symlinkRepo creates symlinks for all the files in `child`.
func symlinkRepo(name string, child *repoTree, roRoot, rwRoot string) error {
	fi, err := os.Stat(filepath.Join(rwRoot, name))
	if err == nil && fi.IsDir() {
		return nil
	}

	for _, e := range child.entries {
		dest := filepath.Join(rwRoot, name, e)

		if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
			return err
		}
		if err := os.Symlink(filepath.Join(roRoot, name, e), dest); err != nil {
			return err
		}
	}
	return nil
}

// createTreeLinks tries to short-cut symlinks for whole trees by
// symlinking to the root of a repository in the RO tree.
func createTreeLinks(ro, rw *repoTree, roRoot, rwRoot string) error {
	allRW := rw.allChildren()

outer:
	for nm, ch := range ro.children {
		foundCheckout := false
		foundRecurse := false
		for k := range allRW {
			if k == "" {
				continue
			}
			if nm == k {
				foundRecurse = true
				break
			}
			rel, err := filepath.Rel(nm, k)
			if err != nil {
				return err
			}

			if strings.HasPrefix(rel, "..") {
				continue
			}

			// we have a checkout below "nm".
			foundCheckout = true
			break
		}

		switch {
		case foundRecurse:
			if err := createTreeLinks(ch, rw.children[nm], filepath.Join(roRoot, nm), filepath.Join(rwRoot, nm)); err != nil {
				return err
			}
			continue outer
		case !foundCheckout:
			dest := filepath.Join(rwRoot, nm)
			if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
				return err
			}
			if err := os.Symlink(filepath.Join(roRoot, nm), dest); err != nil {
				return err
			}
		}
	}
	return nil
}

// createLinks will populate a RW tree with symlinks to the RO tree.
func createLinks(ro, rw *repoTree, roRoot, rwRoot string) error {
	if err := createTreeLinks(ro, rw, roRoot, rwRoot); err != nil {
		return err
	}

	rwc := rw.allChildren()
	for nm, ch := range ro.allChildren() {
		if _, ok := rwc[nm]; !ok {
			if err := symlinkRepo(nm, ch, roRoot, rwRoot); err != nil {
				return err
			}
		}
	}
	return nil
}

// clearLinks removes all symlinks to the RO tree. It returns the workspace name that was linked before.
func clearLinks(mount, dir string) (string, error) {
	mount = filepath.Clean(mount)

	var prefix string
	var dirs []string
	if err := filepath.Walk(dir, func(n string, fi os.FileInfo, err error) error {
		if fi.Mode()&os.ModeSymlink != 0 {
			target, err := os.Readlink(n)
			if err != nil {
				return err
			}
			if strings.HasPrefix(target, mount) {
				prefix = target
				if err := os.Remove(n); err != nil {
					return err
				}
			}
		}
		if fi.IsDir() {
			dirs = append(dirs, n)
		}
		return nil
	}); err != nil {
		return "", err
	}

	// Reverse the ordering, so we get the deepest subdirs first.
	sort.Strings(dirs)
	for i := range dirs {
		d := dirs[len(dirs)-1-i]
		// Ignore error: dir may still contain entries.
		os.Remove(d)
	}

	prefix = strings.TrimPrefix(prefix, mount+"/")
	if i := strings.Index(prefix, "/"); i != -1 {
		prefix = prefix[:i]
	}
	return prefix, nil
}

func getSHA1s(dir string) (map[string]string, error) {
	attr := "user.gitsha1"

	shamap := map[string]string{}

	data := make([]byte, 1024)

	if err := filepath.Walk(dir, func(n string, fi os.FileInfo, err error) error {
		if n == filepath.Join(dir, "manifest.xml") {
			return nil
		}
		if fi.Mode()&os.ModeType != 0 {
			return nil
		}
		if filepath.Base(n) == ".gitid" {
			return nil
		}

		sz, err := syscall.Getxattr(n, attr, data)
		if err != nil {
			return fmt.Errorf("Getxattr(%s, %s): %v", n, attr, err)
		}
		rel, err := filepath.Rel(dir, n)
		if err != nil {
			return err
		}
		shamap[rel] = string(data[:sz])
		return nil
	}); err != nil {
		return nil, err
	}

	return shamap, nil
}

// Returns the filenames (as relative paths) in newDir that have
// changed relative to the files in oldDir.
func changedFiles(oldDir, newDir string) ([]string, error) {
	// TODO(hanwen): could be parallel.
	oldSHA1s, err := getSHA1s(oldDir)
	if err != nil {
		return nil, err
	}
	newSHA1s, err := getSHA1s(newDir)
	if err != nil {
		return nil, err
	}

	var changed []string
	for k, v := range newSHA1s {
		old, ok := oldSHA1s[k]
		if !ok || old != v {
			changed = append(changed, k)
		}
	}
	sort.Strings(changed)
	return changed, nil
}

// populateCheckout updates a RW dir with new symlinks to the given RO dir.
func populateCheckout(ro, rw string) error {
	ro = filepath.Clean(ro)
	wsName, err := clearLinks(filepath.Dir(ro), rw)
	if err != nil {
		log.Fatal(err)
	}

	rwTree := newRepoTree(rw)
	if err := construct(rw, "", rwTree); err != nil {
		return err
	}

	roTree := newRepoTree(ro)
	if err := construct(ro, "", roTree); err != nil {
		return err
	}

	if err := createLinks(roTree, rwTree, ro, rw); err != nil {
		return err
	}

	// TODO(hanwen): can be done in parallel to the other processes.
	oldRoot := filepath.Join(filepath.Dir(ro), wsName)
	changed, err := changedFiles(oldRoot, ro)
	if err != nil {
		return fmt.Errorf("changedFiles: %v", err)
	}

	// TODO(hanwen): parallel?
	now := time.Now()
	for _, n := range changed {
		if err := os.Chtimes(filepath.Join(ro, n), now, now); err != nil {
			return err
		}
	}

	return nil
}

func main() {
	mount := flag.String("ro", "", "path to slothfs-multifs mount.")
	flag.Parse()

	dir := "."
	if len(flag.Args()) == 1 {
		dir = flag.Arg(0)
	} else if len(flag.Args()) > 1 {
		log.Fatal("too many arguments.")
	}

	if err := populateCheckout(*mount, dir); err != nil {
		log.Fatalf("populateCheckout: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/fubarhouse/golang-drush/command"
	"github.com/fubarhouse/golang-drush/make"
	"os"
	"strings"
)

func main() {

	var Path = flag.String("path", "", "Path to site")
	var Site = flag.String("site", "", "Shortname of site")
	var Domain = flag.String("domain", "", "Domain of site")
	var Alias = flag.String("alias", "", "Alias of site")
	var Makes = flag.String("makes", "", "Comma-separated list of make files to use")
	var BuildID = flag.String("build", "", "optional timestamp of site")
	var VHostDir = flag.String("vhost-dir", "/etc/nginx/sites-enabled", "Directory containing virtual host file(s)")
	var WebserverName = flag.String("webserver-name", "nginx", "The name of the web service on the server.")

	// Usage:
	// -path="/path/to/site" \
	// -site="mysite" \
	// -domain="mysite.dev" \
	// -alias="mysite.dev" \
	// -makes="/path/to/make1.make, /path/to/make2.make" \

	flag.Parse()

	if *Site == "" {
		log.Infoln("Site input is empty")
	}
	if *Alias == "" {
		log.Infoln("Alias input is empty")
	}
	if *Path == "" {
		log.Infoln("Path input is empty")
	}
	if *Domain == "" {
		log.Infoln("Domain input is empty")
	}
	if *Makes == "" {
		log.Infoln("Makes input is empty")
	}

	if *Site == "" || *Alias == "" || *Makes == "" || *Path == "" || *Domain == "" {
		flag.Usage()
		os.Exit(1)
	}

	x := make.NewSite(*Makes, *Site, *Path, *Alias, *WebserverName, *Domain, *VHostDir)
	y := make.NewmakeDB("127.0.0.1", "root", "root", 3306)
	x.DatabaseSet(y)
	if *BuildID == "" {
		x.TimeStampReset()
	} else {
		x.TimeStampSet(*BuildID)
	}

	MakefilesFormatted := strings.Replace(*Makes, " ", "", -1)
	MakeFiles := strings.Split(MakefilesFormatted, ",")

	x.ActionRebuildCodebase(MakeFiles)
	x.InstallSiteRef()
	x.InstallPrivateFileSystem()
	x.ActionInstall()
	x.SymReinstall(x.TimeStampGet())
	x.VhostInstall()
	x.AliasInstall()
	command.DrushUpdateDatabase(x.Alias)
	command.DrushRebuildRegistry(x.Alias)
	x.RestartWebServer()
}
<commit_msg>Remove unused imports.<commit_after>package main

import (
	"flag"
	log "github.com/Sirupsen/logrus"
	"github.com/fubarhouse/golang-drush/command"
	"github.com/fubarhouse/golang-drush/make"
	"os"
	"strings"
)

func main() {

	var Path = flag.String("path", "", "Path to site")
	var Site = flag.String("site", "", "Shortname of site")
	var Domain = flag.String("domain", "", "Domain of site")
	var Alias = flag.String("alias", "", "Alias of site")
	var Makes = flag.String("makes", "", "Comma-separated list of make files to use")
	var BuildID = flag.String("build", "", "optional timestamp of site")
	var VHostDir = flag.String("vhost-dir", "/etc/nginx/sites-enabled", "Directory containing virtual host file(s)")
	var WebserverName = flag.String("webserver-name", "nginx", "The name of the web service on the server.")

	// Usage:
	// -path="/path/to/site" \
	// -site="mysite" \
	// -domain="mysite.dev" \
	// -alias="mysite.dev" \
	// -makes="/path/to/make1.make, /path/to/make2.make" \

	flag.Parse()

	if *Site == "" {
		log.Infoln("Site input is empty")
	}
	if *Alias == "" {
		log.Infoln("Alias input is empty")
	}
	if *Path == "" {
		log.Infoln("Path input is empty")
	}
	if *Domain == "" {
		log.Infoln("Domain input is empty")
	}
	if *Makes == "" {
		log.Infoln("Makes input is empty")
	}

	if *Site == "" || *Alias == "" || *Makes == "" || *Path == "" || *Domain == "" {
		flag.Usage()
		os.Exit(1)
	}

	x := make.NewSite(*Makes, *Site, *Path, *Alias, *WebserverName, *Domain, *VHostDir)
	y := make.NewmakeDB("127.0.0.1", "root", "root", 3306)
	x.DatabaseSet(y)
	if *BuildID == "" {
		x.TimeStampReset()
	} else {
		x.TimeStampSet(*BuildID)
	}

	MakefilesFormatted := strings.Replace(*Makes, " ", "", -1)
	MakeFiles := strings.Split(MakefilesFormatted, ",")

	x.ActionRebuildCodebase(MakeFiles)
	x.InstallSiteRef()
	x.InstallPrivateFileSystem()
	x.ActionInstall()
	x.SymReinstall(x.TimeStampGet())
	x.VhostInstall()
	x.AliasInstall()
	command.DrushUpdateDatabase(x.Alias)
	command.DrushRebuildRegistry(x.Alias)
	x.RestartWebServer()
}
<|endoftext|>


<commit_before>package consumer

import (
	"container/list"
	"encoding/json"
	"flume-log-sdk/config"
	"flume-log-sdk/consumer/client"
	"flume-log-sdk/consumer/pool"
	"flume-log-sdk/rpc/flume"
	"fmt"
	"github.com/momotech/GoRedis/libs/stdlog"
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"
)

type counter struct {
	lastSuccValue int64

	currSuccValue int64

	lastFailValue int64

	currFailValue int64
}

const (
	batchSize = 1000
	sendbuff  = 100000
)

//生成对象池用于缓存 thriftEvent对象
var objpool *sync.Pool
var eventPool *sync.Pool

func init() {
	objpool = &sync.Pool{}
	objpool.New = func() interface{} {
		//创建生成thriftevent的
		return client.NewFlumeEvent()
	}

	eventPool = &sync.Pool{}
	eventPool.New = func() interface{} {
		return make([]*flume.ThriftFlumeEvent, 0, batchSize)
	}
}

// 用于向flume中作为sink 通过thrift客户端写入日志
type SourceServer struct {
	flumeClientPool *list.List
	isStop          bool
	monitorCount    counter
	business        string
	batchSize       int
	buffChannel     chan *flume.ThriftFlumeEvent
	sourceLog       stdlog.Logger
	chpool          chan []*flume.ThriftFlumeEvent
}

func newSourceServer(business string, flumePool *list.List, sourceLog stdlog.Logger) (server *SourceServer) {
	buffChannel := make(chan *flume.ThriftFlumeEvent, sendbuff)
	sourceServer := &SourceServer{
		business:        business,
		flumeClientPool: flumePool,
		batchSize:       batchSize,
		buffChannel:     buffChannel,
		sourceLog:       sourceLog}

	//对临时创建的数组切片进行缓存减少gc  50 * 1000 = 5W 个  最多缓存event
	chpool := make(chan []*flume.ThriftFlumeEvent, 50)
	for i := 0; i < 50; i++ {
		chpool <- make([]*flume.ThriftFlumeEvent, 0, sourceServer.batchSize)
	}

	sourceServer.chpool = chpool

	return sourceServer
}

func (self *SourceServer) monitor() (succ, fail int64, bufferSize, arrayPool int) {
	currSucc := self.monitorCount.currSuccValue
	currFail := self.monitorCount.currFailValue
	succ = (currSucc - self.monitorCount.lastSuccValue)
	fail = (currFail - self.monitorCount.lastFailValue)
	self.monitorCount.lastSuccValue = currSucc
	self.monitorCount.lastFailValue = currFail

	//自己的Buffer大小
	bufferSize = len(self.buffChannel)
	arrayPool = len(self.chpool)
	return
}

//启动pop
func (self *SourceServer) start() {

	self.isStop = false

	//创建chan ,buffer 为10
	sendbuff := make(chan []*flume.ThriftFlumeEvent, 100)
	//启动20个go程从channel获取
	for i := 0; i < 10; i++ {
		go func(ch chan []*flume.ThriftFlumeEvent) {
			for !self.isStop {
				events := <-ch
				self.innerSend(events)
				defer func() {
					//回收
					self.chpool <- events[:0]
				}()

			}
		}(sendbuff)
	}

	go func() {
		//批量收集数据
		pack := <-self.chpool
		// item := eventPool.Get()
		// pack := item.([]*flume.ThriftFlumeEvent)
		for !self.isStop {
			event := <-self.buffChannel

			if len(pack) < self.batchSize {
				pack = append(pack, event)
				continue
			}
			sendbuff <- pack[:len(pack)]
			//从池子中获取一个slice
			pack = <-self.chpool
		}

		close(sendbuff)
	}()

	close(self.chpool)
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER [%s]|STARTED\n", self.business)
}

func (self *SourceServer) innerSend(events []*flume.ThriftFlumeEvent) {

	for i := 0; i < 3; i++ {

		pool := self.getFlumeClientPool()
		if nil == pool {
			continue
		}
		flumeclient, err := pool.Get(5 * time.Second)
		if nil != err || nil == flumeclient {
			self.sourceLog.Printf("LOG_SOURCE|GET FLUMECLIENT|FAIL|%s|%s|TRY:%d\n", self.business, err, i)
			continue
		}

		err = flumeclient.AppendBatch(events)
		defer func() {
			if err := recover(); nil != err {
				//回收这个坏的连接
				pool.ReleaseBroken(flumeclient)
			} else {
				pool.Release(flumeclient)
			}
		}()

		if nil != err {
			atomic.AddInt64(&self.monitorCount.currFailValue, int64(len(events)))
			self.sourceLog.Printf("LOG_SOURCE|SEND FLUME|FAIL|%s|%s|TRY:%d\n", self.business, err.Error(), i)

		} else {
			atomic.AddInt64(&self.monitorCount.currSuccValue, int64(1*self.batchSize))
			if rand.Int()%10000 == 0 {
				self.sourceLog.Printf("trace|send 2 flume succ|%s|%d\n", flumeclient.HostPort(), len(events))
			}
			break
		}

	}
}

//解析出decodecommand
func decodeCommand(resp []byte) (string, *flume.ThriftFlumeEvent) {
	var cmd config.Command
	err := json.Unmarshal(resp, &cmd)
	if nil != err {
		log.Printf("command unmarshal fail ! %T | error:%s\n", resp, err.Error())
		return "", nil
	}
	//
	momoid := cmd.Params["momoid"].(string)

	businessName := cmd.Params["businessName"].(string)

	action := cmd.Params["type"].(string)

	bodyContent := cmd.Params["body"]

	//将businessName 加入到body中
	bodyMap := bodyContent.(map[string]interface{})
	bodyMap["business_type"] = businessName

	body, err := json.Marshal(bodyContent)
	if nil != err {
		log.Printf("marshal log body fail %s", err.Error())
		return businessName, nil
	}

	//拼Body
	flumeBody := fmt.Sprintf("%s\t%s\t%s", momoid, action, string(body))
	obj := objpool.Get()
	// obj := client.NewFlumeEvent()
	event := client.EventFillUp(obj, businessName, action, []byte(flumeBody))
	// event := client.NewFlumeEvent(businessName, action, []byte(flumeBody))
	return businessName, event
}

func (self *SourceServer) stop() {
	self.isStop = true
	time.Sleep(5 * time.Second)

	//遍历所有的flumeclientlink，将当前Business从该链表中移除
	for v := self.flumeClientPool.Back(); nil != v; v = v.Prev() {
		v.Value.(*pool.FlumePoolLink).DetachBusiness(self.business)
	}
	close(self.buffChannel)
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER|[%s]|STOPPED\n", self.business)
}

func (self *SourceServer) getFlumeClientPool() *pool.FlumeClientPool {

	//采用轮训算法
	e := self.flumeClientPool.Back()
	if nil == e {
		return nil
	}
	self.flumeClientPool.MoveToFront(e)
	return e.Value.(*pool.FlumePoolLink).FlumePool

}
<commit_msg>	modified:   consumer/log_source.go<commit_after>package consumer

import (
	"container/list"
	"encoding/json"
	"flume-log-sdk/config"
	"flume-log-sdk/consumer/client"
	"flume-log-sdk/consumer/pool"
	"flume-log-sdk/rpc/flume"
	"fmt"
	"github.com/momotech/GoRedis/libs/stdlog"
	"log"
	"math/rand"
	"sync/atomic"
	"time"
)

type counter struct {
	lastSuccValue int64

	currSuccValue int64

	lastFailValue int64

	currFailValue int64
}

const (
	batchSize = 500
	sendbuff  = 10000
)

// 用于向flume中作为sink 通过thrift客户端写入日志
type SourceServer struct {
	flumeClientPool *list.List
	isStop          bool
	monitorCount    counter
	business        string
	batchSize       int
	buffChannel     chan *flume.ThriftFlumeEvent
	sourceLog       stdlog.Logger
	chpool          chan []*flume.ThriftFlumeEvent
}

func newSourceServer(business string, flumePool *list.List, sourceLog stdlog.Logger) (server *SourceServer) {
	buffChannel := make(chan *flume.ThriftFlumeEvent, sendbuff)
	sourceServer := &SourceServer{
		business:        business,
		flumeClientPool: flumePool,
		batchSize:       batchSize,
		buffChannel:     buffChannel,
		sourceLog:       sourceLog}

	//对临时创建的数组切片进行缓存减少gc  50 * 1000 = 5W 个  最多缓存event
	chpool := make(chan []*flume.ThriftFlumeEvent, 50)
	for i := 0; i < 50; i++ {
		chpool <- make([]*flume.ThriftFlumeEvent, 0, sourceServer.batchSize)
	}

	sourceServer.chpool = chpool

	return sourceServer
}

func (self *SourceServer) monitor() (succ, fail int64, bufferSize, arrayPool int) {
	currSucc := self.monitorCount.currSuccValue
	currFail := self.monitorCount.currFailValue
	succ = (currSucc - self.monitorCount.lastSuccValue)
	fail = (currFail - self.monitorCount.lastFailValue)
	self.monitorCount.lastSuccValue = currSucc
	self.monitorCount.lastFailValue = currFail

	//自己的Buffer大小
	bufferSize = len(self.buffChannel)
	arrayPool = len(self.chpool)
	return
}

//启动pop
func (self *SourceServer) start() {

	self.isStop = false

	//创建chan ,buffer 为10
	sendbuff := make(chan []*flume.ThriftFlumeEvent, 50)
	//启动20个go程从channel获取
	for i := 0; i < 10; i++ {
		go func(ch chan []*flume.ThriftFlumeEvent) {
			for !self.isStop {
				events := <-ch
				self.innerSend(events)
				//回收
				self.chpool <- events[:0]

			}
		}(sendbuff)
	}

	go func() {
		//批量收集数据
		pack := <-self.chpool
		for !self.isStop {

			if len(pack) < self.batchSize {
				pack = append(pack, <-self.buffChannel)
				continue
			}
			sendbuff <- pack[:len(pack)]
			//从池子中获取一个slice
			pack = <-self.chpool
		}

		close(sendbuff)
	}()

	close(self.chpool)
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER [%s]|STARTED\n", self.business)
}

func (self *SourceServer) innerSend(events []*flume.ThriftFlumeEvent) {

	for i := 0; i < 3; i++ {

		pool := self.getFlumeClientPool()
		if nil == pool {
			continue
		}
		flumeclient, err := pool.Get(5 * time.Second)
		if nil != err || nil == flumeclient {
			self.sourceLog.Printf("LOG_SOURCE|GET FLUMECLIENT|FAIL|%s|%s|TRY:%d\n", self.business, err, i)
			continue
		}

		err = flumeclient.AppendBatch(events)
		defer func() {
			if err := recover(); nil != err {
				//回收这个坏的连接
				pool.ReleaseBroken(flumeclient)
			} else {
				pool.Release(flumeclient)
			}
		}()

		if nil != err {
			atomic.AddInt64(&self.monitorCount.currFailValue, int64(len(events)))
			self.sourceLog.Printf("LOG_SOURCE|SEND FLUME|FAIL|%s|%s|TRY:%d\n", self.business, err.Error(), i)

		} else {
			atomic.AddInt64(&self.monitorCount.currSuccValue, int64(1*self.batchSize))
			if rand.Int()%10000 == 0 {
				self.sourceLog.Printf("trace|send 2 flume succ|%s|%d\n", flumeclient.HostPort(), len(events))
			}
			break
		}

	}
}

//解析出decodecommand
func decodeCommand(resp []byte) (string, *flume.ThriftFlumeEvent) {
	var cmd config.Command
	err := json.Unmarshal(resp, &cmd)
	if nil != err {
		log.Printf("command unmarshal fail ! %T | error:%s\n", resp, err.Error())
		return "", nil
	}
	//
	momoid := cmd.Params["momoid"].(string)

	businessName := cmd.Params["businessName"].(string)

	action := cmd.Params["type"].(string)

	bodyContent := cmd.Params["body"]

	//将businessName 加入到body中
	bodyMap := bodyContent.(map[string]interface{})
	bodyMap["business_type"] = businessName

	body, err := json.Marshal(bodyContent)
	if nil != err {
		log.Printf("marshal log body fail %s", err.Error())
		return businessName, nil
	}

	//拼Body
	flumeBody := fmt.Sprintf("%s\t%s\t%s", momoid, action, string(body))
	obj := client.NewFlumeEvent()
	event := client.EventFillUp(obj, businessName, action, []byte(flumeBody))
	return businessName, event
}

func (self *SourceServer) stop() {
	self.isStop = true
	time.Sleep(5 * time.Second)

	//遍历所有的flumeclientlink，将当前Business从该链表中移除
	for v := self.flumeClientPool.Back(); nil != v; v = v.Prev() {
		v.Value.(*pool.FlumePoolLink).DetachBusiness(self.business)
	}
	close(self.buffChannel)
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER|[%s]|STOPPED\n", self.business)
}

func (self *SourceServer) getFlumeClientPool() *pool.FlumeClientPool {

	//采用轮训算法
	e := self.flumeClientPool.Back()
	if nil == e {
		return nil
	}
	self.flumeClientPool.MoveToFront(e)
	return e.Value.(*pool.FlumePoolLink).FlumePool

}
<|endoftext|>


<commit_before>package task

import (
	"neon/build"
	"reflect"
	"sync"
)

func init() {
	build.AddTask(build.TaskDesc{
		Name: "threads",
		Func: threads,
		Args: reflect.TypeOf(threadsArgs{}),
		Help: `Run steps in threads.

Arguments:

- threads: number of threads to run (integer).
- input: values to pass to threads in _input property (list, optional).
- steps: steps to run in threads (steps).
- verbose: if you want thread information on console, defaults to false
  (boolean, optional).

Examples:

    # compute squares of 10 first integers in threads and put them in _output
    - threads: =_NCPU
      input:   =range(10)
      steps:
      - '_output = _input * _input'
      - print: '#{_input}^2 = #{_output}'
    # print squares on the console
    - print: '#{_output}'

Notes:

- You might set number of threads to '_NCPU' which is the number of cores in
  the CPU of the machine.
- Property _thread is set with the thread number (starting with 0)
- Property _input is set with the input for each thread.
- Property _output is set with the output of the threads.
- Each thread should write its output in property _output.

Context of the build is cloned in each thread so that you can read and write
properties, they won't affect other threads. But all properties will be lost
when thread is done, except for _output that will be appended to other in
resulting _output property.

Don't change current directory in threads as it would affect other threads as
well.`,
	})
}

type threadsArgs struct {
	Threads int
	Input   []interface{} `neon:"optional"`
	Steps   build.Steps   `neon:"steps"`
	Verbose bool          `neon:"optional"`
}

func threads(context *build.Context, args interface{}) error {
	params := args.(threadsArgs)
	input := make(chan interface{}, len(params.Input))
	for _, d := range params.Input {
		input <- d
	}
	error := make(chan error, params.Threads)
	var wg sync.WaitGroup
	wg.Add(params.Threads)
	if params.Verbose {
		context.Message("Starting %d threads", params.Threads)
	}
	output := make(chan interface{}, len(input))
	for i := 0; i < params.Threads; i++ {
		go runThread(params.Steps, context, i, input, output, &wg, error, params.Verbose)
	}
	wg.Wait()
	var out []interface{}
	stop := false
	for !stop {
		select {
		case o, ok := <-output:
			if ok {
				out = append(out, o)
			} else {
				stop = true
			}
		default:
			stop = true
		}
	}
	context.SetProperty("_output", out)
	select {
	case e, ok := <-error:
		if ok {
			return e
		}
		return nil
	default:
		return nil
	}
}

func runThread(steps build.Steps, ctx *build.Context, index int, input chan interface{}, output chan interface{},
	wg *sync.WaitGroup, errors chan error, verbose bool) {
	if verbose {
		ctx.Message("Thread %d started", index)
		defer ctx.Message("Thread %d done", index)
	}
	defer wg.Done()
	for {
		select {
		case arg, ok := <-input:
			if ok {
				threadContext := ctx.NewThreadContext(index, arg, output)
				if verbose {
					threadContext.Message("Thread %d iteration with input '%v'", index, arg)
				}
				err := steps.Run(threadContext)
				out, _ := threadContext.GetProperty("_output")
				if err != nil {
					errors <- err
					return
				}
				if out != nil {
					output <- out
					if verbose {
						threadContext.Message("Thread %d output '%v'", index, out)
					}
				}
			} else {
				return
			}
		default:
			return
		}
	}
}
<commit_msg>Fixed threads when no input<commit_after>package task

import (
	"neon/build"
	"reflect"
	"sync"
)

func init() {
	build.AddTask(build.TaskDesc{
		Name: "threads",
		Func: threads,
		Args: reflect.TypeOf(threadsArgs{}),
		Help: `Run steps in threads.

Arguments:

- threads: number of threads to run (integer).
- input: values to pass to threads in _input property (list, optional).
- steps: steps to run in threads (steps).
- verbose: if you want thread information on console, defaults to false
  (boolean, optional).

Examples:

    # compute squares of 10 first integers in threads and put them in _output
    - threads: =_NCPU
      input:   =range(10)
      steps:
      - '_output = _input * _input'
      - print: '#{_input}^2 = #{_output}'
    # print squares on the console
    - print: '#{_output}'

Notes:

- You might set number of threads to '_NCPU' which is the number of cores in
  the CPU of the machine.
- Property _thread is set with the thread number (starting with 0)
- Property _input is set with the input for each thread.
- Property _output is set with the output of the threads.
- Each thread should write its output in property _output.

Context of the build is cloned in each thread so that you can read and write
properties, they won't affect other threads. But all properties will be lost
when thread is done, except for _output that will be appended to other in
resulting _output property.

Don't change current directory in threads as it would affect other threads as
well.`,
	})
}

type threadsArgs struct {
	Threads int
	Input   []interface{} `neon:"optional"`
	Steps   build.Steps   `neon:"steps"`
	Verbose bool          `neon:"optional"`
}

func threads(context *build.Context, args interface{}) error {
	params := args.(threadsArgs)
	if params.Input == nil {
		params.Input = make([]interface{}, params.Threads)
	}
	input := make(chan interface{}, len(params.Input))
	for _, d := range params.Input {
		input <- d
	}
	error := make(chan error, params.Threads)
	var wg sync.WaitGroup
	wg.Add(params.Threads)
	if params.Verbose {
		context.Message("Starting %d threads", params.Threads)
	}
	output := make(chan interface{}, len(input))
	for i := 0; i < params.Threads; i++ {
		go runThread(params.Steps, context, i, input, output, &wg, error, params.Verbose)
	}
	wg.Wait()
	var out []interface{}
	stop := false
	for !stop {
		select {
		case o, ok := <-output:
			if ok {
				out = append(out, o)
			} else {
				stop = true
			}
		default:
			stop = true
		}
	}
	context.SetProperty("_output", out)
	select {
	case e, ok := <-error:
		if ok {
			return e
		}
		return nil
	default:
		return nil
	}
}

func runThread(steps build.Steps, ctx *build.Context, index int, input chan interface{}, output chan interface{},
	wg *sync.WaitGroup, errors chan error, verbose bool) {
	if verbose {
		ctx.Message("Thread %d started", index)
		defer ctx.Message("Thread %d done", index)
	}
	defer wg.Done()
	for {
		select {
		case arg, ok := <-input:
			if ok {
				threadContext := ctx.NewThreadContext(index, arg, output)
				if verbose {
					threadContext.Message("Thread %d iteration with input '%v'", index, arg)
				}
				err := steps.Run(threadContext)
				out, _ := threadContext.GetProperty("_output")
				if err != nil {
					errors <- err
					return
				}
				if out != nil {
					output <- out
					if verbose {
						threadContext.Message("Thread %d output '%v'", index, out)
					}
				}
			} else {
				return
			}
		default:
			return
		}
	}
}
<|endoftext|>


<commit_before>package service

import (
	"os"
	"strings"
	"sync"

	pdebug "github.com/lestrrat/go-pdebug"
	"github.com/pkg/errors"
	mailgun "gopkg.in/mailgun/mailgun-go.v1"
)

var mailgunSvc *MailgunSvc
var mailgunOnce sync.Once

func Mailgun() *MailgunSvc {
	mailgunOnce.Do(mailgunSvc.Init)
	return mailgunSvc
}

func (v *MailgunSvc) Init() {
	if pdebug.Enabled {
		g := pdebug.Marker("service.Mailgun.Init")
		defer g.End()
	}

	v = &MailgunSvc{}

	f := func(v *string, envname string) {
		envvar := os.Getenv(envname)
		if envvar == "" {
			panic("Missing required environment variable " + envname)
		}
		*v = envvar
	}

	f(&v.defaultSender, "MAILGUN_DEFAULT_SENDER")

	var domain string
	var apiKey string
	var publicApiKey string
	f(&domain, "MAILGUN_DOMAIN")
	f(&apiKey, "MAILGUN_SECRET_API_KEY")
	f(&publicApiKey, "MAILGUN_PUBLIC_API_KEY")

	if pdebug.Enabled {
		pdebug.Printf(
			"Creating Mailgun client with domain=%s, apiKey=%s, publicApiKey=%s",
			domain,
			strings.Repeat("*", len(apiKey)-4)+apiKey[len(apiKey)-4:],
			strings.Repeat("*", len(publicApiKey)-4)+publicApiKey[len(publicApiKey)-4:],
		)
	}

	v.client = mailgun.NewMailgun(domain, apiKey, publicApiKey)
}

type MailMessage struct {
	From       string
	Subject    string
	Text       string
	Recipients []string
}

func (v *MailgunSvc) Send(mm *MailMessage) (err error) {
	if pdebug.Enabled {
		g := pdebug.Marker("service.Mailgun.Send").BindError(&err)
		defer g.End()
	}

	if mm.From == "" {
		mm.From = v.defaultSender
	}

	m := mailgun.NewMessage(mm.From, mm.Subject, mm.Text, mm.Recipients...)

	mg := v.client
	_, _, err = mg.Send(m)
	if err != nil {
		return errors.Wrap(err, "failed to send message")
	}
	return nil
}
<commit_msg>one more similar fix<commit_after>package service

import (
	"os"
	"strings"
	"sync"

	pdebug "github.com/lestrrat/go-pdebug"
	"github.com/pkg/errors"
	mailgun "gopkg.in/mailgun/mailgun-go.v1"
)

var mailgunSvc MailgunSvc
var mailgunOnce sync.Once

func Mailgun() *MailgunSvc {
	mailgunOnce.Do(mailgunSvc.Init)
	return &mailgunSvc
}

func (v *MailgunSvc) Init() {
	if pdebug.Enabled {
		g := pdebug.Marker("service.Mailgun.Init")
		defer g.End()
	}

	f := func(v *string, envname string) {
		envvar := os.Getenv(envname)
		if envvar == "" {
			panic("Missing required environment variable " + envname)
		}
		*v = envvar
	}

	f(&v.defaultSender, "MAILGUN_DEFAULT_SENDER")

	var domain string
	var apiKey string
	var publicApiKey string
	f(&domain, "MAILGUN_DOMAIN")
	f(&apiKey, "MAILGUN_SECRET_API_KEY")
	f(&publicApiKey, "MAILGUN_PUBLIC_API_KEY")

	if pdebug.Enabled {
		pdebug.Printf(
			"Creating Mailgun client with domain=%s, apiKey=%s, publicApiKey=%s",
			domain,
			strings.Repeat("*", len(apiKey)-4)+apiKey[len(apiKey)-4:],
			strings.Repeat("*", len(publicApiKey)-4)+publicApiKey[len(publicApiKey)-4:],
		)
	}

	v.client = mailgun.NewMailgun(domain, apiKey, publicApiKey)
}

type MailMessage struct {
	From       string
	Subject    string
	Text       string
	Recipients []string
}

func (v *MailgunSvc) Send(mm *MailMessage) (err error) {
	if pdebug.Enabled {
		g := pdebug.Marker("service.Mailgun.Send").BindError(&err)
		defer g.End()
	}

	if mm.From == "" {
		mm.From = v.defaultSender
	}

	m := mailgun.NewMessage(mm.From, mm.Subject, mm.Text, mm.Recipients...)

	mg := v.client
	_, _, err = mg.Send(m)
	if err != nil {
		return errors.Wrap(err, "failed to send message")
	}
	return nil
}
<|endoftext|>


<commit_before>package rabbitsmpp

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/streadway/amqp"
	"golang.org/x/net/context"
)

const (
	defaultPrefetchCount = 20
	defaultPrefetchSize  = 0
	defaultGlobalQos     = false
)

type ConsumeOptionSetter func(*consumeOptions)

type consumeOptions struct {
	prefetchCount int
	prefetchSize  int
	globalQos     bool
}

func SetPrefetchCount(n int) ConsumeOptionSetter {
	return func(o *consumeOptions) {
		o.prefetchCount = n
	}
}

func SetPrefetchSize(n int) ConsumeOptionSetter {
	return func(o *consumeOptions) {
		o.prefetchSize = n
	}
}

func SetGlobalQos(a bool) ConsumeOptionSetter {
	return func(o *consumeOptions) {
		o.globalQos = a
	}
}

type Consumer interface {
	Consume() (<-chan Job, <-chan error, error)
	Stop() error
	ID() string
}

type consumer struct {
	Client
	clientFactory ClientFactory
	channel       Channel
	ctx           context.Context
	cancel        context.CancelFunc
	prefetchCount int
	prefetchSize  int
	globalQos     bool
	queueName     string
	m             *sync.RWMutex
}

func buildConsumeOptions(options ...ConsumeOptionSetter) *consumeOptions {
	o := &consumeOptions{
		prefetchCount: defaultPrefetchCount,
		prefetchSize:  defaultPrefetchSize,
		globalQos:     defaultGlobalQos,
	}
	for _, option := range options {
		option(o)
	}
	return o
}

func NewConsumer(conf Config, options ...ConsumeOptionSetter) (Consumer, error) {
	clientFactory := defaultClientFactory(conf)
	ctx, _ := context.WithCancel(context.Background())

	return NewConsumerWithContext(conf.QueueName, ctx, clientFactory, options...)
}

func NewConsumerWithContext(queueName string, ctx context.Context, clientFactory ClientFactory, options ...ConsumeOptionSetter) (Consumer, error) {
	client, err := clientFactory()
	if err != nil {
		return nil, err
	}

	ctx, cancel := context.WithCancel(ctx)
	o := buildConsumeOptions(options...)

	return &consumer{
		Client:        client,
		clientFactory: clientFactory,
		ctx:           ctx,
		cancel:        cancel,
		prefetchCount: o.prefetchCount,
		prefetchSize:  o.prefetchSize,
		globalQos:     o.globalQos,
		queueName:     queueName,
		m:             &sync.RWMutex{},
	}, nil
}

func (c *consumer) ID() string {
	return c.queueName
}

func (c *consumer) waitOnClosedClient() {
	client, err := c.clientFactory()
	for err != nil {
		log.Println("Failed to recreate client:", err)
		time.Sleep(5 * time.Second)
		client, err = c.clientFactory()
	}
	c.Client = client
}

func (c *consumer) getConsumeChannel() (<-chan amqp.Delivery, error) {
	c.m.Lock()
	defer c.m.Unlock()

	ch, err := c.Channel()
	if err != nil {
		return nil, err
	}
	err = ch.Qos(c.prefetchCount, c.prefetchSize, c.globalQos)
	if err != nil {
		return nil, err
	}

	c.channel = ch

	q, err := c.channel.QueueDeclare(
		c.queueName, // name
		true,        // durable
		false,       // delete when unused
		false,       // exclusive
		false,       // no-wait
		nil,         // arguments
	)
	if err != nil {
		return nil, err
	}

	return c.channel.Consume(
		q.Name, // queue
		"",     // consumer
		false,  // auto-ackey
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
}

func (c *consumer) Consume() (<-chan Job, <-chan error, error) {
	if c.getChannel() != nil {
		return nil, nil, errors.New("consumer already active")
	}
	closeChan := c.Client.GetCloseChan()
	dlvChan, err := c.getConsumeChannel()
	if err != nil {
		return nil, nil, err
	}
	jobChan := make(chan Job)
	errChan := make(chan error)

	go func() {
		defer func() {
			c.m.Lock()
			defer c.m.Unlock()
			if c.channel != nil {
				_ = c.channel.Close()
				c.channel = nil
			}

			close(jobChan)
			close(errChan)
		}()

		for {
			err = c.consume(dlvChan, closeChan, jobChan)
			// if consume returns without an error, means that it was terminated
			// properly, otherwise something went wrong and it needs to restart
			if err == nil {
				log.Printf("EOF consuming for: %s", c.ID())
				return
			}
			log.Println("stopped consuming jobs:", err)

			// we need this because sometimes we don't have a listener here so we don't
			// want to block the whole consuming because we weren't able to send an error
			select {
			case errChan <- err:
			default:
				log.Println("no listener errChan skipping")
			}

			c.waitOnClosedClient()
			closeChan = c.Client.GetCloseChan()
			dlvChan, err = c.getConsumeChannel()
			for err != nil {
				time.Sleep(5 * time.Second)
				dlvChan, err = c.getConsumeChannel()
			}
		}
	}()

	return jobChan, errChan, nil
}

func (c *consumer) consume(dlvChan <-chan amqp.Delivery, closeChan <-chan *amqp.Error, jobChan chan<- Job) error {
	for {
		select {
		case d, ok := <-dlvChan:
			if !ok {
				log.Printf("deliver chan is closed, returning")
				return nil
			}
			j := Job{}
			err := json.Unmarshal(d.Body, &j)
			if err != nil {
				return fmt.Errorf("failed to unmarshal PDU: %v", err)
			}
			j.delivery = &d
			jobChan <- j
		case err := <-closeChan:
			return err
		case <-c.ctx.Done():
			return nil
		}
	}
}

func (c *consumer) Stop() error {
	if c.getChannel() == nil {
		return nil
	}
	// Sends the stop signal
	c.cancel()
	return nil
}

func (c *consumer) getChannel() Channel {
	c.m.RLock()
	defer c.m.RUnlock()
	return c.channel
}
<commit_msg>change-defaultPrefetchCount<commit_after>package rabbitsmpp

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/streadway/amqp"
	"golang.org/x/net/context"
)

const (
	defaultPrefetchCount = 0
	defaultPrefetchSize  = 0
	defaultGlobalQos     = false
)

type ConsumeOptionSetter func(*consumeOptions)

type consumeOptions struct {
	prefetchCount int
	prefetchSize  int
	globalQos     bool
}

func SetPrefetchCount(n int) ConsumeOptionSetter {
	return func(o *consumeOptions) {
		o.prefetchCount = n
	}
}

func SetPrefetchSize(n int) ConsumeOptionSetter {
	return func(o *consumeOptions) {
		o.prefetchSize = n
	}
}

func SetGlobalQos(a bool) ConsumeOptionSetter {
	return func(o *consumeOptions) {
		o.globalQos = a
	}
}

type Consumer interface {
	Consume() (<-chan Job, <-chan error, error)
	Stop() error
	ID() string
}

type consumer struct {
	Client
	clientFactory ClientFactory
	channel       Channel
	ctx           context.Context
	cancel        context.CancelFunc
	prefetchCount int
	prefetchSize  int
	globalQos     bool
	queueName     string
	m             *sync.RWMutex
}

func buildConsumeOptions(options ...ConsumeOptionSetter) *consumeOptions {
	o := &consumeOptions{
		prefetchCount: defaultPrefetchCount,
		prefetchSize:  defaultPrefetchSize,
		globalQos:     defaultGlobalQos,
	}
	for _, option := range options {
		option(o)
	}
	return o
}

func NewConsumer(conf Config, options ...ConsumeOptionSetter) (Consumer, error) {
	clientFactory := defaultClientFactory(conf)
	ctx, _ := context.WithCancel(context.Background())

	return NewConsumerWithContext(conf.QueueName, ctx, clientFactory, options...)
}

func NewConsumerWithContext(queueName string, ctx context.Context, clientFactory ClientFactory, options ...ConsumeOptionSetter) (Consumer, error) {
	client, err := clientFactory()
	if err != nil {
		return nil, err
	}

	ctx, cancel := context.WithCancel(ctx)
	o := buildConsumeOptions(options...)

	return &consumer{
		Client:        client,
		clientFactory: clientFactory,
		ctx:           ctx,
		cancel:        cancel,
		prefetchCount: o.prefetchCount,
		prefetchSize:  o.prefetchSize,
		globalQos:     o.globalQos,
		queueName:     queueName,
		m:             &sync.RWMutex{},
	}, nil
}

func (c *consumer) ID() string {
	return c.queueName
}

func (c *consumer) waitOnClosedClient() {
	client, err := c.clientFactory()
	for err != nil {
		log.Println("Failed to recreate client:", err)
		time.Sleep(5 * time.Second)
		client, err = c.clientFactory()
	}
	c.Client = client
}

func (c *consumer) getConsumeChannel() (<-chan amqp.Delivery, error) {
	c.m.Lock()
	defer c.m.Unlock()

	ch, err := c.Channel()
	if err != nil {
		return nil, err
	}
	err = ch.Qos(c.prefetchCount, c.prefetchSize, c.globalQos)
	if err != nil {
		return nil, err
	}

	c.channel = ch

	q, err := c.channel.QueueDeclare(
		c.queueName, // name
		true,        // durable
		false,       // delete when unused
		false,       // exclusive
		false,       // no-wait
		nil,         // arguments
	)
	if err != nil {
		return nil, err
	}

	return c.channel.Consume(
		q.Name, // queue
		"",     // consumer
		false,  // auto-ackey
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
}

func (c *consumer) Consume() (<-chan Job, <-chan error, error) {
	if c.getChannel() != nil {
		return nil, nil, errors.New("consumer already active")
	}
	closeChan := c.Client.GetCloseChan()
	dlvChan, err := c.getConsumeChannel()
	if err != nil {
		return nil, nil, err
	}
	jobChan := make(chan Job)
	errChan := make(chan error)

	go func() {
		defer func() {
			c.m.Lock()
			defer c.m.Unlock()
			if c.channel != nil {
				_ = c.channel.Close()
				c.channel = nil
			}

			close(jobChan)
			close(errChan)
		}()

		for {
			err = c.consume(dlvChan, closeChan, jobChan)
			// if consume returns without an error, means that it was terminated
			// properly, otherwise something went wrong and it needs to restart
			if err == nil {
				log.Printf("EOF consuming for: %s", c.ID())
				return
			}
			log.Println("stopped consuming jobs:", err)

			// we need this because sometimes we don't have a listener here so we don't
			// want to block the whole consuming because we weren't able to send an error
			select {
			case errChan <- err:
			default:
				log.Println("no listener errChan skipping")
			}

			c.waitOnClosedClient()
			closeChan = c.Client.GetCloseChan()
			dlvChan, err = c.getConsumeChannel()
			for err != nil {
				time.Sleep(5 * time.Second)
				dlvChan, err = c.getConsumeChannel()
			}
		}
	}()

	return jobChan, errChan, nil
}

func (c *consumer) consume(dlvChan <-chan amqp.Delivery, closeChan <-chan *amqp.Error, jobChan chan<- Job) error {
	for {
		select {
		case d, ok := <-dlvChan:
			if !ok {
				log.Printf("deliver chan is closed, returning")
				return nil
			}
			j := Job{}
			err := json.Unmarshal(d.Body, &j)
			if err != nil {
				return fmt.Errorf("failed to unmarshal PDU: %v", err)
			}
			j.delivery = &d
			jobChan <- j
		case err := <-closeChan:
			return err
		case <-c.ctx.Done():
			return nil
		}
	}
}

func (c *consumer) Stop() error {
	if c.getChannel() == nil {
		return nil
	}
	// Sends the stop signal
	c.cancel()
	return nil
}

func (c *consumer) getChannel() Channel {
	c.m.RLock()
	defer c.m.RUnlock()
	return c.channel
}
<|endoftext|>


<commit_before>package lookup

import (
	"fmt"
	"koding/kites/kloud/pkg/multiec2"
	"sync"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/ec2"
)

type Lookup struct {
	// values contains a list of instance tags that are identified as test
	// instances. By default all instances are fetched.
	values []string

	clients *multiec2.Clients
}

func NewAWS(auth aws.Auth) *Lookup {
	return &Lookup{
		clients: multiec2.New(auth, []string{
			"us-east-1",
			"ap-southeast-1",
			"us-west-2",
			"eu-west-1",
		}),
	}
}

// Instances returns all instances that belongs to the given client/region if
func (l *Lookup) Instances(client *ec2.EC2) (Instances, error) {
	instances := make([]ec2.Instance, 0)

	resp, err := client.InstancesPaginate(500, "")
	if err != nil {
		return nil, err
	}

	for _, res := range resp.Reservations {
		instances = append(instances, res.Instances...)
	}

	nextToken := resp.NextToken

	// get all results until nextToken is empty
	for nextToken != "" {
		resp, err := client.InstancesPaginate(0, nextToken)
		if err != nil {
			return nil, err
		}

		for _, res := range resp.Reservations {
			instances = append(instances, res.Instances...)
		}

		nextToken = resp.NextToken
	}

	m := make(Instances, len(instances))

	for _, instance := range instances {
		m[instance.InstanceId] = instance
	}

	return m, nil
}

// FetchInstances fetches all instances from all regions
func (l *Lookup) FetchInstances() *MultiInstances {
	var wg sync.WaitGroup

	allInstances := NewMultiInstances()

	for region, client := range l.clients.Regions() {
		wg.Add(1)
		go func(region string, client *ec2.EC2) {
			defer wg.Done()

			instances, err := l.Instances(client)
			if err != nil {
				fmt.Printf("[%s] fetching error: %s\n", region, err)
				return
			}

			allInstances.Add(client, instances)
		}(region, client)
	}

	wg.Wait()

	return allInstances
}

// Volumes returns all volumes that belongs to the given client/region if
func (l *Lookup) Volumes(client *ec2.EC2) (Volumes, error) {
	volumes := make([]ec2.Volume, 0)

	resp, err := client.VolumesPages(500, "")
	if err != nil {
		return nil, err
	}

	for _, volume := range resp.Volumes {
		volumes = append(volumes, volume)
	}

	nextToken := resp.NextToken

	// get all results until nextToken is empty
	for nextToken != "" {
		resp, err := client.VolumesPages(0, nextToken)
		if err != nil {
			return nil, err
		}

		for _, volume := range resp.Volumes {
			volumes = append(volumes, volume)
		}

		nextToken = resp.NextToken
	}

	m := make(Volumes, len(volumes))

	for _, volume := range volumes {
		m[volume.VolumeId] = volume
	}

	return m, nil
}

// FetchVolumes fetches all instances from all regions
func (l *Lookup) FetchVolumes() MultiVolumes {
	var wg sync.WaitGroup

	allVolumes := make(MultiVolumes, 0)

	for region, client := range l.clients.Regions() {
		wg.Add(1)
		go func(region string, client *ec2.EC2) {
			defer wg.Done()

			volumes, err := l.Volumes(client)
			if err != nil {
				fmt.Printf("[%s] fetching error: %s\n", region, err)
				return
			}

			allVolumes[client] = volumes
		}(region, client)
	}

	wg.Wait()

	return allVolumes
}
<commit_msg>cleaners: fix volumes API call<commit_after>package lookup

import (
	"fmt"
	"koding/kites/kloud/pkg/multiec2"
	"sync"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/ec2"
)

type Lookup struct {
	// values contains a list of instance tags that are identified as test
	// instances. By default all instances are fetched.
	values []string

	clients *multiec2.Clients
}

func NewAWS(auth aws.Auth) *Lookup {
	return &Lookup{
		clients: multiec2.New(auth, []string{
			"us-east-1",
			"ap-southeast-1",
			"us-west-2",
			"eu-west-1",
		}),
	}
}

// Instances returns all instances that belongs to the given client/region if
func (l *Lookup) Instances(client *ec2.EC2) (Instances, error) {
	instances := make([]ec2.Instance, 0)

	resp, err := client.InstancesPaginate(500, "")
	if err != nil {
		return nil, err
	}

	for _, res := range resp.Reservations {
		instances = append(instances, res.Instances...)
	}

	nextToken := resp.NextToken

	// get all results until nextToken is empty
	for nextToken != "" {
		resp, err := client.InstancesPaginate(0, nextToken)
		if err != nil {
			return nil, err
		}

		for _, res := range resp.Reservations {
			instances = append(instances, res.Instances...)
		}

		nextToken = resp.NextToken
	}

	m := make(Instances, len(instances))

	for _, instance := range instances {
		m[instance.InstanceId] = instance
	}

	return m, nil
}

// FetchInstances fetches all instances from all regions
func (l *Lookup) FetchInstances() *MultiInstances {
	var wg sync.WaitGroup

	allInstances := NewMultiInstances()

	for region, client := range l.clients.Regions() {
		wg.Add(1)
		go func(region string, client *ec2.EC2) {
			defer wg.Done()

			instances, err := l.Instances(client)
			if err != nil {
				fmt.Printf("[%s] fetching error: %s\n", region, err)
				return
			}

			allInstances.Add(client, instances)
		}(region, client)
	}

	wg.Wait()

	return allInstances
}

// Volumes returns all volumes that belongs to the given client/region if
func (l *Lookup) Volumes(client *ec2.EC2) (Volumes, error) {
	volumes := make([]ec2.Volume, 0)

	resp, err := client.VolumesPaginate(500, "")
	if err != nil {
		return nil, err
	}

	for _, volume := range resp.Volumes {
		volumes = append(volumes, volume)
	}

	nextToken := resp.NextToken

	// get all results until nextToken is empty
	for nextToken != "" {
		resp, err := client.VolumesPaginate(0, nextToken)
		if err != nil {
			return nil, err
		}

		for _, volume := range resp.Volumes {
			volumes = append(volumes, volume)
		}

		nextToken = resp.NextToken
	}

	m := make(Volumes, len(volumes))

	for _, volume := range volumes {
		m[volume.VolumeId] = volume
	}

	return m, nil
}

// FetchVolumes fetches all instances from all regions
func (l *Lookup) FetchVolumes() MultiVolumes {
	var wg sync.WaitGroup

	allVolumes := make(MultiVolumes, 0)

	for region, client := range l.clients.Regions() {
		wg.Add(1)
		go func(region string, client *ec2.EC2) {
			defer wg.Done()

			volumes, err := l.Volumes(client)
			if err != nil {
				fmt.Printf("[%s] fetching error: %s\n", region, err)
				return
			}

			allVolumes[client] = volumes
		}(region, client)
	}

	wg.Wait()

	return allVolumes
}
<|endoftext|>


<commit_before>package driver

import (
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

// キャッシュ用。
type fileRawDataStore struct {
	path      string
	keyToPath func(string) string
	staleDur  time.Duration
	expiDur   time.Duration
}

// スレッドセーフではない。
func newFileRawDataStore(path string, keyToPath func(string) string, staleDur, expiDur time.Duration) *fileRawDataStore {
	if keyToPath == nil {
		keyToPath = func(key string) string { return key }
	}
	return &fileRawDataStore{path, keyToPath, staleDur, expiDur}
}

func (reg *fileRawDataStore) getStamp(fi os.FileInfo) *Stamp {
	now := time.Now()
	stmp := getFileStamp(fi)
	stmp.StaleDate = now.Add(reg.staleDur)
	stmp.ExpiDate = now.Add(reg.expiDur)
	return stmp
}

func (reg *fileRawDataStore) Get(key string, caStmp *Stamp) (data []byte, newCaStmp *Stamp, err error) {
	path := filepath.Join(reg.path, reg.keyToPath(key))

	fi, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, nil
		} else {
			return nil, nil, erro.Wrap(err)
		}
	}

	newCaStmp = reg.getStamp(fi)

	// 対象のスタンプを取得。

	if caStmp != nil && !caStmp.Older(newCaStmp) {
		// 要求元のキャッシュより新しそうではなかった。
		return nil, newCaStmp, nil
	}

	// 要求元のキャッシュより新しそう。

	data, err = ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, nil
		}
		return nil, nil, erro.Wrap(err)
	}
	return data, newCaStmp, nil
}

func (reg *fileRawDataStore) Put(key string, data []byte) (*Stamp, error) {
	path := filepath.Join(reg.path, reg.keyToPath(key))

	f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, filePerm)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, erro.Wrap(err)
		}

		// ディレクトリが無かっただけかもしれないので、
		// ディレクトリを掘って再挑戦。
		if err := os.MkdirAll(filepath.Dir(path), dirPerm); err != nil {
			return nil, erro.Wrap(err)
		}
		f, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, filePerm)
		if err != nil {
			return nil, erro.Wrap(err)
		}
	}
	defer f.Close()

	if _, err := f.Write(data); err != nil {
		return nil, erro.Wrap(err)
	}

	// 保存できた。

	fi, err := f.Stat()
	if err != nil {
		return nil, erro.Wrap(err)
	}

	return reg.getStamp(fi), nil
}

func (reg *fileRawDataStore) Remove(key string) error {
	path := filepath.Join(reg.path, reg.keyToPath(key))

	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
		return erro.Wrap(err)
	}
	return nil
}
<commit_msg>ファイルの中身が小さくなったときの切り詰め忘れを修正<commit_after>package driver

import (
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

// キャッシュ用。
type fileRawDataStore struct {
	path      string
	keyToPath func(string) string
	staleDur  time.Duration
	expiDur   time.Duration
}

// スレッドセーフではない。
func newFileRawDataStore(path string, keyToPath func(string) string, staleDur, expiDur time.Duration) *fileRawDataStore {
	if keyToPath == nil {
		keyToPath = func(key string) string { return key }
	}
	return &fileRawDataStore{path, keyToPath, staleDur, expiDur}
}

func (reg *fileRawDataStore) getStamp(fi os.FileInfo) *Stamp {
	now := time.Now()
	stmp := getFileStamp(fi)
	stmp.StaleDate = now.Add(reg.staleDur)
	stmp.ExpiDate = now.Add(reg.expiDur)
	return stmp
}

func (reg *fileRawDataStore) Get(key string, caStmp *Stamp) (data []byte, newCaStmp *Stamp, err error) {
	path := filepath.Join(reg.path, reg.keyToPath(key))

	fi, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, nil
		} else {
			return nil, nil, erro.Wrap(err)
		}
	}

	newCaStmp = reg.getStamp(fi)

	// 対象のスタンプを取得。

	if caStmp != nil && !caStmp.Older(newCaStmp) {
		// 要求元のキャッシュより新しそうではなかった。
		return nil, newCaStmp, nil
	}

	// 要求元のキャッシュより新しそう。

	data, err = ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, nil
		}
		return nil, nil, erro.Wrap(err)
	}
	return data, newCaStmp, nil
}

func (reg *fileRawDataStore) Put(key string, data []byte) (*Stamp, error) {
	path := filepath.Join(reg.path, reg.keyToPath(key))

	f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, filePerm)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, erro.Wrap(err)
		}

		// ディレクトリが無かっただけかもしれないので、
		// ディレクトリを掘って再挑戦。
		if err := os.MkdirAll(filepath.Dir(path), dirPerm); err != nil {
			return nil, erro.Wrap(err)
		}
		f, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, filePerm)
		if err != nil {
			return nil, erro.Wrap(err)
		}
	}
	defer f.Close()

	n, err := f.Write(data)
	if err != nil {
		return nil, erro.Wrap(err)
	}

	// 保存できた。

	fi, err := f.Stat()
	if err != nil {
		return nil, erro.Wrap(err)
	} else if int64(n) < fi.Size() {
		// 前の内容の方が大きかった。
		if err := f.Truncate(int64(n)); err != nil {
			return nil, erro.Wrap(err)
		}
		fi, err = f.Stat()
		if err != nil {
			return nil, erro.Wrap(err)
		}
	}

	return reg.getStamp(fi), nil
}

func (reg *fileRawDataStore) Remove(key string) error {
	path := filepath.Join(reg.path, reg.keyToPath(key))

	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
		return erro.Wrap(err)
	}
	return nil
}
<|endoftext|>


<commit_before>// +build windows

package windows

import (
	"testing"
)

func TestMemoryKey(t *testing.T) {
	g := &MemoryGenerator{}

	if g.Key() != "memory" {
		t.Error("key should be memory")
	}
}

func TestMemoryGenerator(t *testing.T) {
	g := &MemoryGenerator{}
	value, err := g.Generate()
	if err != nil {
		t.Errorf("should not raise error: %v", err)
	}

	memory, typeOk := value.(map[string]interface{})
	if !typeOk {
		t.Errorf("value should be map. %+v", value)
	}

	if _, ok := memory["total"]; !ok {
		t.Error("memory should have total")
	}

	if _, ok := memory["free"]; !ok {
		t.Error("memory should have free")
	}

	if _, ok := memory["buffers"]; !ok {
		//t.Error("memory should has buffers")
	}

	if _, ok := memory["cached"]; !ok {
		//t.Error("memory should has cached")
	}

	if _, ok := memory["active"]; !ok {
		//t.Error("memory should has active")
	}

	if _, ok := memory["inactive"]; !ok {
		//t.Error("memory should has inactive")
	}

	if _, ok := memory["high_total"]; !ok {
		//t.Log("Skip: memory should has high_total")
	}

	if _, ok := memory["high_free"]; !ok {
		//t.Log("Skip: memory should has high_free")
	}

	if _, ok := memory["low_total"]; !ok {
		//t.Log("Skip: memory should has low_tatal")
	}

	if _, ok := memory["low_free"]; !ok {
		//t.Log("Skip: memory should has low_free")
	}

	if _, ok := memory["dirty"]; !ok {
		//t.Error("memory should has dirty")
	}

	if _, ok := memory["writeback"]; !ok {
		//t.Error("memory should has writeback")
	}

	if _, ok := memory["anon_pages"]; !ok {
		//t.Error("memory should has anon_pages")
	}

	if _, ok := memory["mapped"]; !ok {
		//t.Error("memory should has mapped")
	}

	if _, ok := memory["slab"]; !ok {
		//t.Error("memory should has slab")
	}

	if _, ok := memory["slab_reclaimable"]; !ok {
		//t.Error("memory should has slab_reclaimable")
	}

	if _, ok := memory["slab_unreclaim"]; !ok {
		//t.Error("memory should has slab_unreclaim")
	}

	if _, ok := memory["page_tables"]; !ok {
		//t.Error("memory should has page_tables")
	}

	if _, ok := memory["nfs_unstable"]; !ok {
		//t.Error("memory should has nfs_unstable")
	}

	if _, ok := memory["bounce"]; !ok {
		//t.Error("memory should has bounce")
	}

	if _, ok := memory["commit_limit"]; !ok {
		//t.Error("memory should has commit_limmit")
	}

	if _, ok := memory["committed_as"]; !ok {
		//t.Error("memory should has committed_as")
	}

	if _, ok := memory["vmalloc_total"]; !ok {
		//t.Error("memory should has vmalloc_total")
	}

	if _, ok := memory["vmalloc_used"]; !ok {
		//t.Error("memory should has vmalloc_used")
	}

	if _, ok := memory["vmalloc_chunk"]; !ok {
		//t.Error("memory should has vmalloc_chunk")
	}

	if _, ok := memory["swap_cached"]; !ok {
		//t.Error("memory should has swap_cached")
	}

	if _, ok := memory["swap_total"]; !ok {
		//t.Error("memory should has swap_total")
	}

	if _, ok := memory["swap_free"]; !ok {
		//t.Error("memory should has swap_free")
	}
}
<commit_msg>remove skipped tests on Windows<commit_after>// +build windows

package windows

import (
	"testing"
)

func TestMemoryKey(t *testing.T) {
	g := &MemoryGenerator{}

	if g.Key() != "memory" {
		t.Error("key should be memory")
	}
}

func TestMemoryGenerator(t *testing.T) {
	g := &MemoryGenerator{}
	value, err := g.Generate()
	if err != nil {
		t.Errorf("should not raise error: %v", err)
	}

	memory, typeOk := value.(map[string]interface{})
	if !typeOk {
		t.Errorf("value should be map. %+v", value)
	}

	if _, ok := memory["total"]; !ok {
		t.Error("memory should have total")
	}

	if _, ok := memory["free"]; !ok {
		t.Error("memory should have free")
	}
}
<|endoftext|>


<commit_before>package io

import (
	"io"
	"sync"

	"fmt"
	"v2ray.com/core/common/alloc"
)

type BufferedWriter struct {
	sync.Mutex
	writer io.Writer
	buffer *alloc.Buffer
	cached bool
}

func NewBufferedWriter(rawWriter io.Writer) *BufferedWriter {
	return &BufferedWriter{
		writer: rawWriter,
		buffer: alloc.NewBuffer().Clear(),
		cached: true,
	}
}

func (this *BufferedWriter) ReadFrom(reader io.Reader) (int64, error) {
	this.Lock()
	defer this.Unlock()

	if this.writer == nil {
		return 0, io.ErrClosedPipe
	}

	totalBytes := int64(0)
	for {
		nBytes, err := this.buffer.FillFrom(reader)
		totalBytes += int64(nBytes)
		if err != nil {
			if err == io.EOF {
				return totalBytes, nil
			}
			return totalBytes, err
		}
		this.FlushWithoutLock()
	}
}

func (this *BufferedWriter) Write(b []byte) (int, error) {
	this.Lock()
	defer this.Unlock()

	if this.writer == nil {
		return 0, io.ErrClosedPipe
	}

	fmt.Printf("BufferedWriter writing: %v\n", b)

	if !this.cached {
		return this.writer.Write(b)
	}
	nBytes, _ := this.buffer.Write(b)
	if this.buffer.IsFull() {
		this.FlushWithoutLock()
	}
	fmt.Printf("BufferedWriter content: %v\n", this.buffer.Value)
	return nBytes, nil
}

func (this *BufferedWriter) Flush() error {
	this.Lock()
	defer this.Unlock()

	if this.writer == nil {
		return io.ErrClosedPipe
	}

	return this.FlushWithoutLock()
}

func (this *BufferedWriter) FlushWithoutLock() error {
	fmt.Println("BufferedWriter flushing")
	defer this.buffer.Clear()
	for !this.buffer.IsEmpty() {
		nBytes, err := this.writer.Write(this.buffer.Value)
		fmt.Printf("BufferedWriting flushed %d bytes.\n", nBytes)
		if err != nil {
			return err
		}
		this.buffer.SliceFrom(nBytes)
	}
	return nil
}

func (this *BufferedWriter) Cached() bool {
	return this.cached
}

func (this *BufferedWriter) SetCached(cached bool) {
	this.cached = cached
	if !cached && !this.buffer.IsEmpty() {
		this.Flush()
	}
}

func (this *BufferedWriter) Release() {
	this.Flush()

	this.Lock()
	defer this.Unlock()

	this.buffer.Release()
	this.buffer = nil
	this.writer = nil
}
<commit_msg>remove test log<commit_after>package io

import (
	"io"
	"sync"
	"v2ray.com/core/common/alloc"
)

type BufferedWriter struct {
	sync.Mutex
	writer io.Writer
	buffer *alloc.Buffer
	cached bool
}

func NewBufferedWriter(rawWriter io.Writer) *BufferedWriter {
	return &BufferedWriter{
		writer: rawWriter,
		buffer: alloc.NewBuffer().Clear(),
		cached: true,
	}
}

func (this *BufferedWriter) ReadFrom(reader io.Reader) (int64, error) {
	this.Lock()
	defer this.Unlock()

	if this.writer == nil {
		return 0, io.ErrClosedPipe
	}

	totalBytes := int64(0)
	for {
		nBytes, err := this.buffer.FillFrom(reader)
		totalBytes += int64(nBytes)
		if err != nil {
			if err == io.EOF {
				return totalBytes, nil
			}
			return totalBytes, err
		}
		this.FlushWithoutLock()
	}
}

func (this *BufferedWriter) Write(b []byte) (int, error) {
	this.Lock()
	defer this.Unlock()

	if this.writer == nil {
		return 0, io.ErrClosedPipe
	}

	if !this.cached {
		return this.writer.Write(b)
	}
	nBytes, _ := this.buffer.Write(b)
	if this.buffer.IsFull() {
		this.FlushWithoutLock()
	}
	return nBytes, nil
}

func (this *BufferedWriter) Flush() error {
	this.Lock()
	defer this.Unlock()

	if this.writer == nil {
		return io.ErrClosedPipe
	}

	return this.FlushWithoutLock()
}

func (this *BufferedWriter) FlushWithoutLock() error {
	defer this.buffer.Clear()
	for !this.buffer.IsEmpty() {
		nBytes, err := this.writer.Write(this.buffer.Value)
		if err != nil {
			return err
		}
		this.buffer.SliceFrom(nBytes)
	}
	return nil
}

func (this *BufferedWriter) Cached() bool {
	return this.cached
}

func (this *BufferedWriter) SetCached(cached bool) {
	this.cached = cached
	if !cached && !this.buffer.IsEmpty() {
		this.Flush()
	}
}

func (this *BufferedWriter) Release() {
	this.Flush()

	this.Lock()
	defer this.Unlock()

	this.buffer.Release()
	this.buffer = nil
	this.writer = nil
}
<|endoftext|>