<commit_before>package cl11

import (
	clw "github.com/rdwilliamson/clw11"
	"strings"
	"unsafe"
)

type Device struct {
	ID                     clw.DeviceID
	Available              bool
	CompilerAvailable      bool
	LittleEndian           bool
	ErrorCorrectionSupport bool
	ImageSupport           bool
	UnifiedHostMemory      bool
	AddressBits            uint32
	GlobalMemCachelineSize uint32
	MaxClockFrequency      uint32
	MaxComputeUnits        uint32
	MaxConstantArgs        uint32
	MaxReadImageArgs       uint32
	MaxSamplers            uint32
	MaxWorkItemDimensions  uint32
	MaxWriteImageArgs      uint32
	MemBaseAddrAlign       uint32
	MinDataTypeAlignSize   uint32
	VendorID               uint32
	PreferredVectorWidths  VectorWidths
	NativeVectorWidths     VectorWidths
	Extensions             string
	Name                   string
	Profile                string
	Vendor                 string
	Version                string
	OpenclCVersion         string
	DriverVersion          string
}

type DeviceType clw.DeviceType

// Bitfield.
const (
	DeviceTypeDefault     = clw.DeviceTypeDefault
	DeviceTypeCpu         = clw.DeviceTypeCpu
	DeviceTypeGpu         = clw.DeviceTypeGpu
	DeviceTypeAccelerator = clw.DeviceTypeAccelerator
	DeviceTypeAll         = clw.DeviceTypeAll
)

type VectorWidths struct {
	Char   uint8
	Short  uint8
	Int    uint8
	Long   uint8
	Float  uint8
	Double uint8
	Half   uint8
}

type FPConfig uint8

type MemCache uint8

type LocalMem uint8

type ExecCapabilities uint8

func (p *Platform) GetDevices() ([]Device, error) {

	if p.Devices != nil {
		return p.Devices, nil
	}

	var numEntries clw.Uint
	err := clw.GetDeviceIDs(p.ID, clw.DeviceTypeAll, 0, nil, &numEntries)
	if err != nil {
		return nil, err
	}

	deviceIDs := make([]clw.DeviceID, numEntries)
	err = clw.GetDeviceIDs(p.ID, clw.DeviceTypeAll, numEntries, &deviceIDs[0], nil)
	if err != nil {
		return nil, err
	}

	p.Devices = make([]Device, len(deviceIDs))
	for i := range p.Devices {

		p.Devices[i].ID = deviceIDs[i]

		err = p.Devices[i].getAllInfo()
		if err != nil {
			return nil, err
		}
	}

	return p.Devices, nil
}

func (d Device) String() string {
	return ""
}

func (d *Device) getAllInfo() (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	d.Available = d.getBool(clw.DeviceAvailable)
	d.CompilerAvailable = d.getBool(clw.DeviceCompilerAvailable)
	d.LittleEndian = d.getBool(clw.DeviceEndianLittle)
	d.ErrorCorrectionSupport = d.getBool(clw.DeviceErrorCorrectionSupport)
	d.ImageSupport = d.getBool(clw.DeviceImageSupport)
	d.UnifiedHostMemory = d.getBool(clw.DeviceHostUnifiedMemory)

	d.AddressBits = d.getUint(clw.DeviceAddressBits)
	d.GlobalMemCachelineSize = d.getUint(clw.DeviceGlobalMemCachelineSize)
	d.MaxClockFrequency = d.getUint(clw.DeviceMaxClockFrequency)
	d.MaxComputeUnits = d.getUint(clw.DeviceMaxComputeUnits)
	d.MaxConstantArgs = d.getUint(clw.DeviceMaxConstantArgs)
	d.MaxReadImageArgs = d.getUint(clw.DeviceMaxReadImageArgs)
	d.MaxSamplers = d.getUint(clw.DeviceMaxSamplers)
	d.MaxWorkItemDimensions = d.getUint(clw.DeviceMaxWorkItemDimensions)
	d.MaxWriteImageArgs = d.getUint(clw.DeviceMaxWriteImageArgs)
	d.MemBaseAddrAlign = d.getUint(clw.DeviceMemBaseAddrAlign)
	d.MinDataTypeAlignSize = d.getUint(clw.DeviceMinDataTypeAlignSize)
	d.VendorID = d.getUint(clw.DeviceVendorID)

	d.PreferredVectorWidths.Char = uint8(d.getUint(clw.DevicePreferredVectorWidthChar))
	d.PreferredVectorWidths.Short = uint8(d.getUint(clw.DevicePreferredVectorWidthShort))
	d.PreferredVectorWidths.Int = uint8(d.getUint(clw.DevicePreferredVectorWidthInt))
	d.PreferredVectorWidths.Long = uint8(d.getUint(clw.DevicePreferredVectorWidthLong))
	d.PreferredVectorWidths.Float = uint8(d.getUint(clw.DevicePreferredVectorWidthFloat))
	d.PreferredVectorWidths.Double = uint8(d.getUint(clw.DevicePreferredVectorWidthDouble))
	d.PreferredVectorWidths.Half = uint8(d.getUint(clw.DevicePreferredVectorWidthHalf))
	d.NativeVectorWidths.Char = uint8(d.getUint(clw.DeviceNativeVectorWidthChar))
	d.NativeVectorWidths.Short = uint8(d.getUint(clw.DeviceNativeVectorWidthShort))
	d.NativeVectorWidths.Int = uint8(d.getUint(clw.DeviceNativeVectorWidthInt))
	d.NativeVectorWidths.Long = uint8(d.getUint(clw.DeviceNativeVectorWidthLong))
	d.NativeVectorWidths.Float = uint8(d.getUint(clw.DeviceNativeVectorWidthFloat))
	d.NativeVectorWidths.Double = uint8(d.getUint(clw.DeviceNativeVectorWidthDouble))
	d.NativeVectorWidths.Half = uint8(d.getUint(clw.DeviceNativeVectorWidthHalf))

	d.Extensions = d.getString(clw.DeviceExtensions)
	d.Name = d.getString(clw.DeviceName)
	d.Profile = d.getString(clw.DeviceProfile)
	d.Vendor = d.getString(clw.DeviceVendor)
	d.Version = d.getString(clw.DeviceVersion)
	d.OpenclCVersion = d.getString(clw.DeviceOpenclCVersion)
	d.DriverVersion = d.getString(clw.DriverVersion)

	return
}

func (d *Device) getInfo(paramName clw.DeviceInfo) (interface{}, error) {

	switch paramName {

	// fp_config
	case clw.DeviceSingleFpConfig:

	// exec_capabilities
	case clw.DeviceExecutionCapabilities:

	// ulong
	case clw.DeviceGlobalMemCacheSize,
		clw.DeviceGlobalMemSize,
		clw.DeviceLocalMemSize,
		clw.DeviceMaxConstantBufferSize,
		clw.DeviceMaxMemAllocSize:

	// mem_cache_type
	case clw.DeviceGlobalMemCacheType:

	// size_t
	case clw.DeviceImage2dMaxHeight,
		clw.DeviceImage2dMaxWidth,
		clw.DeviceImage3dMaxDepth,
		clw.DeviceImage3dMaxHeight,
		clw.DeviceImage3dMaxWidth,
		clw.DeviceMaxParameterSize,
		clw.DeviceMaxWorkGroupSize,
		clw.DeviceMaxWorkItemSizes,
		clw.DeviceProfilingTimerResolution:

	// device_type
	case clw.DeviceTypeInfo:

	// command_queue_properties
	case clw.DeviceQueueProperties:

	// local_mem_type
	case clw.DeviceLocalMemTypeInfo:

	// platform_id
	case clw.DevicePlatform:
	}

	return nil, nil
}

func (d *Device) getBool(paramName clw.DeviceInfo) bool {
	var paramValue clw.Bool
	err := clw.GetDeviceInfo(d.ID, paramName, clw.Size(unsafe.Sizeof(paramValue)), unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return clw.ToGoBool(paramValue)
}

func (d *Device) getUint(paramName clw.DeviceInfo) uint32 {
	var paramValue clw.Uint
	err := clw.GetDeviceInfo(d.ID, paramName, clw.Size(unsafe.Sizeof(paramValue)), unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint32(paramValue)
}

func (d *Device) getString(paramName clw.DeviceInfo) string {
	var paramValueSize clw.Size
	err := clw.GetDeviceInfo(d.ID, paramName, 0, nil, &paramValueSize)
	if err != nil {
		panic(err)
	}

	buffer := make([]byte, paramValueSize)
	err = clw.GetDeviceInfo(d.ID, paramName, paramValueSize, clw.Pointer(buffer), nil)
	if err != nil {
		panic(err)
	}

	// Trim space and trailing \0.
	return strings.TrimSpace(string(buffer[:len(buffer)-1]))
}

func (d *Device) getUlong(paramName clw.DeviceInfo) uint64 {
	var paramValue clw.Ulong
	err := clw.GetDeviceInfo(d.ID, paramName, clw.Size(unsafe.Sizeof(paramValue)), unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint64(paramValue)
}

func (d *Device) getSize(paramName clw.DeviceInfo) uint {
	var paramValue clw.Size
	err := clw.GetDeviceInfo(d.ID, paramName, clw.Size(unsafe.Sizeof(paramValue)), unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint(paramValue)
}
<commit_msg>Thinking about hiding clw types in cl.<commit_after>package cl11

import (
	clw "github.com/rdwilliamson/clw11"
	"strings"
	"unsafe"
)

type Device struct {
	ID                     clw.DeviceID
	Available              bool
	CompilerAvailable      bool
	LittleEndian           bool
	ErrorCorrectionSupport bool
	ImageSupport           bool
	UnifiedHostMemory      bool
	AddressBits            uint32
	GlobalMemCachelineSize uint32
	MaxClockFrequency      uint32
	MaxComputeUnits        uint32
	MaxConstantArgs        uint32
	MaxReadImageArgs       uint32
	MaxSamplers            uint32
	MaxWorkItemDimensions  uint32
	MaxWriteImageArgs      uint32
	MemBaseAddrAlign       uint32
	MinDataTypeAlignSize   uint32
	VendorID               uint32
	PreferredVectorWidths  VectorWidths
	NativeVectorWidths     VectorWidths
	Extensions             string
	Name                   string
	Profile                string
	Vendor                 string
	Version                string
	OpenclCVersion         string
	DriverVersion          string
}

// Bitfield.
const (
	DeviceTypeDefault     = clw.DeviceTypeDefault
	DeviceTypeCpu         = clw.DeviceTypeCpu
	DeviceTypeGpu         = clw.DeviceTypeGpu
	DeviceTypeAccelerator = clw.DeviceTypeAccelerator
	DeviceTypeAll         = clw.DeviceTypeAll
)

type VectorWidths struct {
	Char   uint8
	Short  uint8
	Int    uint8
	Long   uint8
	Float  uint8
	Double uint8
	Half   uint8
}

type FPConfig uint8

type MemCache uint8

type LocalMem uint8

type ExecCapabilities uint8

func (p *Platform) GetDevices() ([]Device, error) {

	if p.Devices != nil {
		return p.Devices, nil
	}

	var numEntries clw.Uint
	err := clw.GetDeviceIDs(p.ID, clw.DeviceTypeAll, 0, nil, &numEntries)
	if err != nil {
		return nil, err
	}

	deviceIDs := make([]clw.DeviceID, numEntries)
	err = clw.GetDeviceIDs(p.ID, clw.DeviceTypeAll, numEntries, &deviceIDs[0], nil)
	if err != nil {
		return nil, err
	}

	p.Devices = make([]Device, len(deviceIDs))
	for i := range p.Devices {

		p.Devices[i].ID = deviceIDs[i]

		err = p.Devices[i].getAllInfo()
		if err != nil {
			return nil, err
		}
	}

	return p.Devices, nil
}

func (d Device) String() string {
	return ""
}

func (d *Device) getAllInfo() (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	d.Available = d.getBool(clw.DeviceAvailable)
	d.CompilerAvailable = d.getBool(clw.DeviceCompilerAvailable)
	d.LittleEndian = d.getBool(clw.DeviceEndianLittle)
	d.ErrorCorrectionSupport = d.getBool(clw.DeviceErrorCorrectionSupport)
	d.ImageSupport = d.getBool(clw.DeviceImageSupport)
	d.UnifiedHostMemory = d.getBool(clw.DeviceHostUnifiedMemory)

	d.AddressBits = d.getUint(clw.DeviceAddressBits)
	d.GlobalMemCachelineSize = d.getUint(clw.DeviceGlobalMemCachelineSize)
	d.MaxClockFrequency = d.getUint(clw.DeviceMaxClockFrequency)
	d.MaxComputeUnits = d.getUint(clw.DeviceMaxComputeUnits)
	d.MaxConstantArgs = d.getUint(clw.DeviceMaxConstantArgs)
	d.MaxReadImageArgs = d.getUint(clw.DeviceMaxReadImageArgs)
	d.MaxSamplers = d.getUint(clw.DeviceMaxSamplers)
	d.MaxWorkItemDimensions = d.getUint(clw.DeviceMaxWorkItemDimensions)
	d.MaxWriteImageArgs = d.getUint(clw.DeviceMaxWriteImageArgs)
	d.MemBaseAddrAlign = d.getUint(clw.DeviceMemBaseAddrAlign)
	d.MinDataTypeAlignSize = d.getUint(clw.DeviceMinDataTypeAlignSize)
	d.VendorID = d.getUint(clw.DeviceVendorID)

	d.PreferredVectorWidths.Char = uint8(d.getUint(clw.DevicePreferredVectorWidthChar))
	d.PreferredVectorWidths.Short = uint8(d.getUint(clw.DevicePreferredVectorWidthShort))
	d.PreferredVectorWidths.Int = uint8(d.getUint(clw.DevicePreferredVectorWidthInt))
	d.PreferredVectorWidths.Long = uint8(d.getUint(clw.DevicePreferredVectorWidthLong))
	d.PreferredVectorWidths.Float = uint8(d.getUint(clw.DevicePreferredVectorWidthFloat))
	d.PreferredVectorWidths.Double = uint8(d.getUint(clw.DevicePreferredVectorWidthDouble))
	d.PreferredVectorWidths.Half = uint8(d.getUint(clw.DevicePreferredVectorWidthHalf))
	d.NativeVectorWidths.Char = uint8(d.getUint(clw.DeviceNativeVectorWidthChar))
	d.NativeVectorWidths.Short = uint8(d.getUint(clw.DeviceNativeVectorWidthShort))
	d.NativeVectorWidths.Int = uint8(d.getUint(clw.DeviceNativeVectorWidthInt))
	d.NativeVectorWidths.Long = uint8(d.getUint(clw.DeviceNativeVectorWidthLong))
	d.NativeVectorWidths.Float = uint8(d.getUint(clw.DeviceNativeVectorWidthFloat))
	d.NativeVectorWidths.Double = uint8(d.getUint(clw.DeviceNativeVectorWidthDouble))
	d.NativeVectorWidths.Half = uint8(d.getUint(clw.DeviceNativeVectorWidthHalf))

	d.Extensions = d.getString(clw.DeviceExtensions)
	d.Name = d.getString(clw.DeviceName)
	d.Profile = d.getString(clw.DeviceProfile)
	d.Vendor = d.getString(clw.DeviceVendor)
	d.Version = d.getString(clw.DeviceVersion)
	d.OpenclCVersion = d.getString(clw.DeviceOpenclCVersion)
	d.DriverVersion = d.getString(clw.DriverVersion)

	return
}

func (d *Device) getInfo(paramName clw.DeviceInfo) (interface{}, error) {

	switch paramName {

	// fp_config
	case clw.DeviceSingleFpConfig:

	// exec_capabilities
	case clw.DeviceExecutionCapabilities:

	// ulong
	case clw.DeviceGlobalMemCacheSize,
		clw.DeviceGlobalMemSize,
		clw.DeviceLocalMemSize,
		clw.DeviceMaxConstantBufferSize,
		clw.DeviceMaxMemAllocSize:

	// mem_cache_type
	case clw.DeviceGlobalMemCacheType:

	// size_t
	case clw.DeviceImage2dMaxHeight,
		clw.DeviceImage2dMaxWidth,
		clw.DeviceImage3dMaxDepth,
		clw.DeviceImage3dMaxHeight,
		clw.DeviceImage3dMaxWidth,
		clw.DeviceMaxParameterSize,
		clw.DeviceMaxWorkGroupSize,
		clw.DeviceMaxWorkItemSizes,
		clw.DeviceProfilingTimerResolution:

	// device_type
	case clw.DeviceTypeInfo:

	// command_queue_properties
	case clw.DeviceQueueProperties:

	// local_mem_type
	case clw.DeviceLocalMemTypeInfo:

	// platform_id
	case clw.DevicePlatform:
	}

	return nil, nil
}

func (d *Device) getBool(paramName clw.DeviceInfo) bool {
	var paramValue clw.Bool
	err := clw.GetDeviceInfo(d.ID, paramName, clw.Size(unsafe.Sizeof(paramValue)), unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return clw.ToGoBool(paramValue)
}

func (d *Device) getUint(paramName clw.DeviceInfo) uint32 {
	var paramValue clw.Uint
	err := clw.GetDeviceInfo(d.ID, paramName, clw.Size(unsafe.Sizeof(paramValue)), unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint32(paramValue)
}

func (d *Device) getString(paramName clw.DeviceInfo) string {
	var paramValueSize clw.Size
	err := clw.GetDeviceInfo(d.ID, paramName, 0, nil, &paramValueSize)
	if err != nil {
		panic(err)
	}

	buffer := make([]byte, paramValueSize)
	err = clw.GetDeviceInfo(d.ID, paramName, paramValueSize, clw.Pointer(buffer), nil)
	if err != nil {
		panic(err)
	}

	// Trim space and trailing \0.
	return strings.TrimSpace(string(buffer[:len(buffer)-1]))
}

func (d *Device) getUlong(paramName clw.DeviceInfo) uint64 {
	var paramValue clw.Ulong
	err := clw.GetDeviceInfo(d.ID, paramName, clw.Size(unsafe.Sizeof(paramValue)), unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint64(paramValue)
}

func (d *Device) getSize(paramName clw.DeviceInfo) uint {
	var paramValue clw.Size
	err := clw.GetDeviceInfo(d.ID, paramName, clw.Size(unsafe.Sizeof(paramValue)), unsafe.Pointer(&paramValue), nil)
	if err != nil {
		panic(err)
	}
	return uint(paramValue)
}
<|endoftext|>


<commit_before>package audio

type Mixer struct {
	streamers []Streamer
}

func (m *Mixer) Len() int {
	return len(m.streamers)
}

func (m *Mixer) Play(s ...Streamer) {
	m.streamers = append(m.streamers, s...)
}

func (m *Mixer) Stream(samples [][2]float64) (n int, ok bool) {
	var tmp [512][2]float64

	for len(samples) > 0 {
		toStream := len(tmp)
		if toStream > len(samples) {
			toStream = len(samples)
		}

		// clear the samples
		for i := range samples[:toStream] {
			samples[i] = [2]float64{}
		}

		for si := 0; si < len(m.streamers); si++ {
			// mix the stream
			sn, sok := m.streamers[si].Stream(tmp[:toStream])
			for i := range tmp[:sn] {
				samples[i][0] += tmp[i][0]
				samples[i][1] += tmp[i][1]
			}
			if !sok {
				// remove drained streamer
				sj := len(m.streamers) - 1
				m.streamers[si], m.streamers[sj] = m.streamers[sj], m.streamers[si]
				m.streamers = m.streamers[:sj]
				si--
			}
		}

		samples = samples[toStream:]
		n += toStream
	}

	return n, true
}
<commit_msg>audio: add Mixer doc<commit_after>package audio

// Mixer allows for dynamic mixing of arbitrary number of Streamers. Mixer automatically removes
// drained Streamers. Mixer's stream never drains, when empty, Mixer streams silence.
type Mixer struct {
	streamers []Streamer
}

// Len returns the number of Streamers currently playing in the Mixer.
func (m *Mixer) Len() int {
	return len(m.streamers)
}

// Play adds Streamers to the Mixer.
func (m *Mixer) Play(s ...Streamer) {
	m.streamers = append(m.streamers, s...)
}

// Stream streams all Streamers currently in the Mixer mixed together. This method always returns
// len(samples), true. If there are no Streamers available, this methods streams silence.
func (m *Mixer) Stream(samples [][2]float64) (n int, ok bool) {
	var tmp [512][2]float64

	for len(samples) > 0 {
		toStream := len(tmp)
		if toStream > len(samples) {
			toStream = len(samples)
		}

		// clear the samples
		for i := range samples[:toStream] {
			samples[i] = [2]float64{}
		}

		for si := 0; si < len(m.streamers); si++ {
			// mix the stream
			sn, sok := m.streamers[si].Stream(tmp[:toStream])
			for i := range tmp[:sn] {
				samples[i][0] += tmp[i][0]
				samples[i][1] += tmp[i][1]
			}
			if !sok {
				// remove drained streamer
				sj := len(m.streamers) - 1
				m.streamers[si], m.streamers[sj] = m.streamers[sj], m.streamers[si]
				m.streamers = m.streamers[:sj]
				si--
			}
		}

		samples = samples[toStream:]
		n += toStream
	}

	return n, true
}
<|endoftext|>


<commit_before>package policy

import (
	"log"
	"time"

	"github.com/coredns/coredns/plugin/dnstap"
	"github.com/coredns/coredns/plugin/dnstap/taprw"
	tap "github.com/dnstap/golang-dnstap"
	"github.com/golang/protobuf/proto"
	pb "github.com/infobloxopen/themis/contrib/coredns/policy/dnstap"
	"github.com/miekg/dns"
)

// ProxyWriter is designed for intercepting a DNS response being writen to
// ResponseWriter. It also provides access to the base ResponseWriter of type
// "github.com/coredns/coredns/plugin/dnstap/taprw".ResponseWriter
type ProxyWriter struct {
	*taprw.ResponseWriter
	msg *dns.Msg
}

// NewProxyWriter function creates new ProxyWriter from a ResponseWriter derived
// from "github.com/coredns/coredns/plugin/dnstap/taprw".ResponseWriter and
// turns off sending CQ and CR dnstap messages by dnstap plugin
// If ResponseWriter is of other type, NewProxyWriter returns nil
func NewProxyWriter(w dns.ResponseWriter) *ProxyWriter {
	if tapRW, ok := w.(*taprw.ResponseWriter); ok {
		// turn off sending the CQ and CR dnstap messages by dnstap plugin
		tapRW.Send = &taprw.SendOption{}
		return &ProxyWriter{ResponseWriter: tapRW}
	}
	return nil
}

// WriteMsg saves pointer to DNS message and forwards it to base ResponseWriter
func (w *ProxyWriter) WriteMsg(msg *dns.Msg) error {
	w.msg = msg
	return w.ResponseWriter.WriteMsg(msg)
}

type DnstapSender interface {
	SendCRExtraMsg(pw *ProxyWriter, ah *attrHolder)
}

type policyDnstapSender struct {
	ior dnstap.IORoutine
}

func NewPolicyDnstapSender(io dnstap.IORoutine) DnstapSender {
	return &policyDnstapSender{ior: io}
}

// SendCRExtraMsg creates Client Response (CR) dnstap Message and writes an array
// of extra attributes to Dnstap.Extra field. Then it asynchronously sends the
// message with IORoutine interface
// Parameter tapIO must not be nil
func (s *policyDnstapSender) SendCRExtraMsg(pw *ProxyWriter, ah *attrHolder) {
	if pw == nil || pw.msg == nil {
		log.Printf("[ERROR] Failed to create dnstap CR message - no DNS response message found")
		return
	}
	go func(now time.Time) {
		b := pw.TapBuilder()
		b.TimeSec = uint64(now.Unix())
		timeNs := uint32(now.Nanosecond())
		err := b.AddrMsg(pw.RemoteAddr(), pw.msg)
		if err != nil {
			log.Printf("[ERROR] Failed to create dnstap CR message (%v)", err)
			return
		}
		crMsg := b.ToClientResponse()
		crMsg.ResponseTimeNsec = &timeNs
		t := tap.Dnstap_MESSAGE

		var extra []byte
		if ah != nil {
			extra, err = proto.Marshal(&pb.Extra{Attrs: ah.convertAttrs()})
			if err != nil {
				log.Printf("[ERROR] Failed to create extra data for dnstap CR message (%v)", err)
			}
		}
		dnstapMsg := tap.Dnstap{Type: &t, Message: crMsg, Extra: extra}
		s.ior.Dnstap(dnstapMsg)
	}(time.Now())
}
<commit_msg>remove go routine for sending dnstap message from policy plugin (#101)<commit_after>package policy

import (
	"log"
	"time"

	"github.com/coredns/coredns/plugin/dnstap"
	"github.com/coredns/coredns/plugin/dnstap/taprw"
	tap "github.com/dnstap/golang-dnstap"
	"github.com/golang/protobuf/proto"
	pb "github.com/infobloxopen/themis/contrib/coredns/policy/dnstap"
	"github.com/miekg/dns"
)

// ProxyWriter is designed for intercepting a DNS response being writen to
// ResponseWriter. It also provides access to the base ResponseWriter of type
// "github.com/coredns/coredns/plugin/dnstap/taprw".ResponseWriter
type ProxyWriter struct {
	*taprw.ResponseWriter
	msg *dns.Msg
}

// NewProxyWriter function creates new ProxyWriter from a ResponseWriter derived
// from "github.com/coredns/coredns/plugin/dnstap/taprw".ResponseWriter and
// turns off sending CQ and CR dnstap messages by dnstap plugin
// If ResponseWriter is of other type, NewProxyWriter returns nil
func NewProxyWriter(w dns.ResponseWriter) *ProxyWriter {
	if tapRW, ok := w.(*taprw.ResponseWriter); ok {
		// turn off sending the CQ and CR dnstap messages by dnstap plugin
		tapRW.Send = &taprw.SendOption{}
		return &ProxyWriter{ResponseWriter: tapRW}
	}
	return nil
}

// WriteMsg saves pointer to DNS message and forwards it to base ResponseWriter
func (w *ProxyWriter) WriteMsg(msg *dns.Msg) error {
	w.msg = msg
	return w.ResponseWriter.WriteMsg(msg)
}

type DnstapSender interface {
	SendCRExtraMsg(pw *ProxyWriter, ah *attrHolder)
}

type policyDnstapSender struct {
	ior dnstap.IORoutine
}

func NewPolicyDnstapSender(io dnstap.IORoutine) DnstapSender {
	return &policyDnstapSender{ior: io}
}

// SendCRExtraMsg creates Client Response (CR) dnstap Message and writes an array
// of extra attributes to Dnstap.Extra field. Then it asynchronously sends the
// message with IORoutine interface
// Parameter tapIO must not be nil
func (s *policyDnstapSender) SendCRExtraMsg(pw *ProxyWriter, ah *attrHolder) {
	if pw == nil || pw.msg == nil {
		log.Printf("[ERROR] Failed to create dnstap CR message - no DNS response message found")
		return
	}
	now := time.Now()
	b := pw.TapBuilder()
	b.TimeSec = uint64(now.Unix())
	timeNs := uint32(now.Nanosecond())
	err := b.AddrMsg(pw.RemoteAddr(), pw.msg)
	if err != nil {
		log.Printf("[ERROR] Failed to create dnstap CR message (%v)", err)
		return
	}
	crMsg := b.ToClientResponse()
	crMsg.ResponseTimeNsec = &timeNs
	t := tap.Dnstap_MESSAGE

	var extra []byte
	if ah != nil {
		extra, err = proto.Marshal(&pb.Extra{Attrs: ah.convertAttrs()})
		if err != nil {
			log.Printf("[ERROR] Failed to create extra data for dnstap CR message (%v)", err)
		}
	}
	dnstapMsg := tap.Dnstap{Type: &t, Message: crMsg, Extra: extra}
	s.ior.Dnstap(dnstapMsg)
}
<|endoftext|>


<commit_before>package auth

import (
	"http"
	"fmt"
	"strings"
	"strconv"
	"time"
	"sort"
)

type digest_client struct {
	nc        uint64
	last_seen int64
}

type DigestAuth struct {
	Realm   string
	Opaque  string
	Secrets SecretProvider

	/*
	 Approximate size of Client's Cache. When actual number of
	 tracked client nonces exceeds
	 ClientCacheSize+ClientCacheTolerance, ClientCacheTolerance*2
	 older entries are purged.
	*/
	ClientCacheSize      int
	ClientCacheTolerance int

	clients map[string]*digest_client
}

type digest_cache_entry struct {
	nonce     string
	last_seen int64
}

type digest_cache []digest_cache_entry

func (c digest_cache) Less(i, j int) bool {
	return c[i].last_seen < c[j].last_seen
}

func (c digest_cache) Len() int {
	return len(c)
}

func (c digest_cache) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

/*
 Remove count oldest entries from DigestAuth.clients
*/
func (a *DigestAuth) Purge(count int) {
	entries := make([]digest_cache_entry, 0, len(a.clients))
	for nonce, client := range a.clients {
		entries = append(entries, digest_cache_entry{nonce, client.last_seen})
	}
	cache := digest_cache(entries)
	sort.Sort(cache)
	for _, client := range cache[:count] {
		a.clients[client.nonce] = a.clients[client.nonce], false
	}
}

/*
 http.Handler for DigestAuth which initiates the authentication process
 (or requires reauthentication).
*/
func (a *DigestAuth) RequireAuth(w http.ResponseWriter, r *http.Request) {
	if len(a.clients) > a.ClientCacheSize+a.ClientCacheTolerance {
		a.Purge(a.ClientCacheTolerance * 2)
	}
	nonce := RandomKey()
	a.clients[nonce] = &digest_client{nc: 0, last_seen: time.Nanoseconds()}
	w.Header().Set("WWW-Authenticate",
		fmt.Sprintf(`Digest realm="%s", nonce="%s", opaque="%s", algorithm="MD5", qop="auth"`,
			a.Realm, nonce, a.Opaque))
	w.WriteHeader(401)
	w.Write([]byte("401 Unauthorized\n"))
}

/*
 Parse Authorization header from the http.Request. Returns a map of
 auth parameters or nil if the header is not a valid parsable Digest
 auth header.
*/
func DigestAuthParams(r *http.Request) map[string]string {
	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 || s[0] != "Digest" {
		return nil
	}

	result := map[string]string{}
	for _, kv := range strings.Split(s[1], ",") {
		parts := strings.SplitN(kv, "=", 2)
		if len(parts) != 2 {
			continue
		}
		result[strings.Trim(parts[0], "\" ")] = strings.Trim(parts[1], "\" ")
	}
	return result
}

/*
 Check if request contains valid authentication data. Returns a pair
 of username, authinfo where username is the name of the authenticated
 user or an empty string and authinfo is the contents for the optional
 Authentication-Info response header.
*/
func (da *DigestAuth) CheckAuth(r *http.Request) (username string, authinfo *string) {
	username = ""
	authinfo = nil
	auth := DigestAuthParams(r)
	if auth == nil || da.Opaque != auth["opaque"] || auth["algorithm"] != "MD5" || auth["qop"] != "auth" {
		return
	}

	// Check if the requested URI matches auth header
	if r.URL == nil || len(auth["uri"]) > len(r.URL.Path) || r.URL.Path[:len(auth["uri"])] != auth["uri"] {
		return
	}

	HA1 := da.Secrets(auth["username"], da.Realm)
	HA2 := H(r.Method + ":" + auth["uri"])
	KD := H(strings.Join([]string{HA1, auth["nonce"], auth["nc"], auth["cnonce"], auth["qop"], HA2}, ":"))

	if KD != auth["response"] {
		return
	}

	// At this point crypto checks are completed and validated.
	// Now check if the session is valid.

	nc, err := strconv.Btoui64(auth["nc"], 16)
	if err != nil {
		return
	}

	if client, ok := da.clients[auth["nonce"]]; !ok {
		return
	} else {
		if client.nc != 0 && client.nc >= nc {
			return
		}
		client.nc = nc
		client.last_seen = time.Nanoseconds()
	}

	resp_HA2 := H(":" + auth["uri"])
	rspauth := H(strings.Join([]string{HA1, auth["nonce"], auth["nc"], auth["cnonce"], auth["qop"], resp_HA2}, ":"))

	info := fmt.Sprintf(`qop="auth", rspauth="%s", cnonce="%s", nc="%s"`, rspauth, auth["cnonce"], auth["nc"])
	return auth["username"], &info
}

/*
 Default values for ClientCacheSize and ClientCacheTolerance for DigestAuth
*/
const DefaultClientCacheSize = 1000
const DefaultClientCacheTolerance = 100

/*
 DigestAuthenticator returns an Authenticator which uses HTTP Digest
 authentication. Arguments:

 realm: The authentication realm.

 uri: Protection base uri for the Authorization header.

 secrets: SecretProvider which must return HA1 digests for the same
 realm as above.

 cache: Optional one or two arguments, first is the size of the
 clients cache, second is the tolerance for the cache. Default values
 are used if not given.
*/
func DigestAuthenticator(realm string, uri string, secrets SecretProvider, cache ...int) Authenticator {
	da := &DigestAuth{
		Opaque:               RandomKey(),
		Realm:                realm,
		Secrets:              secrets,
		ClientCacheSize:      DefaultClientCacheSize,
		ClientCacheTolerance: DefaultClientCacheTolerance,
		clients:              map[string]*digest_client{}}

	switch {
	case len(cache) > 0:
		da.ClientCacheSize = cache[0]
		fallthrough
	case len(cache) > 1:
		da.ClientCacheTolerance = cache[1]
		fallthrough
	case len(cache) > 2:
		panic("Unknown extra arguments to DigestAuthenticator")
	}

	return func(wrapped AuthenticatedHandlerFunc) http.HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request) {
			if username, authinfo := da.CheckAuth(r); username == "" {
				da.RequireAuth(w, r)
			} else {
				dr := &AuthenticatedRequest{Request: *r, Username: username}
				if authinfo != nil {
					w.Header().Set("Authentication-Info", *authinfo)
				}
				wrapped(w, dr)
			}
		}
	}
}
<commit_msg>uri argument for Digest was dropped some time ago<commit_after>package auth

import (
	"http"
	"fmt"
	"strings"
	"strconv"
	"time"
	"sort"
)

type digest_client struct {
	nc        uint64
	last_seen int64
}

type DigestAuth struct {
	Realm   string
	Opaque  string
	Secrets SecretProvider

	/*
	 Approximate size of Client's Cache. When actual number of
	 tracked client nonces exceeds
	 ClientCacheSize+ClientCacheTolerance, ClientCacheTolerance*2
	 older entries are purged.
	*/
	ClientCacheSize      int
	ClientCacheTolerance int

	clients map[string]*digest_client
}

type digest_cache_entry struct {
	nonce     string
	last_seen int64
}

type digest_cache []digest_cache_entry

func (c digest_cache) Less(i, j int) bool {
	return c[i].last_seen < c[j].last_seen
}

func (c digest_cache) Len() int {
	return len(c)
}

func (c digest_cache) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

/*
 Remove count oldest entries from DigestAuth.clients
*/
func (a *DigestAuth) Purge(count int) {
	entries := make([]digest_cache_entry, 0, len(a.clients))
	for nonce, client := range a.clients {
		entries = append(entries, digest_cache_entry{nonce, client.last_seen})
	}
	cache := digest_cache(entries)
	sort.Sort(cache)
	for _, client := range cache[:count] {
		a.clients[client.nonce] = a.clients[client.nonce], false
	}
}

/*
 http.Handler for DigestAuth which initiates the authentication process
 (or requires reauthentication).
*/
func (a *DigestAuth) RequireAuth(w http.ResponseWriter, r *http.Request) {
	if len(a.clients) > a.ClientCacheSize+a.ClientCacheTolerance {
		a.Purge(a.ClientCacheTolerance * 2)
	}
	nonce := RandomKey()
	a.clients[nonce] = &digest_client{nc: 0, last_seen: time.Nanoseconds()}
	w.Header().Set("WWW-Authenticate",
		fmt.Sprintf(`Digest realm="%s", nonce="%s", opaque="%s", algorithm="MD5", qop="auth"`,
			a.Realm, nonce, a.Opaque))
	w.WriteHeader(401)
	w.Write([]byte("401 Unauthorized\n"))
}

/*
 Parse Authorization header from the http.Request. Returns a map of
 auth parameters or nil if the header is not a valid parsable Digest
 auth header.
*/
func DigestAuthParams(r *http.Request) map[string]string {
	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 || s[0] != "Digest" {
		return nil
	}

	result := map[string]string{}
	for _, kv := range strings.Split(s[1], ",") {
		parts := strings.SplitN(kv, "=", 2)
		if len(parts) != 2 {
			continue
		}
		result[strings.Trim(parts[0], "\" ")] = strings.Trim(parts[1], "\" ")
	}
	return result
}

/*
 Check if request contains valid authentication data. Returns a pair
 of username, authinfo where username is the name of the authenticated
 user or an empty string and authinfo is the contents for the optional
 Authentication-Info response header.
*/
func (da *DigestAuth) CheckAuth(r *http.Request) (username string, authinfo *string) {
	username = ""
	authinfo = nil
	auth := DigestAuthParams(r)
	if auth == nil || da.Opaque != auth["opaque"] || auth["algorithm"] != "MD5" || auth["qop"] != "auth" {
		return
	}

	// Check if the requested URI matches auth header
	if r.URL == nil || len(auth["uri"]) > len(r.URL.Path) || r.URL.Path[:len(auth["uri"])] != auth["uri"] {
		return
	}

	HA1 := da.Secrets(auth["username"], da.Realm)
	HA2 := H(r.Method + ":" + auth["uri"])
	KD := H(strings.Join([]string{HA1, auth["nonce"], auth["nc"], auth["cnonce"], auth["qop"], HA2}, ":"))

	if KD != auth["response"] {
		return
	}

	// At this point crypto checks are completed and validated.
	// Now check if the session is valid.

	nc, err := strconv.Btoui64(auth["nc"], 16)
	if err != nil {
		return
	}

	if client, ok := da.clients[auth["nonce"]]; !ok {
		return
	} else {
		if client.nc != 0 && client.nc >= nc {
			return
		}
		client.nc = nc
		client.last_seen = time.Nanoseconds()
	}

	resp_HA2 := H(":" + auth["uri"])
	rspauth := H(strings.Join([]string{HA1, auth["nonce"], auth["nc"], auth["cnonce"], auth["qop"], resp_HA2}, ":"))

	info := fmt.Sprintf(`qop="auth", rspauth="%s", cnonce="%s", nc="%s"`, rspauth, auth["cnonce"], auth["nc"])
	return auth["username"], &info
}

/*
 Default values for ClientCacheSize and ClientCacheTolerance for DigestAuth
*/
const DefaultClientCacheSize = 1000
const DefaultClientCacheTolerance = 100

/*
 DigestAuthenticator returns an Authenticator which uses HTTP Digest
 authentication. Arguments:

 realm: The authentication realm.

 secrets: SecretProvider which must return HA1 digests for the same
 realm as above.

 cache: Optional one or two arguments, first is the size of the
 clients cache, second is the tolerance for the cache. Default values
 are used if not given.
*/
func DigestAuthenticator(realm string, secrets SecretProvider, cache ...int) Authenticator {
	da := &DigestAuth{
		Opaque:               RandomKey(),
		Realm:                realm,
		Secrets:              secrets,
		ClientCacheSize:      DefaultClientCacheSize,
		ClientCacheTolerance: DefaultClientCacheTolerance,
		clients:              map[string]*digest_client{}}

	switch {
	case len(cache) > 0:
		da.ClientCacheSize = cache[0]
		fallthrough
	case len(cache) > 1:
		da.ClientCacheTolerance = cache[1]
		fallthrough
	case len(cache) > 2:
		panic("Unknown extra arguments to DigestAuthenticator")
	}

	return func(wrapped AuthenticatedHandlerFunc) http.HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request) {
			if username, authinfo := da.CheckAuth(r); username == "" {
				da.RequireAuth(w, r)
			} else {
				dr := &AuthenticatedRequest{Request: *r, Username: username}
				if authinfo != nil {
					w.Header().Set("Authentication-Info", *authinfo)
				}
				wrapped(w, dr)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

/*
Update sets are sets of packages generated by Gemnasium, aim to be test
in projects to determine if updates are going to pass.
These functions are meant to be used during CI tests.
*/

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"

	"github.com/sergi/go-diff/diffmatchpatch"
)

const (
	ENV_GEMNASIUM_TESTSUITE          = "GEMNASIUM_TESTSUITE"
	ENV_GEMNASIUM_BUNDLE_UPDATE_CMD  = "GEMNASIUM_BUNDLE_UPDATE_CMD"
	ENV_GEMNASIUM_BUNDLE_INSTALL_CMD = "GEMNASIUM_BUNDLE_INSTALL_CMD"
	ENV_BRANCH                       = "BRANCH"
	ENV_REVISION                     = "REVISION"
	AUTOUPDATE_MAX_DURATION          = 1800
	BUNDLE_INSTALL_CMD               = "bundle install"
	BUNDLE_UPDATE_CMD                = "bundle update"
	UPDATE_SET_SUCCESS               = "succeeded"
	UPDATE_SET_FAIL                  = "failed"
)

type RequirementUpdate struct {
	File  RequirementFile `json:"file"`
	Patch string          `json:"patch"`
}

type RequirementFile struct {
	Path string `json:"path"`
	SHA1 string `json:"sha1"`
}

type VersionUpdate struct {
	Package       Package
	OldVersion    string `json:"old_version"`
	TargetVersion string `json:"target_version"`
}

type UpdateSet struct {
	ID                 int                 `json:"id"`
	RequirementUpdates []RequirementUpdate `json:"requirement_updates"`
	VersionUpdates     []VersionUpdate     `json:"version_updates"`
}

// Download and loop over update sets, apply changes, run test suite, and finally notify gemnasium
func AutoUpdate(projectSlug string, testSuite []string, config *Config) error {
	if projectSlug == "" {
		return errors.New("Arg [projectSlug] can't be empty")
	}
	if envTS := os.Getenv(ENV_GEMNASIUM_TESTSUITE); envTS != "" {
		testSuite = []string{os.Getenv(ENV_GEMNASIUM_TESTSUITE)}
	}
	if len(testSuite) == 0 {
		return errors.New("Arg [testSuite] can't be empty")
	}

	fmt.Printf("Executing test script: ")
	start := time.Now()
	err := exec.Command(testSuite[0], testSuite[1:]...).Run()
	fmt.Printf("done (%fs)\n", time.Since(start).Seconds())
	if err != nil {
		fmt.Println("Aborting, initial test suite run is failing")
		return err
	}

	// We'll be checking loop duration on each iteration
	startTime := time.Now()
	// Loop until tests are green
	for {
		if time.Since(startTime).Seconds() > AUTOUPDATE_MAX_DURATION {
			fmt.Println("Max loop duration reached, aborting.")
			break
		}
		updateSet, err := fetchUpdateSet(projectSlug, config)
		if err != nil {
			return err
		}
		if updateSet.ID == 0 {
			fmt.Println("Job done!")
			break
		}
		fmt.Printf("========= [UpdateSet #%d] =========\n", updateSet.ID)

		// We have an updateSet, let's patch files and run tests
		err = applyUpdateSet(updateSet)
		if err != nil {
			return err
		}

		fmt.Printf("Executing test script: ")
		start := time.Now()
		err = exec.Command(testSuite[0], testSuite[1:]...).Run()
		fmt.Printf("done (%fs)\n", time.Since(start).Seconds())
		if err == nil {
			// we found a valid candidate, ending.
			err := pushUpdateSetResult(projectSlug, updateSet.ID, UPDATE_SET_SUCCESS, config)
			if err != nil {
				return err
			}

			break
		}
		err = pushUpdateSetResult(projectSlug, updateSet.ID, UPDATE_SET_FAIL, config)
		if err != nil {
			return err
		}
		// Let's continue with another set
	}
	return nil
}

// Fetch an update set and apply it
func fetchUpdateSet(projectSlug string, config *Config) (*UpdateSet, error) {
	client := &http.Client{}
	url := fmt.Sprintf("%s/projects/%s/branches/%s/update_sets/next", config.APIEndpoint, projectSlug, getCurrentBranch())
	revision := getCurrentRevision()
	if revision == "" {
		return nil, errors.New("Can't determine current revision, please use REVISION env var to specify it")
	}
	revisionJSON, err := json.Marshal(&map[string]string{"revision": revision})
	if err != nil {
		return nil, err
	}

	req, err := NewAPIRequest("POST", url, config.APIKey, bytes.NewReader(revisionJSON))
	if err != nil {
		return nil, err
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Server returned non-200 status: %v\n", resp.Status)
	}

	var updateSet *UpdateSet
	if err := json.Unmarshal(body, &updateSet); err != nil {
		fmt.Printf("body: %s\n", body)
		return nil, err
	}

	// if RawFormat flag is set, don't format the output
	if config.RawFormat {
		fmt.Printf("%s", body)
	}
	return updateSet, nil
}

// Patch files if needed, and update packages
func applyUpdateSet(updateSet *UpdateSet) error {
	for _, ru := range updateSet.RequirementUpdates {
		f := ru.File
		err := checkFileSHA1(f.Path, f.SHA1)
		if err != nil {
			return err
		}
		fmt.Println("Patching", f.Path)
		patchFile(f.Path, ru.Patch)
		// TODO: Make this command generic
		bi := BUNDLE_INSTALL_CMD
		if biCMDEnv := os.Getenv(ENV_GEMNASIUM_BUNDLE_INSTALL_CMD); biCMDEnv != "" {
			bi = biCMDEnv
		}
		parts := strings.Fields(bi)
		cmd := exec.Command(parts[0], parts[1:]...)
		cmd.Dir = path.Dir("f.Path")
		err = cmd.Run()
		if err != nil {
			return err
		}
	}

	upt := BUNDLE_UPDATE_CMD
	if uptEnv := os.Getenv(ENV_GEMNASIUM_BUNDLE_UPDATE_CMD); uptEnv != "" {
		upt = uptEnv
	}
	parts := strings.Fields(upt)
	for _, vu := range updateSet.VersionUpdates {
		// TODO: run bundle update here
		fmt.Printf("Updating dependency %s (%s => %s)\n", vu.Package.Name, vu.OldVersion, vu.TargetVersion)
		parts = append(parts, vu.Package.Name)
	}
	fmt.Printf("Executing update commmand: %s\n", strings.Join(parts, " "))
	err := exec.Command(parts[0], parts[1:]...).Run()
	if err != nil {
		return err
	}
	fmt.Println("Done")
	return nil
}

// Once update set has been tested, we must send the result to Gemnasium,
// in order to update statitics.
func pushUpdateSetResult(projectSlug string, updateSetID int, status string, config *Config) error {
	fmt.Printf("Pushing result (status='%s'): ", status)
	if updateSetID == 0 || status == "" {
		return errors.New("Missing updateSet ID and/or status args")
	}
	client := &http.Client{}
	url := fmt.Sprintf("%s/projects/%s/branches/%s/update_sets/%d", config.APIEndpoint, projectSlug, getCurrentBranch(), updateSetID)
	update := &map[string]string{"state": status}
	updateJSON, err := json.Marshal(update)
	if err != nil {
		return err
	}

	req, err := NewAPIRequest("PATCH", url, config.APIKey, bytes.NewReader(updateJSON))
	if err != nil {
		return err
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Server returned non-200 status: %v\n", resp.Status)
	}
	fmt.Printf("done\n")
	return nil
}

func checkFileSHA1(filePath, fileSHA1 string) error {
	dat, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}
	h := sha1.New()
	header := fmt.Sprintf("blob %d\x00", len(dat))
	io.WriteString(h, header)
	io.Copy(h, bytes.NewReader(dat))
	hash := h.Sum(nil)

	sum := fmt.Sprintf("%x", hash)
	if sum != fileSHA1 {
		return fmt.Errorf("%s: File signature doesn't match (expected: %s, got: %s)", filePath, fileSHA1, sum)
	}
	return nil
}

// Return the current branch name, using git.
// If the env var "BRANCH" is declared, its value is returned diretly
func getCurrentBranch() string {
	if envBranch := os.Getenv(ENV_BRANCH); envBranch != "" {
		return envBranch
	}
	out, err := exec.Command(gitPath(), "rev-parse", "--abbrev-ref", "HEAD").Output()
	if err != nil {
		return "master"
	}
	return strings.TrimSpace(string(out))
}

// return the current commit sha, using git
// If the env var "REVISION" is specified, its value is returned directly
func getCurrentRevision() string {
	if envRevision := os.Getenv(ENV_REVISION); envRevision != "" {
		return envRevision
	}
	out, err := exec.Command(gitPath(), "rev-parse", "--verify", "HEAD").Output()
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(out))
}

// Lookup for "git" in $PATH
func gitPath() string {
	path, _ := exec.LookPath("git")
	return path
}

// Apply patch to a file.
// The file will be opened, updated, and written
func patchFile(filePath, patch string) error {
	dmp := diffmatchpatch.New()
	patches, err := dmp.PatchFromText(patch)
	if err != nil {
		return err
	}
	dat, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}

	var patchesApplied []bool
	patchedDat, patchesApplied := dmp.PatchApply(patches, string(dat))
	for i, applied := range patchesApplied {
		if !applied {
			return fmt.Errorf("Patching failed: %s", patches[i])
		}
	}
	fmt.Printf("%d patches applied\n", len(patchesApplied))
	err = ioutil.WriteFile(filePath, []byte(patchedDat), 0644)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Display output in case of failure<commit_after>package main

/*
Update sets are sets of packages generated by Gemnasium, aim to be test
in projects to determine if updates are going to pass.
These functions are meant to be used during CI tests.
*/

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"

	"github.com/sergi/go-diff/diffmatchpatch"
)

const (
	ENV_GEMNASIUM_TESTSUITE          = "GEMNASIUM_TESTSUITE"
	ENV_GEMNASIUM_BUNDLE_UPDATE_CMD  = "GEMNASIUM_BUNDLE_UPDATE_CMD"
	ENV_GEMNASIUM_BUNDLE_INSTALL_CMD = "GEMNASIUM_BUNDLE_INSTALL_CMD"
	ENV_BRANCH                       = "BRANCH"
	ENV_REVISION                     = "REVISION"
	AUTOUPDATE_MAX_DURATION          = 1800
	BUNDLE_INSTALL_CMD               = "bundle install"
	BUNDLE_UPDATE_CMD                = "bundle update"
	UPDATE_SET_SUCCESS               = "succeeded"
	UPDATE_SET_FAIL                  = "failed"
)

type RequirementUpdate struct {
	File  RequirementFile `json:"file"`
	Patch string          `json:"patch"`
}

type RequirementFile struct {
	Path string `json:"path"`
	SHA1 string `json:"sha1"`
}

type VersionUpdate struct {
	Package       Package
	OldVersion    string `json:"old_version"`
	TargetVersion string `json:"target_version"`
}

type UpdateSet struct {
	ID                 int                 `json:"id"`
	RequirementUpdates []RequirementUpdate `json:"requirement_updates"`
	VersionUpdates     []VersionUpdate     `json:"version_updates"`
}

// Download and loop over update sets, apply changes, run test suite, and finally notify gemnasium
func AutoUpdate(projectSlug string, testSuite []string, config *Config) error {
	if projectSlug == "" {
		return errors.New("Arg [projectSlug] can't be empty")
	}
	if envTS := os.Getenv(ENV_GEMNASIUM_TESTSUITE); envTS != "" {
		testSuite = []string{os.Getenv(ENV_GEMNASIUM_TESTSUITE)}
	}
	if len(testSuite) == 0 {
		return errors.New("Arg [testSuite] can't be empty")
	}

	fmt.Printf("Executing test script: ")
	start := time.Now()
	out, err := exec.Command(testSuite[0], testSuite[1:]...).Output()
	fmt.Printf("done (%fs)\n", time.Since(start).Seconds())
	if err != nil {
		fmt.Println("Aborting, initial test suite run is failing:")
		fmt.Printf("%s\n", out)
		return err
	}

	// We'll be checking loop duration on each iteration
	startTime := time.Now()
	// Loop until tests are green
	for {
		if time.Since(startTime).Seconds() > AUTOUPDATE_MAX_DURATION {
			fmt.Println("Max loop duration reached, aborting.")
			break
		}
		updateSet, err := fetchUpdateSet(projectSlug, config)
		if err != nil {
			return err
		}
		if updateSet.ID == 0 {
			fmt.Println("Job done!")
			break
		}
		fmt.Printf("========= [UpdateSet #%d] =========\n", updateSet.ID)

		// We have an updateSet, let's patch files and run tests
		err = applyUpdateSet(updateSet)
		if err != nil {
			return err
		}

		fmt.Printf("Executing test script: ")
		start := time.Now()
		out, err = exec.Command(testSuite[0], testSuite[1:]...).Output()
		fmt.Printf("done (%fs)\n", time.Since(start).Seconds())
		if err == nil {
			// we found a valid candidate, ending.
			err := pushUpdateSetResult(projectSlug, updateSet.ID, UPDATE_SET_SUCCESS, config)
			if err != nil {
				return err
			}

			break
		}
		// display cmd output
		fmt.Printf("%s\n", out)
		err = pushUpdateSetResult(projectSlug, updateSet.ID, UPDATE_SET_FAIL, config)
		if err != nil {
			return err
		}
		// Let's continue with another set
	}
	return nil
}

// Fetch an update set and apply it
func fetchUpdateSet(projectSlug string, config *Config) (*UpdateSet, error) {
	client := &http.Client{}
	url := fmt.Sprintf("%s/projects/%s/branches/%s/update_sets/next", config.APIEndpoint, projectSlug, getCurrentBranch())
	revision := getCurrentRevision()
	if revision == "" {
		return nil, errors.New("Can't determine current revision, please use REVISION env var to specify it")
	}
	revisionJSON, err := json.Marshal(&map[string]string{"revision": revision})
	if err != nil {
		return nil, err
	}

	req, err := NewAPIRequest("POST", url, config.APIKey, bytes.NewReader(revisionJSON))
	if err != nil {
		return nil, err
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Server returned non-200 status: %v\n", resp.Status)
	}

	var updateSet *UpdateSet
	if err := json.Unmarshal(body, &updateSet); err != nil {
		fmt.Printf("body: %s\n", body)
		return nil, err
	}

	// if RawFormat flag is set, don't format the output
	if config.RawFormat {
		fmt.Printf("%s", body)
	}
	return updateSet, nil
}

// Patch files if needed, and update packages
func applyUpdateSet(updateSet *UpdateSet) error {
	for _, ru := range updateSet.RequirementUpdates {
		f := ru.File
		err := checkFileSHA1(f.Path, f.SHA1)
		if err != nil {
			return err
		}
		fmt.Println("Patching", f.Path)
		patchFile(f.Path, ru.Patch)
		// TODO: Make this command generic
		bi := BUNDLE_INSTALL_CMD
		if biCMDEnv := os.Getenv(ENV_GEMNASIUM_BUNDLE_INSTALL_CMD); biCMDEnv != "" {
			bi = biCMDEnv
		}
		parts := strings.Fields(bi)
		cmd := exec.Command(parts[0], parts[1:]...)
		cmd.Dir = path.Dir("f.Path")
		err = cmd.Run()
		if err != nil {
			return err
		}
	}

	upt := BUNDLE_UPDATE_CMD
	if uptEnv := os.Getenv(ENV_GEMNASIUM_BUNDLE_UPDATE_CMD); uptEnv != "" {
		upt = uptEnv
	}
	parts := strings.Fields(upt)
	for _, vu := range updateSet.VersionUpdates {
		// TODO: run bundle update here
		fmt.Printf("Updating dependency %s (%s => %s)\n", vu.Package.Name, vu.OldVersion, vu.TargetVersion)
		parts = append(parts, vu.Package.Name)
	}
	fmt.Printf("Executing update commmand: %s\n", strings.Join(parts, " "))
	out, err := exec.Command(parts[0], parts[1:]...).Output()
	if err != nil {
		fmt.Printf("%s\n", out)
		return err
	}
	fmt.Println("Done")
	return nil
}

// Once update set has been tested, we must send the result to Gemnasium,
// in order to update statitics.
func pushUpdateSetResult(projectSlug string, updateSetID int, status string, config *Config) error {
	fmt.Printf("Pushing result (status='%s'): ", status)
	if updateSetID == 0 || status == "" {
		return errors.New("Missing updateSet ID and/or status args")
	}
	client := &http.Client{}
	url := fmt.Sprintf("%s/projects/%s/branches/%s/update_sets/%d", config.APIEndpoint, projectSlug, getCurrentBranch(), updateSetID)
	update := &map[string]string{"state": status}
	updateJSON, err := json.Marshal(update)
	if err != nil {
		return err
	}

	req, err := NewAPIRequest("PATCH", url, config.APIKey, bytes.NewReader(updateJSON))
	if err != nil {
		return err
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Server returned non-200 status: %v\n", resp.Status)
	}
	fmt.Printf("done\n")
	return nil
}

func checkFileSHA1(filePath, fileSHA1 string) error {
	dat, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}
	h := sha1.New()
	header := fmt.Sprintf("blob %d\x00", len(dat))
	io.WriteString(h, header)
	io.Copy(h, bytes.NewReader(dat))
	hash := h.Sum(nil)

	sum := fmt.Sprintf("%x", hash)
	if sum != fileSHA1 {
		return fmt.Errorf("%s: File signature doesn't match (expected: %s, got: %s)", filePath, fileSHA1, sum)
	}
	return nil
}

// Return the current branch name, using git.
// If the env var "BRANCH" is declared, its value is returned diretly
func getCurrentBranch() string {
	if envBranch := os.Getenv(ENV_BRANCH); envBranch != "" {
		return envBranch
	}
	out, err := exec.Command(gitPath(), "rev-parse", "--abbrev-ref", "HEAD").Output()
	if err != nil {
		return "master"
	}
	return strings.TrimSpace(string(out))
}

// return the current commit sha, using git
// If the env var "REVISION" is specified, its value is returned directly
func getCurrentRevision() string {
	if envRevision := os.Getenv(ENV_REVISION); envRevision != "" {
		return envRevision
	}
	out, err := exec.Command(gitPath(), "rev-parse", "--verify", "HEAD").Output()
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(out))
}

// Lookup for "git" in $PATH
func gitPath() string {
	path, _ := exec.LookPath("git")
	return path
}

// Apply patch to a file.
// The file will be opened, updated, and written
func patchFile(filePath, patch string) error {
	dmp := diffmatchpatch.New()
	patches, err := dmp.PatchFromText(patch)
	if err != nil {
		return err
	}
	dat, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}

	var patchesApplied []bool
	patchedDat, patchesApplied := dmp.PatchApply(patches, string(dat))
	for i, applied := range patchesApplied {
		if !applied {
			return fmt.Errorf("Patching failed: %s", patches[i])
		}
	}
	fmt.Printf("%d patches applied\n", len(patchesApplied))
	err = ioutil.WriteFile(filePath, []byte(patchedDat), 0644)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"time"
)

const (
	MODE_READ  = 1
	MODE_WRITE = 2
)

type IoArgs struct {
	f       FileStore
	ioMode  int
	buf     []byte
	offset  int64
	context interface{}
}

type WriteContext struct {
	peer       *peerState
	whichPiece uint32 //piece index
	begin      uint32
	length     int
	realLength int
}

type ReadContext struct {
	peer         *peerState
	msgBuf       []byte
	globalOffset int64
	length       int
	realLength   int
}

func IoRoutine(request <-chan *IoArgs, responce chan<- interface{}) {
	log.Println("start IoRoutine")

	for arg := range request {
		//todo: sort by offset and batch process io
		HandleIo(arg)
		responce <- arg.context
	}

	log.Println("exit IoRoutine")
}

func HandleIo(arg *IoArgs) {
	var realLength = 0
	var err error = nil
	start := time.Now()
	if cfg.doRealReadWrite {
		if arg.ioMode == MODE_READ {
			//log.Println("read offset", arg.offset, "bufffer size", len(arg.buf))
			realLength, err = arg.f.ReadAt(arg.buf, arg.offset)
			if err != nil || realLength < 0 {
				panic("")
			}

			if c, ok := arg.context.(*ReadContext); ok {
				c.realLength = realLength
				//log.Println("read", c.realLength, "offset", arg.offset, "bufffer size", len(arg.buf))
			} else {
				panic("")
			}
		} else { //write
			realLength, err = arg.f.WriteAt(arg.buf, arg.offset)
			if err != nil || realLength < 0 {
				panic("")
			}

			if c, ok := arg.context.(*WriteContext); ok {
				c.realLength = realLength
				//log.Println("write", c.realLength, "offset", arg.offset)
			} else {
				panic("")
			}
		}
	} else {
		if arg.ioMode == MODE_READ {
			if c, ok := arg.context.(*ReadContext); ok {
				c.realLength = c.length
			} else {
				panic("")
			}
		} else { //write
			if c, ok := arg.context.(*WriteContext); ok {
				c.realLength = c.length
			} else {
				panic("")
			}
		}
	}

	sec := time.Now().Sub(start).Seconds()
	if sec >= 2 {
		var mod = "READ"
		if arg.ioMode == MODE_WRITE {
			mod = "WRITE"
		}
		log.Printf("\nwarning, disk io too slow, use %v seconds, mod:%v, offset:%v\n", sec, mod, arg.offset)
	}
}

<commit_msg>batch ordered io<commit_after>package main

import (
	"log"
	"time"
	"github.com/petar/GoLLRB/llrb"
)

const (
	MODE_READ  = 1
	MODE_WRITE = 2
)

type IoArgs struct {
	f       FileStore
	ioMode  int
	buf     []byte
	offset  int64
	context interface{}
}

type WriteContext struct {
	peer       *peerState
	whichPiece uint32 //piece index
	begin      uint32
	length     int
	realLength int
}

type ReadContext struct {
	peer         *peerState
	msgBuf       []byte
	globalOffset int64
	length       int
	realLength   int
}

func ioLessFun(a, b interface{}) bool {
	return a.(*IoArgs).offset < b.(*IoArgs).offset
}

func IoRoutine(request <-chan *IoArgs, responce chan<- interface{}) {
	log.Println("start IoRoutine")

	sortPieces := llrb.New(ioLessFun)

	for arg := range request {
		//todo: sort by offset and batch process io
		cnt := len(request)

		//batch get request, then sort by offset
		sortPieces.InsertNoReplace(arg)
		for i := 0; i < cnt; i++ {
			a := <-request
			sortPieces.InsertNoReplace(a)
			if a.ioMode == MODE_WRITE {
				break
			}
		}

		//batch process
		if cnt > 50 {
			log.Println("io is busy, batch io count", cnt)
		}

		for  {
			min := sortPieces.DeleteMin()
			if min == nil {
				break
			}

			//log.Println("io offset", min.(*IoArgs).offset)

			HandleIo(min.(*IoArgs))
			responce <- min.(*IoArgs).context
		}
	}

	log.Println("exit IoRoutine")
}

func HandleIo(arg *IoArgs) {
	var realLength = 0
	var err error = nil
	start := time.Now()
	if cfg.doRealReadWrite {
		if arg.ioMode == MODE_READ {
			//log.Println("read offset", arg.offset, "bufffer size", len(arg.buf))
			realLength, err = arg.f.ReadAt(arg.buf, arg.offset)
			if err != nil || realLength < 0 {
				panic("")
			}

			if c, ok := arg.context.(*ReadContext); ok {
				c.realLength = realLength
				//log.Println("read", c.realLength, "offset", arg.offset, "bufffer size", len(arg.buf))
			} else {
				panic("")
			}
		} else { //write
			realLength, err = arg.f.WriteAt(arg.buf, arg.offset)
			if err != nil || realLength < 0 {
				panic("")
			}

			if c, ok := arg.context.(*WriteContext); ok {
				c.realLength = realLength
				//log.Println("write", c.realLength, "offset", arg.offset)
			} else {
				panic("")
			}
		}
	} else {
		if arg.ioMode == MODE_READ {
			if c, ok := arg.context.(*ReadContext); ok {
				c.realLength = c.length
			} else {
				panic("")
			}
		} else { //write
			if c, ok := arg.context.(*WriteContext); ok {
				c.realLength = c.length
			} else {
				panic("")
			}
		}
	}

	sec := time.Now().Sub(start).Seconds()
	if sec >= 2 {
		var mod = "READ"
		if arg.ioMode == MODE_WRITE {
			mod = "WRITE"
		}
		log.Printf("warning, disk io too slow, use %v seconds, mod:%v, offset:%v\n", sec, mod, arg.offset)
	}
}

<|endoftext|>


<commit_before>package autorestart

import (
	"github.com/howeyc/fsnotify"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
)

const logPrefix = "@(dim:[autorebuild]) "

// Restart the current program when the program's executable is updated.
// This detects the program's location in the filesystem by inspecting os.Args[0],
// then watches the filesystem for changes to that path. When it detects such a
// change, it restarts the process by calling syscall.Exec (and thus this is not
// portable to OSes such as Windows that do not support exec).
func RestartOnChange() {
	logger := log.New(os.Stderr, "[autorestart.RestartOnChange] ", log.LstdFlags)
	exePath, err := exec.LookPath(os.Args[0])
	if err != nil {
		logger.Printf("Failed to resolve path to current program: %s\n", err)
		return
	}
	exePath, err = filepath.Abs(exePath)
	if err != nil {
		logger.Printf("Failed to resolve absolute path to current program: %s\n", err)
	}
	exePath = filepath.Clean(exePath)
	exeDir := filepath.Dir(exePath)
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		logger.Printf("Failed to initialize howeyc/fsnotify watcher: %s\n", err)
		return
	}
	abs, _ := filepath.Abs(exeDir)
	err = watcher.Watch(abs)
	if err != nil {
		logger.Printf("Failed to start filesystem watcher on %s: %s\n", exeDir, err)
	}
	for {
		select {
		case err := <-watcher.Error:
			logger.Printf("Watcher error: %s\n", err)
		case ev := <-watcher.Event:
			// log.Println("change", ev.Name, exePath, ev)
			if ev.Name == exePath && (ev.IsModify() || ev.IsCreate()) {
				logger.Printf("%s changed. Restarting via exec.\n", exePath)
				syscall.Exec(exePath, os.Args, os.Environ())
			}
		}
	}
}
<commit_msg>Break apart RestartOnChange into NotifyOnChange and RestartViaExec<commit_after>package autorestart

import (
	"github.com/howeyc/fsnotify"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
)

func logf(format string, args ...interface{}) {
	log.Printf("[autorestart] "+format+"\n", args...)
}

const errorPath = "*error*"

var _exePath = errorPath

func getExePath() string {
	var err error
	if _exePath == errorPath {
		_exePath, err = exec.LookPath(os.Args[0])
		if err != nil {
			logf("Failed to resolve path to current program: %s", err)
			_exePath = errorPath
		} else {
			_exePath, err = filepath.Abs(_exePath)
			if err != nil {
				logf("Failed to resolve absolute path to current program: %s", err)
				_exePath = errorPath
			} else {
				_exePath = filepath.Clean(_exePath)
			}
		}
	}
	return _exePath
}

// Restart the current program when the program's executable is updated.
// This function is a wrapper around NotifyOnChange and RestartViaExec, calling the
// latter when the former signals that a change was detected.
func RestartOnChange() {
	notifyChan := NotifyOnChange()
	<-notifyChan
	logf("%s changed. Restarting via exec.", getExePath())
	RestartViaExec()
}

// Subscribe to a notification when the current process' executable file is modified.
// Returns a channel to which notifications (just `true`) will be sent whenever a
// change is detected.
func NotifyOnChange() chan bool {
	notifyChan := make(chan bool)
	go func() {
		exePath := getExePath()
		if exePath == errorPath {
			return
		}
		exeDir := filepath.Dir(exePath)
		watcher, err := fsnotify.NewWatcher()
		if err != nil {
			logf("Failed to initialize howeyc/fsnotify watcher: %s", err)
			return
		}
		abs, _ := filepath.Abs(exeDir)
		err = watcher.Watch(abs)
		if err != nil {
			logf("Failed to start filesystem watcher on %s: %s", exeDir, err)
			return
		}
		for {
			select {
			case err := <-watcher.Error:
				logf("Watcher error: %s", err)
			case ev := <-watcher.Event:
				// log.Println("change", ev.Name, exePath, ev)
				if ev.Name == exePath && (ev.IsModify() || ev.IsCreate()) {
					notifyChan <- true
				}
			}
		}
	}()
	return notifyChan
}

// Restart the current process by calling syscall.Exec, using os.Args (with filepath.LookPath)
// and os.Environ() to recreate the same args & environment that was used when the process was
// originally started.
// Due to using syscall.Exec, this function is not portable to systems that don't support exec.
func RestartViaExec() {
	exePath := getExePath()
	if exePath == errorPath {
		return
	}
	syscall.Exec(exePath, os.Args, os.Environ())
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"net/url"
	"regexp"
	"time"

	"github.com/dfordsoft/golib/ebook"
	"github.com/dfordsoft/golib/httputil"
)

type tocPattern struct {
	host            string
	bookTitle       string
	bookTitlePos    int
	item            string
	articleTitlePos int
	articleURLPos   int
}

type pageContentMarker struct {
	host  string
	start []byte
	end   []byte
}

var (
	tocPatterns = []tocPattern{
		{
			host:            "www.biqudu.com",
			bookTitle:       `<h1>([^<]+)</h1>$`,
			bookTitlePos:    1,
			item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
			articleURLPos:   1,
			articleTitlePos: 2,
		},
		{
			host:            "www.qu.la",
			bookTitle:       `<h1>([^<]+)</h1>$`,
			bookTitlePos:    1,
			item:            `<dd>\s*<a\s*(style=""\s*)?href="([^"]+)">([^<]+)</a></dd>$`,
			articleURLPos:   2,
			articleTitlePos: 3,
		},
	}
	pageContentMarkers = []pageContentMarker{
		{
			host:  "www.biqudu.com",
			start: []byte(`<div id="content"><script>readx();</script>`),
			end:   []byte(`<script>chaptererror();</script>`),
		},
		{
			host:  "www.qu.la",
			start: []byte(`<div id="content">`),
			end:   []byte(`<script>chaptererror();</script>`),
		},
	}
)

func init() {
	registerNovelSiteHandler(&novelSiteHandler{
		Match:    isBiquge,
		Download: dlBiquge,
	})
}

func isBiquge(u string) bool {
	urlPatterns := []string{
		`http://www\.biquge\.cm/[0-9]+/[0-9]+/`,
		`http://www\.biqugezw\.com/[0-9]+_[0-9]+/`,
		`http://www\.630zw\.com/[0-9]+_[0-9]+/`,
		`http://www\.biqudu\.com/[0-9]+_[0-9]+/`,
		`http://www\.biquge\.lu/book/[0-9]+/`,
		`http://www\.qu\.la/book/[0-9]+/`,
	}

	for _, pattern := range urlPatterns {
		r, _ := regexp.Compile(pattern)
		if r.MatchString(u) {
			return true
		}
	}
	return false
}

func dlBiquge(u string) {
	theURL, _ := url.Parse(u)
	headers := map[string]string{
		"Referer":                   fmt.Sprintf("%s://%s", theURL.Scheme, theURL.Host),
		"User-Agent":                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
		"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
		"Accept-Language":           `en-US,en;q=0.8`,
		"Upgrade-Insecure-Requests": "1",
	}
	b, err := httputil.GetBytes(u, headers, 60*time.Second, 3)
	if err != nil {
		return
	}

	mobi := &ebook.Mobi{}
	mobi.Begin()

	var title string
	var lines []string

	var p tocPattern
	for _, patt := range tocPatterns {
		if theURL.Host == patt.host {
			p = patt
			break
		}
	}
	r, _ := regexp.Compile(p.item)
	re, _ := regexp.Compile(p.bookTitle)
	scanner := bufio.NewScanner(bytes.NewReader(b))
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if title == "" {
			ss := re.FindAllStringSubmatch(line, -1)
			if len(ss) > 0 && len(ss[0]) > 0 {
				s := ss[0]
				title = s[p.bookTitlePos]
				mobi.SetTitle(title)
				continue
			}
		}
		if r.MatchString(line) {
			lines = append(lines, line)
		}
	}
	for i := len(lines) - 1; i >= 0 && i < len(lines) && lines[0] == lines[i]; i -= 2 {
		lines = lines[1:]
	}

	for _, line := range lines {
		ss := r.FindAllStringSubmatch(line, -1)
		s := ss[0]
		finalURL := fmt.Sprintf("%s://%s%s", theURL.Scheme, theURL.Host, s[p.articleURLPos])
		c := dlBiqugePage(finalURL)
		mobi.AppendContent(s[p.articleTitlePos], finalURL, string(c))
		fmt.Println(s[p.articleTitlePos], finalURL, len(c), "bytes")
	}
	mobi.End()
}

func dlBiqugePage(u string) (c []byte) {
	var err error
	theURL, _ := url.Parse(u)
	headers := map[string]string{
		"Referer":                   fmt.Sprintf("%s://%s", theURL.Scheme, theURL.Host),
		"User-Agent":                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
		"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
		"Accept-Language":           `en-US,en;q=0.8`,
		"Upgrade-Insecure-Requests": "1",
	}
	c, err = httputil.GetBytes(u, headers, 60*time.Second, 3)
	if err != nil {
		return
	}
	c = bytes.Replace(c, []byte("\r\n"), []byte(""), -1)
	c = bytes.Replace(c, []byte("\r"), []byte(""), -1)
	c = bytes.Replace(c, []byte("\n"), []byte(""), -1)
	for _, m := range pageContentMarkers {
		if theURL.Host == m.host {
			idx := bytes.Index(c, m.start)
			if idx > 1 {
				fmt.Println("found start")
				c = c[idx+len(m.start):]
			}
			idx = bytes.Index(c, m.end)
			if idx > 1 {
				fmt.Println("found end")
				c = c[:idx]
			}
			break
		}
	}

	c = bytes.Replace(c, []byte("

"), []byte("

"), -1)
	c = bytes.Replace(c, []byte(`　　`), []byte(""), -1)
	return
}
<commit_msg>(*)biquge WIP<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"net/url"
	"regexp"
	"time"

	"github.com/dfordsoft/golib/ebook"
	"github.com/dfordsoft/golib/httputil"
)

type tocPattern struct {
	host            string
	bookTitle       string
	bookTitlePos    int
	item            string
	articleTitlePos int
	articleURLPos   int
}

type pageContentMarker struct {
	host  string
	start []byte
	end   []byte
}

var (
	tocPatterns = []tocPattern{
		{
			host:            "www.biqudu.com",
			bookTitle:       `<h1>([^<]+)</h1>$`,
			bookTitlePos:    1,
			item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
			articleURLPos:   1,
			articleTitlePos: 2,
		},
		{
			host:            "www.biquge.cm",
			bookTitle:       `<h1>([^<]+)</h1>$`,
			bookTitlePos:    1,
			item:            `<dd>\s*<a\s*href="([^"]+)">([^<]+)</a></dd>$`,
			articleURLPos:   1,
			articleTitlePos: 2,
		},
		{
			host:            "www.qu.la",
			bookTitle:       `<h1>([^<]+)</h1>$`,
			bookTitlePos:    1,
			item:            `<dd>\s*<a\s*(style=""\s*)?href="([^"]+)">([^<]+)</a></dd>$`,
			articleURLPos:   2,
			articleTitlePos: 3,
		},
	}
	pageContentMarkers = []pageContentMarker{
		{
			host:  "www.biqudu.com",
			start: []byte(`<div id="content"><script>readx();</script>`),
			end:   []byte(`<script>chaptererror();</script>`),
		},
		{
			host:  "www.biquge.cm",
			start: []byte(`<div id="content">    `),
			end:   []byte(`找本站搜索"笔趣阁CM" 或输入网址:www.biquge.cm</div>`),
		},
		{
			host:  "www.qu.la",
			start: []byte(`<div id="content">`),
			end:   []byte(`<script>chaptererror();</script>`),
		},
	}
)

func init() {
	registerNovelSiteHandler(&novelSiteHandler{
		Match:    isBiquge,
		Download: dlBiquge,
	})
}

func isBiquge(u string) bool {
	urlPatterns := []string{
		`http://www\.biquge\.cm/[0-9]+/[0-9]+/`,
		`http://www\.biqugezw\.com/[0-9]+_[0-9]+/`,
		`http://www\.630zw\.com/[0-9]+_[0-9]+/`,
		`http://www\.biqudu\.com/[0-9]+_[0-9]+/`,
		`http://www\.biquge\.lu/book/[0-9]+/`,
		`http://www\.qu\.la/book/[0-9]+/`,
	}

	for _, pattern := range urlPatterns {
		r, _ := regexp.Compile(pattern)
		if r.MatchString(u) {
			return true
		}
	}
	return false
}

func dlBiquge(u string) {
	theURL, _ := url.Parse(u)
	headers := map[string]string{
		"Referer":                   fmt.Sprintf("%s://%s", theURL.Scheme, theURL.Host),
		"User-Agent":                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
		"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
		"Accept-Language":           `en-US,en;q=0.8`,
		"Upgrade-Insecure-Requests": "1",
	}
	b, err := httputil.GetBytes(u, headers, 60*time.Second, 3)
	if err != nil {
		return
	}

	mobi := &ebook.Mobi{}
	mobi.Begin()

	var title string
	var lines []string

	var p tocPattern
	for _, patt := range tocPatterns {
		if theURL.Host == patt.host {
			p = patt
			break
		}
	}
	r, _ := regexp.Compile(p.item)
	re, _ := regexp.Compile(p.bookTitle)
	scanner := bufio.NewScanner(bytes.NewReader(b))
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if title == "" {
			ss := re.FindAllStringSubmatch(line, -1)
			if len(ss) > 0 && len(ss[0]) > 0 {
				s := ss[0]
				title = s[p.bookTitlePos]
				mobi.SetTitle(title)
				continue
			}
		}
		if r.MatchString(line) {
			lines = append(lines, line)
		}
	}
	for i := len(lines) - 1; i >= 0 && i < len(lines) && lines[0] == lines[i]; i -= 2 {
		lines = lines[1:]
	}

	for _, line := range lines {
		ss := r.FindAllStringSubmatch(line, -1)
		s := ss[0]
		finalURL := fmt.Sprintf("%s://%s%s", theURL.Scheme, theURL.Host, s[p.articleURLPos])
		c := dlBiqugePage(finalURL)
		mobi.AppendContent(s[p.articleTitlePos], finalURL, string(c))
		fmt.Println(s[p.articleTitlePos], finalURL, len(c), "bytes")
	}
	mobi.End()
}

func dlBiqugePage(u string) (c []byte) {
	var err error
	theURL, _ := url.Parse(u)
	headers := map[string]string{
		"Referer":                   fmt.Sprintf("%s://%s", theURL.Scheme, theURL.Host),
		"User-Agent":                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0",
		"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
		"Accept-Language":           `en-US,en;q=0.8`,
		"Upgrade-Insecure-Requests": "1",
	}
	c, err = httputil.GetBytes(u, headers, 60*time.Second, 3)
	if err != nil {
		return
	}
	c = bytes.Replace(c, []byte("\r\n"), []byte(""), -1)
	c = bytes.Replace(c, []byte("\r"), []byte(""), -1)
	c = bytes.Replace(c, []byte("\n"), []byte(""), -1)
	for _, m := range pageContentMarkers {
		if theURL.Host == m.host {
			idx := bytes.Index(c, m.start)
			if idx > 1 {
				fmt.Println("found start")
				c = c[idx+len(m.start):]
			}
			idx = bytes.Index(c, m.end)
			if idx > 1 {
				fmt.Println("found end")
				c = c[:idx]
			}
			break
		}
	}

	c = bytes.Replace(c, []byte("

    "), []byte("

"), -1)
	c = bytes.Replace(c, []byte("

"), []byte("

"), -1)
	c = bytes.Replace(c, []byte(`　　`), []byte(""), -1)
	return
}
<|endoftext|>


<commit_before>package nsq

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"net"
	"strconv"
)

type ProtocolClient struct {
	conn io.ReadWriteCloser
}

type ProtocolCommand struct {
	name   []byte
	params [][]byte
}

type ProtocolResponse struct {
	FrameType int32
	Data      interface{}
}

func (c *ProtocolClient) Connect(address string, port int) error {
	fqAddress := address + ":" + strconv.Itoa(port)
	conn, err := net.Dial("tcp", fqAddress)
	if err != nil {
		return err
	}
	c.conn = conn
	return nil
}

func (c *ProtocolClient) Version(version string) error {
	_, err := c.Write([]byte(version))
	return err
}

func (c *ProtocolClient) Write(data []byte) (int, error) {
	return c.conn.Write(data)
}

func (c *ProtocolClient) WriteCommand(cmd *ProtocolCommand) error {
	if len(cmd.params) > 0 {
		_, err := fmt.Fprintf(c.conn, "%s %s\n", cmd.name, string(bytes.Join(cmd.params, []byte(" "))))
		if err != nil {
			return err
		}
	} else {
		_, err := fmt.Fprintf(c.conn, "%s\n", cmd.name)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *ProtocolClient) ReadResponse() (*ProtocolResponse, error) {
	var err error
	var msgSize int32
	var frameType int32

	// message size
	err = binary.Read(c.conn, binary.BigEndian, &msgSize)
	if err != nil {
		return nil, err
	}

	// frame type
	err = binary.Read(c.conn, binary.BigEndian, &frameType)
	if err != nil {
		return nil, err
	}

	// message binary data
	buf := make([]byte, msgSize-4)
	_, err = io.ReadFull(c.conn, buf)
	if err != nil {
		return nil, err
	}

	resp := ProtocolResponse{}
	resp.FrameType = frameType
	switch resp.FrameType {
	case FrameTypeMessage:
		resp.Data = NewMessage(buf)
		break
	default:
		resp.Data = buf
	}

	return &resp, nil
}
<commit_msg>use net.TCPAddr<commit_after>package nsq

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"net"
)

type ProtocolClient struct {
	conn io.ReadWriteCloser
}

type ProtocolCommand struct {
	name   []byte
	params [][]byte
}

type ProtocolResponse struct {
	FrameType int32
	Data      interface{}
}

func (c *ProtocolClient) Connect(tcpAddr *net.TCPAddr) error {
	conn, err := net.Dial("tcp", tcpAddr.String())
	if err != nil {
		return err
	}
	c.conn = conn
	return nil
}

func (c *ProtocolClient) Version(version string) error {
	_, err := c.Write([]byte(version))
	return err
}

func (c *ProtocolClient) Write(data []byte) (int, error) {
	return c.conn.Write(data)
}

func (c *ProtocolClient) WriteCommand(cmd *ProtocolCommand) error {
	if len(cmd.params) > 0 {
		_, err := fmt.Fprintf(c.conn, "%s %s\n", cmd.name, string(bytes.Join(cmd.params, []byte(" "))))
		if err != nil {
			return err
		}
	} else {
		_, err := fmt.Fprintf(c.conn, "%s\n", cmd.name)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *ProtocolClient) ReadResponse() (*ProtocolResponse, error) {
	var err error
	var msgSize int32
	var frameType int32

	// message size
	err = binary.Read(c.conn, binary.BigEndian, &msgSize)
	if err != nil {
		return nil, err
	}

	// frame type
	err = binary.Read(c.conn, binary.BigEndian, &frameType)
	if err != nil {
		return nil, err
	}

	// message binary data
	buf := make([]byte, msgSize-4)
	_, err = io.ReadFull(c.conn, buf)
	if err != nil {
		return nil, err
	}

	resp := ProtocolResponse{}
	resp.FrameType = frameType
	switch resp.FrameType {
	case FrameTypeMessage:
		resp.Data = NewMessage(buf)
		break
	default:
		resp.Data = buf
	}

	return &resp, nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"reflect"
	"strconv"
	"time"

	docker_types "github.com/docker/docker/api/types"
	docker_types_container "github.com/docker/docker/api/types/container"
	docker_types_event "github.com/docker/docker/api/types/events"
	docker_types_network "github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	docker_nat "github.com/docker/go-connections/nat"
)

// DockerPorts describes docker ports for docker run
type DockerPorts struct {
	HostIP        string `json:"host_ip"`
	HostPort      int    `json:"host_port"`
	ContainerPort int    `json:"container_port"`
	Protocol      string `json:"protocol"`
}

// DockerCreateOpts describes docker run options
type DockerCreateOpts struct {
	Name       string         `json:"name"`
	Image      string         `json:"image"`
	Remove     bool           `json:"rm"`
	Ports      []*DockerPorts `json:"ports"`
	PublishAll bool           `json:"publish_all"`
	Command    string         `json:"command"`
	Entrypoint string         `json:"entrypoint"`
	Env        []string       `json:"env"`
	Binds      []string       `json:"binds"`
}

// DockerPingOpts describes the structure to ping docker containers in pikacloud API
type DockerPingOpts struct {
	Containers []string `json:"containers_id"`
}

// DockerPullOpts describes docker pull options
type DockerPullOpts struct {
	Image string `json:"image"`
}

// DockerUnpauseOpts describes docker unpause options
type DockerUnpauseOpts struct {
	ID string `json:"id"`
}

// DockerPauseOpts describes docker pause options
type DockerPauseOpts struct {
	ID string `json:"id"`
}

// DockerStopOpts describes docker stop options
type DockerStopOpts struct {
	ID string `json:"id"`
}

// DockerStartOpts describes docker start options
type DockerStartOpts struct {
	ID string `json:"id"`
}

// DockerRemoveOpts describes docker remove options
type DockerRemoveOpts struct {
	ID            string `json:"id"`
	Force         bool   `json:"force"`
	RemoveLinks   bool   `json:"remove_links"`
	RemoveVolumes bool   `json:"remove_volumes"`
}

// AgentDockerInfo describes docker info
type AgentDockerInfo struct {
	Info docker_types.Info `json:"info"`
}

// AgentContainer describes docker container
type AgentContainer struct {
	ID        string `json:"cid"`
	Container string `json:"container"`
	Config    string `json:"config"`
}

type syncDockerContainersOptions struct {
	ContainersID []string
}

func (agent *Agent) dockerPull(opts *DockerPullOpts) error {
	log.Printf("Pulling %s", opts.Image)
	ctx := context.Background()
	pullOpts := docker_types.ImagePullOptions{}
	out, err := agent.DockerClient.ImagePull(ctx, opts.Image, pullOpts)
	if err != nil {
		return err
	}
	defer out.Close()
	if _, err = io.Copy(ioutil.Discard, out); err != nil {
		return err
	}
	log.Printf("New image pulled %s", opts.Image)
	return nil
}

func (agent *Agent) dockerCreate(opts *DockerCreateOpts) (*docker_types_container.ContainerCreateCreatedBody, error) {
	ctx := context.Background()
	config := &docker_types_container.Config{
		Image: opts.Image,
		Env:   opts.Env,
	}
	if opts.Entrypoint != "" {
		config.Entrypoint = strslice.StrSlice{opts.Entrypoint}
	}
	if opts.Command != "" {
		config.Cmd = strslice.StrSlice{opts.Command}
	}
	natPortmap := docker_nat.PortMap{}
	for _, p := range opts.Ports {
		containerPortProto := docker_nat.Port(fmt.Sprintf("%d/%s", p.ContainerPort, p.Protocol))
		dockerHostConfig := docker_nat.PortBinding{
			HostIP:   p.HostIP,
			HostPort: strconv.Itoa(p.HostPort),
		}
		natPortmap[containerPortProto] = append(natPortmap[containerPortProto], dockerHostConfig)
	}
	hostConfig := &docker_types_container.HostConfig{
		Binds:           opts.Binds,
		PublishAllPorts: opts.PublishAll,
		PortBindings:    natPortmap,
	}
	networkingConfig := &docker_types_network.NetworkingConfig{}

	container, err := agent.DockerClient.ContainerCreate(ctx, config, hostConfig, networkingConfig, opts.Name)
	if err != nil {
		return nil, err
	}
	log.Printf("New container created %s", container.ID)
	return &container, nil
}

func (agent *Agent) dockerStart(containerID string) error {
	ctx := context.Background()
	startOpts := docker_types.ContainerStartOptions{}
	if err := agent.DockerClient.ContainerStart(ctx, containerID, startOpts); err != nil {
		return err
	}
	log.Printf("New container started %s", containerID)
	return nil
}

func (agent *Agent) dockerUnpause(containerID string) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerUnpause(ctx, containerID); err != nil {
		return err
	}
	log.Printf("Container %s unpaused", containerID)
	return nil
}

func (agent *Agent) dockerPause(containerID string) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerPause(ctx, containerID); err != nil {
		return err
	}
	log.Printf("Container %s paused", containerID)
	return nil
}

func (agent *Agent) dockerStop(containerID string, timeout time.Duration) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerStop(ctx, containerID, &timeout); err != nil {
		return err
	}
	log.Printf("Container %s stopped", containerID)
	return nil
}

func (agent *Agent) dockerRemove(containerID string, opts *DockerRemoveOpts) error {
	removeOpts := docker_types.ContainerRemoveOptions{
		Force:         opts.Force,
		RemoveVolumes: opts.RemoveVolumes,
		RemoveLinks:   opts.RemoveLinks,
	}
	ctx := context.Background()
	if err := agent.DockerClient.ContainerRemove(ctx, containerID, removeOpts); err != nil {
		return err
	}
	log.Printf("Container %s remove", containerID)
	return nil
}

func (agent *Agent) infiniteSyncDockerInfo() {
	dockerInfoState := docker_types.Info{}
	for {
		info, err := agent.DockerClient.Info(context.Background())
		if err != nil {
			time.Sleep(3 * time.Second)
			continue
		}
		// compare
		dockerInfoState.SystemTime = ""
		info.SystemTime = ""
		if !reflect.DeepEqual(dockerInfoState, info) {
			err := agent.syncDockerInfo(info)
			if err != nil {
				log.Printf("Cannot sync docker info: %+v", err)
			} else {
				dockerInfoState = info
				log.Println("Sync docker info OK")
			}
		}
		time.Sleep(3 * time.Second)
	}
}

func (agent *Agent) syncDockerInfo(info docker_types.Info) error {
	uri := fmt.Sprintf("run/agents/%s/docker/info/", agent.ID)
	pingInfo := AgentDockerInfo{
		Info: info,
	}
	status, err := agent.Client.Put(uri, pingInfo, nil)
	if err != nil {
		return err
	}
	if status != 200 {
		return fmt.Errorf("Failed to push docker info: %d", status)
	}
	return nil
}

func (agent *Agent) syncDockerContainers(opts syncDockerContainersOptions) error {
	containersListOpts := docker_types.ContainerListOptions{
		All: true,
	}
	var containersCreateList []AgentContainer
	uri := fmt.Sprintf("run/agents/%s/docker/containers/", agent.ID)
	containers, err := agent.DockerClient.ContainerList(context.Background(), containersListOpts)
	if err != nil {
		return err
	}
	for _, container := range containers {
		data, err := json.Marshal(container)
		if err != nil {
			log.Printf("Cannot decode %v", container)
			continue
		}
		if len(opts.ContainersID) > 0 {
			skip := true
			for _, c := range opts.ContainersID {
				if c == container.ID {
					skip = false
					break
				}
			}
			if skip {
				continue
			}
		}
		inspect, err := agent.DockerClient.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			log.Printf("Cannot inspect container %v", err)
			continue
		}
		inspectConfig, err := json.Marshal(inspect)
		if err != nil {
			log.Printf("Cannot decode %v", inspect)
			continue
		}
		containersCreateList = append(containersCreateList,
			AgentContainer{
				ID:        container.ID,
				Container: string(data),
				Config:    string(inspectConfig),
			})
	}
	if len(containersCreateList) > 0 {
		status, err := agent.Client.Post(uri, containersCreateList, nil)
		if err != nil {
			return err
		}
		if status != 200 {
			return fmt.Errorf("Failed to push docker containers: %d", status)
		}
		log.Printf("Sync docker %d containers of %d OK", len(containersCreateList), len(containers))
	}

	return nil

}

func (agent *Agent) unsyncDockerContainer(containerID string) error {
	deleteContainerURI := fmt.Sprintf("run/agents/%s/docker/containers/%s/", agent.ID, containerID)
	_, err := agent.Client.Delete(deleteContainerURI, nil, nil)
	if err != nil {
		return err
	}
	log.Printf("Container %s garbage collected", containerID)
	return nil
}

func (agent *Agent) parseContainerEvent(msg docker_types_event.Message) error {
	containerID := msg.ID
	switch msg.Action {
	case "destroy":
		err := agent.unsyncDockerContainer(containerID)
		if err != nil {
			return err
		}
	default:

		opts := syncDockerContainersOptions{
			ContainersID: []string{containerID},
		}
		err := agent.syncDockerContainers(opts)
		if err != nil {
			return err
		}
	}
	return nil
}

func (agent *Agent) parseDockerEvent(msg docker_types_event.Message) error {
	switch msg.Type {
	case "container":
		err := agent.parseContainerEvent(msg)
		if err != nil {
			return err
		}
	}
	return nil
}

func (agent *Agent) listenDockerEvents() error {
	events, errs := agent.DockerClient.Events(context.Background(), docker_types.EventsOptions{})
	for {
		select {
		case dMsg := <-events:
			err := agent.parseDockerEvent(dMsg)
			if err != nil {
				log.Println(err)
			}
		case dErr := <-errs:
			if dErr != nil {
				fmt.Println(dErr)
			}
		}
	}
}

// Run docker container
func (agent *Agent) Run() {

}
<commit_msg>labels / user / working dir<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"reflect"
	"strconv"
	"time"

	docker_types "github.com/docker/docker/api/types"
	docker_types_container "github.com/docker/docker/api/types/container"
	docker_types_event "github.com/docker/docker/api/types/events"
	docker_types_network "github.com/docker/docker/api/types/network"
	"github.com/docker/docker/api/types/strslice"
	docker_nat "github.com/docker/go-connections/nat"
)

// DockerPorts describes docker ports for docker run
type DockerPorts struct {
	HostIP        string `json:"host_ip"`
	HostPort      int    `json:"host_port"`
	ContainerPort int    `json:"container_port"`
	Protocol      string `json:"protocol"`
}

// DockerCreateOpts describes docker run options
type DockerCreateOpts struct {
	Name       string            `json:"name"`
	Image      string            `json:"image"`
	Remove     bool              `json:"rm"`
	Ports      []*DockerPorts    `json:"ports"`
	PublishAll bool              `json:"publish_all"`
	Command    string            `json:"command"`
	Entrypoint string            `json:"entrypoint"`
	Env        []string          `json:"env"`
	Binds      []string          `json:"binds"`
	User       string            `json:"user"`
	WorkingDir string            `json:"working_dir"`
	Labels     map[string]string `json:"labels"`
}

// DockerPingOpts describes the structure to ping docker containers in pikacloud API
type DockerPingOpts struct {
	Containers []string `json:"containers_id"`
}

// DockerPullOpts describes docker pull options
type DockerPullOpts struct {
	Image string `json:"image"`
}

// DockerUnpauseOpts describes docker unpause options
type DockerUnpauseOpts struct {
	ID string `json:"id"`
}

// DockerPauseOpts describes docker pause options
type DockerPauseOpts struct {
	ID string `json:"id"`
}

// DockerStopOpts describes docker stop options
type DockerStopOpts struct {
	ID string `json:"id"`
}

// DockerStartOpts describes docker start options
type DockerStartOpts struct {
	ID string `json:"id"`
}

// DockerRemoveOpts describes docker remove options
type DockerRemoveOpts struct {
	ID            string `json:"id"`
	Force         bool   `json:"force"`
	RemoveLinks   bool   `json:"remove_links"`
	RemoveVolumes bool   `json:"remove_volumes"`
}

// AgentDockerInfo describes docker info
type AgentDockerInfo struct {
	Info docker_types.Info `json:"info"`
}

// AgentContainer describes docker container
type AgentContainer struct {
	ID        string `json:"cid"`
	Container string `json:"container"`
	Config    string `json:"config"`
}

type syncDockerContainersOptions struct {
	ContainersID []string
}

func (agent *Agent) dockerPull(opts *DockerPullOpts) error {
	log.Printf("Pulling %s", opts.Image)
	ctx := context.Background()
	pullOpts := docker_types.ImagePullOptions{}
	out, err := agent.DockerClient.ImagePull(ctx, opts.Image, pullOpts)
	if err != nil {
		return err
	}
	defer out.Close()
	if _, err = io.Copy(ioutil.Discard, out); err != nil {
		return err
	}
	log.Printf("New image pulled %s", opts.Image)
	return nil
}

func (agent *Agent) dockerCreate(opts *DockerCreateOpts) (*docker_types_container.ContainerCreateCreatedBody, error) {
	ctx := context.Background()
	config := &docker_types_container.Config{
		Image:  opts.Image,
		Env:    opts.Env,
		Labels: opts.Labels,
	}
	if opts.User != "" {
		config.User = opts.User
	}
	if opts.WorkingDir != "" {
		config.WorkingDir = opts.WorkingDir
	}
	if opts.Entrypoint != "" {
		config.Entrypoint = strslice.StrSlice{opts.Entrypoint}
	}
	if opts.Command != "" {
		config.Cmd = strslice.StrSlice{opts.Command}
	}
	natPortmap := docker_nat.PortMap{}
	for _, p := range opts.Ports {
		containerPortProto := docker_nat.Port(fmt.Sprintf("%d/%s", p.ContainerPort, p.Protocol))
		dockerHostConfig := docker_nat.PortBinding{
			HostIP:   p.HostIP,
			HostPort: strconv.Itoa(p.HostPort),
		}
		natPortmap[containerPortProto] = append(natPortmap[containerPortProto], dockerHostConfig)
	}
	hostConfig := &docker_types_container.HostConfig{
		Binds:           opts.Binds,
		PublishAllPorts: opts.PublishAll,
		PortBindings:    natPortmap,
	}
	networkingConfig := &docker_types_network.NetworkingConfig{}

	container, err := agent.DockerClient.ContainerCreate(ctx, config, hostConfig, networkingConfig, opts.Name)
	if err != nil {
		return nil, err
	}
	log.Printf("New container created %s", container.ID)
	return &container, nil
}

func (agent *Agent) dockerStart(containerID string) error {
	ctx := context.Background()
	startOpts := docker_types.ContainerStartOptions{}
	if err := agent.DockerClient.ContainerStart(ctx, containerID, startOpts); err != nil {
		return err
	}
	log.Printf("New container started %s", containerID)
	return nil
}

func (agent *Agent) dockerUnpause(containerID string) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerUnpause(ctx, containerID); err != nil {
		return err
	}
	log.Printf("Container %s unpaused", containerID)
	return nil
}

func (agent *Agent) dockerPause(containerID string) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerPause(ctx, containerID); err != nil {
		return err
	}
	log.Printf("Container %s paused", containerID)
	return nil
}

func (agent *Agent) dockerStop(containerID string, timeout time.Duration) error {
	ctx := context.Background()
	if err := agent.DockerClient.ContainerStop(ctx, containerID, &timeout); err != nil {
		return err
	}
	log.Printf("Container %s stopped", containerID)
	return nil
}

func (agent *Agent) dockerRemove(containerID string, opts *DockerRemoveOpts) error {
	removeOpts := docker_types.ContainerRemoveOptions{
		Force:         opts.Force,
		RemoveVolumes: opts.RemoveVolumes,
		RemoveLinks:   opts.RemoveLinks,
	}
	ctx := context.Background()
	if err := agent.DockerClient.ContainerRemove(ctx, containerID, removeOpts); err != nil {
		return err
	}
	log.Printf("Container %s remove", containerID)
	return nil
}

func (agent *Agent) infiniteSyncDockerInfo() {
	dockerInfoState := docker_types.Info{}
	for {
		info, err := agent.DockerClient.Info(context.Background())
		if err != nil {
			time.Sleep(3 * time.Second)
			continue
		}
		// compare
		dockerInfoState.SystemTime = ""
		info.SystemTime = ""
		if !reflect.DeepEqual(dockerInfoState, info) {
			err := agent.syncDockerInfo(info)
			if err != nil {
				log.Printf("Cannot sync docker info: %+v", err)
			} else {
				dockerInfoState = info
				log.Println("Sync docker info OK")
			}
		}
		time.Sleep(3 * time.Second)
	}
}

func (agent *Agent) syncDockerInfo(info docker_types.Info) error {
	uri := fmt.Sprintf("run/agents/%s/docker/info/", agent.ID)
	pingInfo := AgentDockerInfo{
		Info: info,
	}
	status, err := agent.Client.Put(uri, pingInfo, nil)
	if err != nil {
		return err
	}
	if status != 200 {
		return fmt.Errorf("Failed to push docker info: %d", status)
	}
	return nil
}

func (agent *Agent) syncDockerContainers(opts syncDockerContainersOptions) error {
	containersListOpts := docker_types.ContainerListOptions{
		All: true,
	}
	var containersCreateList []AgentContainer
	uri := fmt.Sprintf("run/agents/%s/docker/containers/", agent.ID)
	containers, err := agent.DockerClient.ContainerList(context.Background(), containersListOpts)
	if err != nil {
		return err
	}
	for _, container := range containers {
		data, err := json.Marshal(container)
		if err != nil {
			log.Printf("Cannot decode %v", container)
			continue
		}
		if len(opts.ContainersID) > 0 {
			skip := true
			for _, c := range opts.ContainersID {
				if c == container.ID {
					skip = false
					break
				}
			}
			if skip {
				continue
			}
		}
		inspect, err := agent.DockerClient.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			log.Printf("Cannot inspect container %v", err)
			continue
		}
		inspectConfig, err := json.Marshal(inspect)
		if err != nil {
			log.Printf("Cannot decode %v", inspect)
			continue
		}
		containersCreateList = append(containersCreateList,
			AgentContainer{
				ID:        container.ID,
				Container: string(data),
				Config:    string(inspectConfig),
			})
	}
	if len(containersCreateList) > 0 {
		status, err := agent.Client.Post(uri, containersCreateList, nil)
		if err != nil {
			return err
		}
		if status != 200 {
			return fmt.Errorf("Failed to push docker containers: %d", status)
		}
		log.Printf("Sync docker %d containers of %d OK", len(containersCreateList), len(containers))
	}

	return nil

}

func (agent *Agent) unsyncDockerContainer(containerID string) error {
	deleteContainerURI := fmt.Sprintf("run/agents/%s/docker/containers/%s/", agent.ID, containerID)
	_, err := agent.Client.Delete(deleteContainerURI, nil, nil)
	if err != nil {
		return err
	}
	log.Printf("Container %s garbage collected", containerID)
	return nil
}

func (agent *Agent) parseContainerEvent(msg docker_types_event.Message) error {
	containerID := msg.ID
	switch msg.Action {
	case "destroy":
		err := agent.unsyncDockerContainer(containerID)
		if err != nil {
			return err
		}
	default:

		opts := syncDockerContainersOptions{
			ContainersID: []string{containerID},
		}
		err := agent.syncDockerContainers(opts)
		if err != nil {
			return err
		}
	}
	return nil
}

func (agent *Agent) parseDockerEvent(msg docker_types_event.Message) error {
	switch msg.Type {
	case "container":
		err := agent.parseContainerEvent(msg)
		if err != nil {
			return err
		}
	}
	return nil
}

func (agent *Agent) listenDockerEvents() error {
	events, errs := agent.DockerClient.Events(context.Background(), docker_types.EventsOptions{})
	for {
		select {
		case dMsg := <-events:
			err := agent.parseDockerEvent(dMsg)
			if err != nil {
				log.Println(err)
			}
		case dErr := <-errs:
			if dErr != nil {
				fmt.Println(dErr)
			}
		}
	}
}

// Run docker container
func (agent *Agent) Run() {

}
<|endoftext|>


<commit_before>package bark

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"

	"github.com/jprobinson/newshound"
)

type slackKey struct {
	key     string
	botName string
}

func AddSlackAlertBot(d *Distributor, key, botName string) {
	skey := slackKey{key, botName}
	d.AddAlertBarker(&SlackAlertBarker{skey})
}

func AddSlackEventBot(d *Distributor, key, botName string) {
	skey := slackKey{key, botName}
	d.AddEventBarker(&SlackEventBarker{skey})
}

type SlackAlertBarker struct {
	key slackKey
}

func (s *SlackAlertBarker) Bark(alert newshound.NewsAlertLite) error {
	title := fmt.Sprintf("%s - %s", strings.TrimSuffix(alert.Sender, ".com"), alert.Subject)
	link := fmt.Sprintf("http://newshound.jprbnsn.com/#/calendar?start=%s&display=alerts&alert=%s",
		alert.Timestamp.Format("2006-01-02"),
		alert.ID.Hex())
	message := fmt.Sprintf("\n%s\n<%s|more...>", alert.TopSentence, link)
	color := SenderColors[strings.ToLower(alert.Sender)]
	return sendSlack(s.key.botName, s.key.key, title, link, message, color)
}

type SlackEventBarker struct {
	key slackKey
}

func (s *SlackEventBarker) Bark(event newshound.NewsEvent) error {
	title := fmt.Sprintf("New Event With %d Alerts!", len(event.NewsAlerts))
	link := fmt.Sprintf("http://newshound.jprbnsn.com/#/calendar?start=%s&display=events&alert=%s",
		event.EventStart.Format("2006-01-02"),
		event.ID.Hex())
	message := fmt.Sprintf("_key quote_\n%s\n_from_\n%s\n<%s|more info...>",
		event.TopSentence,
		strings.TrimSuffix(event.TopSender, ".com"),
		link)
	return sendSlack(s.key.botName, s.key.key, title, link, message, "#439FE0")
}

type slackAttachment struct {
	Title     string   `json:"title"`
	TitleLink string   `json:"title_link,omitempty"`
	Text      string   `json:"text"`
	Fallback  string   `json:"fallback"`
	Color     string   `json:"color"`
	MrkDownIn []string `json:"mrkdwn_in"`
}

func sendSlack(bot, key, title, link, message, color string) error {
	data := struct {
		Username    string            `json:"username"`
		Unfurl      bool              `json:"unfurl_links"`
		MrkDwn      bool              `json:"mrkdwn"`
		Attachments []slackAttachment `json:"attachments"`
	}{
		bot,
		false,
		true,
		[]slackAttachment{
			slackAttachment{
				title,
				link,
				message,
				message,
				color,
				[]string{"text", "fallback"},
			},
		},
	}

	var payload bytes.Buffer
	err := json.NewEncoder(&payload).Encode(data)
	if err != nil {
		log.Print("unable to encode slack json:", err)
		return err
	}

	slackURL := fmt.Sprintf("https://hooks.slack.com/services/%s", key)
	r, err := http.Post(slackURL, "application/json", &payload)
	defer r.Body.Close()
	if err != nil {
		resp, _ := ioutil.ReadAll(r.Body)
		log.Printf("unable to send slack notification: %s\nresponse:\n%s", err, string(resp))
	}

	return err
}
<commit_msg>fixing event URL<commit_after>package bark

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"

	"github.com/jprobinson/newshound"
)

type slackKey struct {
	key     string
	botName string
}

func AddSlackAlertBot(d *Distributor, key, botName string) {
	skey := slackKey{key, botName}
	d.AddAlertBarker(&SlackAlertBarker{skey})
}

func AddSlackEventBot(d *Distributor, key, botName string) {
	skey := slackKey{key, botName}
	d.AddEventBarker(&SlackEventBarker{skey})
}

type SlackAlertBarker struct {
	key slackKey
}

func (s *SlackAlertBarker) Bark(alert newshound.NewsAlertLite) error {
	title := fmt.Sprintf("%s - %s", strings.TrimSuffix(alert.Sender, ".com"), alert.Subject)
	link := fmt.Sprintf("http://newshound.jprbnsn.com/#/calendar?start=%s&display=alerts&alert=%s",
		alert.Timestamp.Format("2006-01-02"),
		alert.ID.Hex())
	message := fmt.Sprintf("\n%s\n<%s|more...>", alert.TopSentence, link)
	color := SenderColors[strings.ToLower(alert.Sender)]
	return sendSlack(s.key.botName, s.key.key, title, link, message, color)
}

type SlackEventBarker struct {
	key slackKey
}

func (s *SlackEventBarker) Bark(event newshound.NewsEvent) error {
	title := fmt.Sprintf("New Event With %d Alerts!", len(event.NewsAlerts))
	link := fmt.Sprintf("http://newshound.jprbnsn.com/#/calendar?start=%s&display=events&event=%s",
		event.EventStart.Format("2006-01-02"),
		event.ID.Hex())
	message := fmt.Sprintf("_key quote_\n%s\n_from_\n%s\n<%s|more info...>",
		event.TopSentence,
		strings.TrimSuffix(event.TopSender, ".com"),
		link)
	return sendSlack(s.key.botName, s.key.key, title, link, message, "#439FE0")
}

type slackAttachment struct {
	Title     string   `json:"title"`
	TitleLink string   `json:"title_link,omitempty"`
	Text      string   `json:"text"`
	Fallback  string   `json:"fallback"`
	Color     string   `json:"color"`
	MrkDownIn []string `json:"mrkdwn_in"`
}

func sendSlack(bot, key, title, link, message, color string) error {
	data := struct {
		Username    string            `json:"username"`
		Unfurl      bool              `json:"unfurl_links"`
		MrkDwn      bool              `json:"mrkdwn"`
		Attachments []slackAttachment `json:"attachments"`
	}{
		bot,
		false,
		true,
		[]slackAttachment{
			slackAttachment{
				title,
				link,
				message,
				message,
				color,
				[]string{"text", "fallback"},
			},
		},
	}

	var payload bytes.Buffer
	err := json.NewEncoder(&payload).Encode(data)
	if err != nil {
		log.Print("unable to encode slack json:", err)
		return err
	}

	slackURL := fmt.Sprintf("https://hooks.slack.com/services/%s", key)
	r, err := http.Post(slackURL, "application/json", &payload)
	defer r.Body.Close()
	if err != nil {
		resp, _ := ioutil.ReadAll(r.Body)
		log.Printf("unable to send slack notification: %s\nresponse:\n%s", err, string(resp))
	}

	return err
}
<|endoftext|>