<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"strconv"

	"github.com/unrolled/render"
	"github.com/zenazn/goji/web"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"
)

func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	ren := render.New(render.Options{
		Layout:     "layout",
		Extensions: []string{".html"},
	})
	ren.HTML(w, http.StatusOK, "index", nil)
}

func spotHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	ren := render.New()
	ctx := appengine.NewContext(r)
	spots := []Spot{}
	_, err := datastore.NewQuery("Spot").Order("-UpdatedAt").GetAll(ctx, &spots)
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]interface{}{"message": "error"})
		return
	}
	ren.JSON(w, http.StatusOK, map[string]interface{}{"items": spots})
}

func spotGetHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	ren := render.New()
	spotCode, err := strconv.ParseInt(c.URLParams["spotCode"], 10, 64)
	if err != nil {
		log.Infof(ctx, "failed to parse int :%v", err)
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, bad resource id"})
		return
	}
	var spot Spot
	spot.SpotCode = spotCode
	err = datastore.Get(ctx, spot.key(ctx), &spot)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]interface{}{"message": "error, entity not found"})
		return
	}
	ren.JSON(w, http.StatusOK, map[string]interface{}{"item": spot})
}

func spotCreateHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	var spot Spot
	ctx := appengine.NewContext(r)
	ren := render.New()
	err := json.NewDecoder(r.Body).Decode(&spot)
	if err != nil {
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, cannot decode json"})
		log.Infof(ctx, "failed to parse JSON:%v", err)
		return
	}
	_, err = spot.Create(ctx)
	if err != nil {
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, cannot create new entity"})
		return
	}
	ren.JSON(w, http.StatusCreated, map[string]interface{}{"message": "new entity created", "item": spot})
}

func spotUpdateHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	ren := render.New()
	spotCode, err := strconv.ParseInt(c.URLParams["spotCode"], 10, 64)
	if err != nil {
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, bad resource id"})
		return
	}
	var checkSpot Spot
	checkSpot.SpotCode = spotCode
	currentUser := user.Current(ctx)
	err = datastore.Get(ctx, checkSpot.key(ctx), &checkSpot)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]interface{}{"message": "error, entity not found"})
		return
	}
	if currentUser.ID != checkSpot.Editor {
		ren.JSON(w, http.StatusForbidden, map[string]interface{}{"message": "error, you don't have right to write this entity"})
		return
	}
	var updateSpot Spot
	err = json.NewDecoder(r.Body).Decode(&updateSpot)
	if err != nil {
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, cannot decode json"})
		log.Infof(ctx, "failed to parse JSON:%v", err)
		return
	}
	_, err = updateSpot.Update(ctx, spotCode)
	if err != nil {
		ren.JSON(w, http.StatusForbidden, map[string]interface{}{"message": "error, you cannot edit this spot"})
		return
	}
	ren.JSON(w, http.StatusCreated, map[string]interface{}{"message": "new entity created"})
}
<commit_msg>update update function<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"strconv"

	"github.com/unrolled/render"
	"github.com/zenazn/goji/web"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"
)

func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	ren := render.New(render.Options{
		Layout:     "layout",
		Extensions: []string{".html"},
	})
	ren.HTML(w, http.StatusOK, "index", nil)
}

func spotHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	ren := render.New()
	ctx := appengine.NewContext(r)
	spots := []Spot{}
	_, err := datastore.NewQuery("Spot").Order("-UpdatedAt").GetAll(ctx, &spots)
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]interface{}{"message": "error"})
		return
	}
	ren.JSON(w, http.StatusOK, map[string]interface{}{"items": spots})
}

func spotGetHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	ren := render.New()
	spotCode, err := strconv.ParseInt(c.URLParams["spotCode"], 10, 64)
	if err != nil {
		log.Infof(ctx, "failed to parse int :%v", err)
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, bad resource id"})
		return
	}
	var spot Spot
	spot.SpotCode = spotCode
	err = datastore.Get(ctx, spot.key(ctx), &spot)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]interface{}{"message": "error, entity not found"})
		return
	}
	ren.JSON(w, http.StatusOK, map[string]interface{}{"item": spot})
}

func spotCreateHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	var spot Spot
	ctx := appengine.NewContext(r)
	ren := render.New()
	err := json.NewDecoder(r.Body).Decode(&spot)
	if err != nil {
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, cannot decode json"})
		log.Infof(ctx, "failed to parse JSON:%v", err)
		return
	}
	_, err = spot.Create(ctx)
	if err != nil {
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, cannot create new entity"})
		return
	}
	ren.JSON(w, http.StatusCreated, map[string]interface{}{"message": "new entity created", "item": spot})
}

func spotUpdateHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	ren := render.New()
	spotCode, err := strconv.ParseInt(c.URLParams["spotCode"], 10, 64)
	if err != nil {
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, bad resource id"})
		return
	}
	var checkSpot Spot
	checkSpot.SpotCode = spotCode
	currentUser := user.Current(ctx)
	err = datastore.Get(ctx, checkSpot.key(ctx), &checkSpot)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]interface{}{"message": "error, entity not found"})
		return
	}
	if currentUser.ID != checkSpot.Editor {
		ren.JSON(w, http.StatusForbidden, map[string]interface{}{"message": "error, you don't have right to write this entity"})
		return
	}
	var updateSpot Spot
	err = json.NewDecoder(r.Body).Decode(&updateSpot)
	if err != nil {
		ren.JSON(w, http.StatusBadRequest, map[string]interface{}{"message": "error, cannot decode json"})
		log.Infof(ctx, "failed to parse JSON:%v", err)
		return
	}
	updateSpot.Editor = checkSpot.Editor
	_, err = updateSpot.Update(ctx, spotCode)
	if err != nil {
		ren.JSON(w, http.StatusForbidden, map[string]interface{}{"message": "error, you cannot edit this spot"})
		return
	}
	ren.JSON(w, http.StatusCreated, map[string]interface{}{"message": "entity updated", "item":updateSpot})
}
<|endoftext|>


<commit_before>package quick3way

import (
	"fmt"
	"math/rand"
	"testing"
	"time"
)

type quick []int

func (q quick) Len() int {
	return len(q)
}

func (q quick) Less(i, j int) bool {
	return q[i] < q[j]
}

func (q quick) More(i, j int) bool {
	return q[i] > q[j]
}

func (q quick) Swap(i, j int) {
	q[i], q[j] = q[j], q[i]
}

func (q quick) Divide(i, j int) Interface {
	return q[i:j]
}

func (q quick) IsSorted() bool {
	for i := 1; i < q.Len(); i++ {
		if q.More(i-1, i) {
			return false
		}
	}
	return true
}

func Test_Sort(t *testing.T) {
	rand.Seed(time.Now().UnixNano())
	n := 10
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	fmt.Println(q)
	Sort(q)
	fmt.Println(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_100(t *testing.T) {
	n := 100
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_1000(t *testing.T) {
	n := 1000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_10000(t *testing.T) {
	n := 10000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_100000(t *testing.T) {
	n := 100000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_1000000(t *testing.T) {
	n := 1000000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_10000000(t *testing.T) {
	n := 10000000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}
<commit_msg>添加测试文件，证明quick3way也能处理没有重复项的情况。<commit_after>package quick3way

import (
	"fmt"
	"math/rand"
	"testing"
	"time"
)

type quick []int

func (q quick) Len() int {
	return len(q)
}

func (q quick) Less(i, j int) bool {
	return q[i] < q[j]
}

func (q quick) More(i, j int) bool {
	return q[i] > q[j]
}

func (q quick) Swap(i, j int) {
	q[i], q[j] = q[j], q[i]
}

func (q quick) Divide(i, j int) Interface {
	return q[i:j]
}

func (q quick) IsSorted() bool {
	for i := 1; i < q.Len(); i++ {
		if q.More(i-1, i) {
			return false
		}
	}
	return true
}

func Test_Sort(t *testing.T) {
	rand.Seed(time.Now().UnixNano())
	n := 10
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	fmt.Println(q)
	Sort(q)
	fmt.Println(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}
func Test_Sort_without_Repeating(t *testing.T) {
	rand.Seed(time.Now().UnixNano())
	n := 20
	q := make(quick, n)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
	}
	fmt.Println(q)
	Sort(q)
	fmt.Println(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}
func Test_Sort_100(t *testing.T) {
	n := 100
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_1000(t *testing.T) {
	n := 1000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_10000(t *testing.T) {
	n := 10000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_100000(t *testing.T) {
	n := 100000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_1000000(t *testing.T) {
	n := 1000000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}

func Test_Sort_10000000(t *testing.T) {
	n := 10000000
	q := make(quick, n*3)
	for i := 0; i < n; i++ {
		q[i] = n - i - 1
		q[i+n], q[i+n+n] = q[i], q[i]
	}
	Sort(q)
	if !q.IsSorted() {
		t.Error("没能排序好")
	}
}
<|endoftext|>


<commit_before>// Package power provides interface to power managemnt peripheral.
package power

import (
	"bits"
	"mmio"
	"unsafe"

	"nrf5/hal/internal/mmap"
	"nrf5/hal/te"
)

type Periph struct {
	te.Regs

	resetreas mmio.U32     // 0x400
	_         [9]mmio.U32  //
	ramstatus mmio.U32     // 0x428
	_         [53]mmio.U32 //
	systemoff mmio.U32     // 0x500
	_         [3]mmio.U32  //
	pofcon    mmio.U32     // 0x510
	_         [2]mmio.U32  //
	gpregret  [2]mmio.U32  // 0x51C
	ramon     mmio.U32     // 0x524
	_         [7]mmio.U32  //
	reset     mmio.U32     // 0x544
	_         [3]mmio.U32  //
	ramonb    mmio.U32     // 0x554
	_         [8]mmio.U32  //
	dcdcen    mmio.U32     // 0x578
	_         [225]mmio.U32
	ram       [8]struct{ power, powerset, powerclr mmio.U32 }
}

//emgo:const
var POWER = (*Periph)(unsafe.Pointer(mmap.APB_BASE))

type Task byte

const (
	CONSTLAT Task = 0x78 // Enable constant latency mode.
	LOWPWR   Task = 0x7C // Enable low power mode (variable latency).
)

type Event byte

const (
	POFWARN    Event = 0x08 // Power failure warning.
	SLEEPENTER Event = 0x14 // CPU entered WFI/WFE sleep (nRF52).
	SLEEPEXIT  Event = 0x18 // CPU exited WFI/WFE sleep (nRF52).
)

func (p *Periph) Task(t Task) *te.Task    { return p.Regs.Task(int(t)) }
func (p *Periph) Event(e Event) *te.Event { return p.Regs.Event(int(e)) }

// ResetReas is a bitfield that describes reset reason.
type ResetReas uint32

const (
	RESETPIN ResetReas = 1 << 0  // Reset from pin-reset.
	DOG      ResetReas = 1 << 1  // Reset from watchdog.
	SREQ     ResetReas = 1 << 2  // Reset from AIRCR.SYSRESETREQ.
	LOCKUP   ResetReas = 1 << 3  // Reset from CPU lock-up.
	OFF      ResetReas = 1 << 16 // Wake up from OFF mode by GPIO DETECT.
	LPCOMP   ResetReas = 1 << 17 // Wake up from OFF mode by LPCOMP ANADETECT.
	DIF      ResetReas = 1 << 18 // Wake up from OFF mode by debug interface.
	NFC      ResetReas = 1 << 19 // Wake up from OFF mode by NFC.
)

// LoadRESETREAS returns reset reason bits.
func (p *Periph) LoadRESETREAS() ResetReas {
	return ResetReas(p.resetreas.Load())
}

// ClearRESETREAS clears reset reason bits specified by mask.
func (p *Periph) ClearRESETREAS(mask ResetReas) {
	p.resetreas.Store(uint32(mask))
}

// RAMBlocks is a bitfield that describes RAM blocks.
type RAMBlocks byte

const (
	RAMBLOCK0 RAMBlocks = 1 << 0 // RAM block 0.
	RAMBLOCK1 RAMBlocks = 1 << 1 // RAM block 1.
	RAMBLOCK2 RAMBlocks = 1 << 2 // RAM block 2.
	RAMBLOCK3 RAMBlocks = 1 << 3 // RAM block 3.
)

// LoadRAMSTATUS returns bitfield that lists RAM blocks that are powered up.
func (p *Periph) LoadRAMSTATUS() RAMBlocks {
	return RAMBlocks(p.ramstatus.Load())
}

// SetSYSTEMOFF sets system into OFF state.
func (p *Periph) SetSYSTEMOFF() {
	p.systemoff.Store(1)
}

// POFCon is power failure comparator configuration.
type POFCon byte

const (
	POF       POFCon = 1 << 0  // Set if power failure comparoator is enabled.
	THRESHOLD POFCon = 15 << 1 // Power failure comparator threshold mask.

	// Power failure comparator thresholds.

	V2_1 POFCon = 0 << 1 // Threshold: 2.1 V (nrF51).
	V2_3 POFCon = 1 << 1 // Threshold: 2.3 V (nrF51).
	V2_5 POFCon = 2 << 1 // Threshold: 2.5 V (nrF51).
	V2_7 POFCon = 3 << 1 // Threshold: 2.5 V (nrF51).

	V17 POFCon = 4 << 1  // Threshold: 1.7 V (nRF52).
	V18 POFCon = 5 << 1  // Threshold: 1.8 V (nRF52).
	V19 POFCon = 6 << 1  // Threshold: 1.9 V (nRF52).
	V20 POFCon = 7 << 1  // Threshold: 2.0 V (nRF52).
	V21 POFCon = 8 << 1  // Threshold: 2.1 V (nRF52).
	V22 POFCon = 9 << 1  // Threshold: 2.2 V (nRF52).
	V23 POFCon = 10 << 1 // Threshold: 2.3 V (nRF52).
	V24 POFCon = 11 << 1 // Threshold: 2.4 V (nRF52).
	V25 POFCon = 12 << 1 // Threshold: 2.5 V (nRF52).
	V26 POFCon = 13 << 1 // Threshold: 2.6 V (nRF52).
	V27 POFCon = 14 << 1 // Threshold: 2.7 V (nRF52).
	V28 POFCon = 15 << 1 // Threshold: 2.8 V (nRF52).
)

// LoadPOFCON returns power failure comparator configuration.
func (p *Periph) LoadPOFCON() POFCon {
	return POFCon(p.pofcon.Load())
}

// StorePOFCON sets power failure comparator configuration.
func (p *Periph) StorePOFCON(pofcon POFCon) {
	p.pofcon.Store(uint32(pofcon))
}

// GPREGRET returns pointer to n-th general purpose retention register. nRF51
// supports one, nRF52 supports two. Only lowest 8 bits can be used.
func (p *Periph) GPREGRET(n int) *mmio.U32 {
	return &p.gpregret[n]
}

// LoadRAMON returns configuration of four RAM blocks. On lists RAM blocks
// that are kept on in system ON mode, retain lists RAM blocks that should be
// retained when RAM block is off.
func (p *Periph) LoadRAMON() (on, retain RAMBlocks) {
	a := p.ramon.Load()
	b := p.ramonb.Load()
	return RAMBlocks(a | b<<2), RAMBlocks(a>>16 | b>>14)
}

// StoreRAMON sets configuration of four RAM blocks. On lists RAM blocks that
// should be kept on in system ON mode, retain lists RAM blocks that should be
// retained in system off mode.
func (p *Periph) StoreRAMON(on, retain RAMBlocks) {
	p.ramon.Store(uint32(on&3) | uint32(retain&3)<<16)
	p.ramonb.Store(uint32(on&12)>>2 | uint32(retain&12)<<14)
}

// LoadRESET reports wheter pin reset is enabled in debug mode (nRF51).
func (p *Periph) LoadRESET() bool {
	return p.reset.Load()&1 != 0
}

// StoreRESET enables/disables pin reset in debug mode (nRF51).
func (p *Periph) StoreRESET(pinreset bool) {
	p.reset.Store(uint32(bits.One(pinreset)))
}

// LoadDCDCEN reports wheter the DC/DC converter is enabled.
func (p *Periph) LoadDCDCEN() bool {
	return p.dcdcen.Load()&1 != 0
}

// StoreDCDCEN enables/disables DC/DC converter.
func (p *Periph) StoreDCDCEN(en bool) {
	p.dcdcen.Store(uint32(bits.One(en)))
}

// RAMPower describes power configuration for two sections of RAM block (nRF52).
type RAMPower uint32

const (
	S0POWER     RAMPower = 1 << 0  // Keep RAM section S0 on in system on mode.
	S1POWER     RAMPower = 1 << 1  // Keep RAM section S1 on in system on mode.
	S0RETENTION RAMPower = 1 << 16 // Keep retention of S0 when RAM is off.
	S1RETENTION RAMPower = 1 << 17 // Keep retention of S1 when RAM is off.
)

// LoadRAMPOWER returns power configuration of RAM block n (nRF52).
func (p *Periph) LoadRAMPOWER(n int) RAMPower {
	return RAMPower(p.ram[n].power.Load())
}

// LoadRAMPOWER power configuration of RAM block n (nRF52).
func (p *Periph) StoreRAMPOWER(n int, val RAMPower) {
	p.ram[n].power.Store(uint32(val))
}

// SetRAMPOWER sets on power configuration of RAM block n according to mask
// (nRF52).
func (p *Periph) SetRAMPOWER(n int, mask RAMPower) {
	p.ram[n].powerset.Store(uint32(mask))
}

// ClearRAMPOWER sets off power configuration of RAM block n according to mask
// (nRF52).
func (p *Periph) ClearRAMPOWER(n int, mask RAMPower) {
	p.ram[n].powerclr.Store(uint32(mask))
}
<commit_msg>nrf5/hal/power: Fix Task and Event constants.<commit_after>// Package power provides interface to power managemnt peripheral.
package power

import (
	"bits"
	"mmio"
	"unsafe"

	"nrf5/hal/internal/mmap"
	"nrf5/hal/te"
)

type Periph struct {
	te.Regs

	resetreas mmio.U32     // 0x400
	_         [9]mmio.U32  //
	ramstatus mmio.U32     // 0x428
	_         [53]mmio.U32 //
	systemoff mmio.U32     // 0x500
	_         [3]mmio.U32  //
	pofcon    mmio.U32     // 0x510
	_         [2]mmio.U32  //
	gpregret  [2]mmio.U32  // 0x51C
	ramon     mmio.U32     // 0x524
	_         [7]mmio.U32  //
	reset     mmio.U32     // 0x544
	_         [3]mmio.U32  //
	ramonb    mmio.U32     // 0x554
	_         [8]mmio.U32  //
	dcdcen    mmio.U32     // 0x578
	_         [225]mmio.U32
	ram       [8]struct{ power, powerset, powerclr mmio.U32 }
}

//emgo:const
var POWER = (*Periph)(unsafe.Pointer(mmap.APB_BASE))

type Task byte

const (
	CONSTLAT Task = 30 // Enable constant latency mode.
	LOWPWR   Task = 31 // Enable low power mode (variable latency).
)

type Event byte

const (
	POFWARN    Event = 2 // Power failure warning.
	SLEEPENTER Event = 5 // CPU entered WFI/WFE sleep (nRF52).
	SLEEPEXIT  Event = 6 // CPU exited WFI/WFE sleep (nRF52).
)

func (p *Periph) Task(t Task) *te.Task    { return p.Regs.Task(int(t)) }
func (p *Periph) Event(e Event) *te.Event { return p.Regs.Event(int(e)) }

// ResetReas is a bitfield that describes reset reason.
type ResetReas uint32

const (
	RESETPIN ResetReas = 1 << 0  // Reset from pin-reset.
	DOG      ResetReas = 1 << 1  // Reset from watchdog.
	SREQ     ResetReas = 1 << 2  // Reset from AIRCR.SYSRESETREQ.
	LOCKUP   ResetReas = 1 << 3  // Reset from CPU lock-up.
	OFF      ResetReas = 1 << 16 // Wake up from OFF mode by GPIO DETECT.
	LPCOMP   ResetReas = 1 << 17 // Wake up from OFF mode by LPCOMP ANADETECT.
	DIF      ResetReas = 1 << 18 // Wake up from OFF mode by debug interface.
	NFC      ResetReas = 1 << 19 // Wake up from OFF mode by NFC.
)

// LoadRESETREAS returns reset reason bits.
func (p *Periph) LoadRESETREAS() ResetReas {
	return ResetReas(p.resetreas.Load())
}

// ClearRESETREAS clears reset reason bits specified by mask.
func (p *Periph) ClearRESETREAS(mask ResetReas) {
	p.resetreas.Store(uint32(mask))
}

// RAMBlocks is a bitfield that describes RAM blocks.
type RAMBlocks byte

const (
	RAMBLOCK0 RAMBlocks = 1 << 0 // RAM block 0.
	RAMBLOCK1 RAMBlocks = 1 << 1 // RAM block 1.
	RAMBLOCK2 RAMBlocks = 1 << 2 // RAM block 2.
	RAMBLOCK3 RAMBlocks = 1 << 3 // RAM block 3.
)

// LoadRAMSTATUS returns bitfield that lists RAM blocks that are powered up.
func (p *Periph) LoadRAMSTATUS() RAMBlocks {
	return RAMBlocks(p.ramstatus.Load())
}

// SetSYSTEMOFF sets system into OFF state.
func (p *Periph) SetSYSTEMOFF() {
	p.systemoff.Store(1)
}

// POFCon is power failure comparator configuration.
type POFCon byte

const (
	POF       POFCon = 1 << 0  // Set if power failure comparoator is enabled.
	THRESHOLD POFCon = 15 << 1 // Power failure comparator threshold mask.

	// Power failure comparator thresholds.

	V2_1 POFCon = 0 << 1 // Threshold: 2.1 V (nrF51).
	V2_3 POFCon = 1 << 1 // Threshold: 2.3 V (nrF51).
	V2_5 POFCon = 2 << 1 // Threshold: 2.5 V (nrF51).
	V2_7 POFCon = 3 << 1 // Threshold: 2.5 V (nrF51).

	V17 POFCon = 4 << 1  // Threshold: 1.7 V (nRF52).
	V18 POFCon = 5 << 1  // Threshold: 1.8 V (nRF52).
	V19 POFCon = 6 << 1  // Threshold: 1.9 V (nRF52).
	V20 POFCon = 7 << 1  // Threshold: 2.0 V (nRF52).
	V21 POFCon = 8 << 1  // Threshold: 2.1 V (nRF52).
	V22 POFCon = 9 << 1  // Threshold: 2.2 V (nRF52).
	V23 POFCon = 10 << 1 // Threshold: 2.3 V (nRF52).
	V24 POFCon = 11 << 1 // Threshold: 2.4 V (nRF52).
	V25 POFCon = 12 << 1 // Threshold: 2.5 V (nRF52).
	V26 POFCon = 13 << 1 // Threshold: 2.6 V (nRF52).
	V27 POFCon = 14 << 1 // Threshold: 2.7 V (nRF52).
	V28 POFCon = 15 << 1 // Threshold: 2.8 V (nRF52).
)

// LoadPOFCON returns power failure comparator configuration.
func (p *Periph) LoadPOFCON() POFCon {
	return POFCon(p.pofcon.Load())
}

// StorePOFCON sets power failure comparator configuration.
func (p *Periph) StorePOFCON(pofcon POFCon) {
	p.pofcon.Store(uint32(pofcon))
}

// GPREGRET returns pointer to n-th general purpose retention register. nRF51
// supports one, nRF52 supports two. Only lowest 8 bits can be used.
func (p *Periph) GPREGRET(n int) *mmio.U32 {
	return &p.gpregret[n]
}

// LoadRAMON returns configuration of four RAM blocks. On lists RAM blocks
// that are kept on in system ON mode, retain lists RAM blocks that should be
// retained when RAM block is off.
func (p *Periph) LoadRAMON() (on, retain RAMBlocks) {
	a := p.ramon.Load()
	b := p.ramonb.Load()
	return RAMBlocks(a | b<<2), RAMBlocks(a>>16 | b>>14)
}

// StoreRAMON sets configuration of four RAM blocks. On lists RAM blocks that
// should be kept on in system ON mode, retain lists RAM blocks that should be
// retained in system off mode.
func (p *Periph) StoreRAMON(on, retain RAMBlocks) {
	p.ramon.Store(uint32(on&3) | uint32(retain&3)<<16)
	p.ramonb.Store(uint32(on&12)>>2 | uint32(retain&12)<<14)
}

// LoadRESET reports wheter pin reset is enabled in debug mode (nRF51).
func (p *Periph) LoadRESET() bool {
	return p.reset.Load()&1 != 0
}

// StoreRESET enables/disables pin reset in debug mode (nRF51).
func (p *Periph) StoreRESET(pinreset bool) {
	p.reset.Store(uint32(bits.One(pinreset)))
}

// LoadDCDCEN reports wheter the DC/DC converter is enabled.
func (p *Periph) LoadDCDCEN() bool {
	return p.dcdcen.Load()&1 != 0
}

// StoreDCDCEN enables/disables DC/DC converter.
func (p *Periph) StoreDCDCEN(en bool) {
	p.dcdcen.Store(uint32(bits.One(en)))
}

// RAMPower describes power configuration for two sections of RAM block (nRF52).
type RAMPower uint32

const (
	S0POWER     RAMPower = 1 << 0  // Keep RAM section S0 on in system on mode.
	S1POWER     RAMPower = 1 << 1  // Keep RAM section S1 on in system on mode.
	S0RETENTION RAMPower = 1 << 16 // Keep retention of S0 when RAM is off.
	S1RETENTION RAMPower = 1 << 17 // Keep retention of S1 when RAM is off.
)

// LoadRAMPOWER returns power configuration of RAM block n (nRF52).
func (p *Periph) LoadRAMPOWER(n int) RAMPower {
	return RAMPower(p.ram[n].power.Load())
}

// LoadRAMPOWER power configuration of RAM block n (nRF52).
func (p *Periph) StoreRAMPOWER(n int, val RAMPower) {
	p.ram[n].power.Store(uint32(val))
}

// SetRAMPOWER sets on power configuration of RAM block n according to mask
// (nRF52).
func (p *Periph) SetRAMPOWER(n int, mask RAMPower) {
	p.ram[n].powerset.Store(uint32(mask))
}

// ClearRAMPOWER sets off power configuration of RAM block n according to mask
// (nRF52).
func (p *Periph) ClearRAMPOWER(n int, mask RAMPower) {
	p.ram[n].powerclr.Store(uint32(mask))
}
<|endoftext|>


<commit_before>// Package postgres implements the Driver interface.
package postgres

import (
	"database/sql"
	"fmt"
	"strconv"
	"strings"

	"github.com/db-journey/migrate/direction"
	"github.com/db-journey/migrate/driver"
	"github.com/db-journey/migrate/file"
	"github.com/lib/pq"
)

var _ driver.Driver = (*Driver)(nil)

// Driver is the postgres driver for journey.
type Driver struct {
	db *sql.DB
}

const tableName = "schema_migrations"
const txDisabledOption = "disable_ddl_transaction"

// Initialize opens and verifies the database handle.
func (driver *Driver) Initialize(url string) error {
	db, err := sql.Open("postgres", url)
	if err != nil {
		return err
	}
	if err := db.Ping(); err != nil {
		return err
	}
	driver.db = db

	return driver.ensureVersionTableExists()
}

// SetDB replaces the current database handle.
func (driver *Driver) SetDB(db *sql.DB) {
	driver.db = db
}

// Close closes the database handle.
func (driver *Driver) Close() error {
	return driver.db.Close()
}

func (driver *Driver) ensureVersionTableExists() error {
	// avoid DDL statements if possible for BDR (see #23)
	var c int
	if err := driver.db.QueryRow("SELECT count(*) FROM information_schema.tables WHERE table_name = $1", tableName).Scan(&c); err != nil {
		return err
	}

	if c <= 0 {
		_, err := driver.db.Exec("CREATE TABLE IF NOT EXISTS " + tableName + " (version bigint not null primary key)")
		return err
	}

	// table schema_migrations already exists, check if the schema is correct, ie: version is a bigint

	var dataType string
	if err := driver.db.QueryRow("SELECT data_type FROM information_schema.columns where table_name = $1 and column_name = 'version'", tableName).Scan(&dataType); err != nil {
		return err
	}

	if dataType == "bigint" {
		return nil
	}

	_, err := driver.db.Exec("ALTER TABLE " + tableName + " ALTER COLUMN version TYPE bigint USING version::bigint")
	return err
}

// FilenameExtension returns "sql".
func (driver *Driver) FilenameExtension() string {
	return "sql"
}

// Migrate performs the migration of any one file.
func (driver *Driver) Migrate(f file.File, pipe chan interface{}) {
	defer close(pipe)
	pipe <- f

	tx, err := driver.db.Begin()
	if err != nil {
		pipe <- err
		return
	}

	if f.Direction == direction.Up {
		if _, err = tx.Exec("INSERT INTO "+tableName+" (version) VALUES ($1)", f.Version); err != nil {
			pipe <- err
			if err = tx.Rollback(); err != nil {
				pipe <- err
			}
			return
		}
	} else if f.Direction == direction.Down {
		if _, err = tx.Exec("DELETE FROM "+tableName+" WHERE version=$1", f.Version); err != nil {
			pipe <- err
			if err = tx.Rollback(); err != nil {
				pipe <- err
			}
			return
		}
	}

	if err = f.ReadContent(); err != nil {
		pipe <- err
		return
	}

	if txDisabled(fileOptions(f.Content)) {
		_, err = driver.db.Exec(string(f.Content))
	} else {
		_, err = tx.Exec(string(f.Content))
	}

	if err != nil {
		pqErr := err.(*pq.Error)
		offset, err := strconv.Atoi(pqErr.Position)
		if err == nil && offset >= 0 {
			lineNo, columnNo := file.LineColumnFromOffset(f.Content, offset-1)
			errorPart := file.LinesBeforeAndAfter(f.Content, lineNo, 5, 5, true)
			pipe <- fmt.Errorf("%s %v: %s in line %v, column %v:\n\n%s", pqErr.Severity, pqErr.Code, pqErr.Message, lineNo, columnNo, string(errorPart))
		} else {
			pipe <- fmt.Errorf("%s %v: %s", pqErr.Severity, pqErr.Code, pqErr.Message)
		}

		if err := tx.Rollback(); err != nil {
			pipe <- err
		}
		return
	}

	if err := tx.Commit(); err != nil {
		pipe <- err
		return
	}
}

// Version returns the current migration version.
func (driver *Driver) Version() (file.Version, error) {
	var version file.Version
	err := driver.db.QueryRow("SELECT version FROM " + tableName + " ORDER BY version DESC LIMIT 1").Scan(&version)
	if err == sql.ErrNoRows {
		return version, nil
	}

	return version, err
}

// Versions returns the list of applied migrations.
func (driver *Driver) Versions() (file.Versions, error) {
	rows, err := driver.db.Query("SELECT version FROM " + tableName + " ORDER BY version DESC")
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	versions := file.Versions{}
	for rows.Next() {
		var version file.Version
		if err = rows.Scan(&version); err != nil {
			return nil, err
		}
		versions = append(versions, version)
	}

	if err = rows.Err(); err != nil {
		return nil, err
	}

	return versions, err
}

// fileOptions returns the list of options extracted from the first line of the file content.
// Format: "-- <option1> <option2> <...>"
func fileOptions(content []byte) []string {
	firstLine := strings.SplitN(string(content), "\n", 2)[0]
	if !strings.HasPrefix(firstLine, "-- ") {
		return []string{}
	}
	opts := strings.TrimPrefix(firstLine, "-- ")
	return strings.Split(opts, " ")
}

func txDisabled(opts []string) bool {
	for _, v := range opts {
		if v == txDisabledOption {
			return true
		}
	}
	return false
}

func init() {
	driver.RegisterDriver("postgres", &Driver{})
}
<commit_msg>Add Execute func to driver<commit_after>// Package postgres implements the Driver interface.
package postgres

import (
	"database/sql"
	"fmt"
	"strconv"
	"strings"

	"github.com/db-journey/migrate/direction"
	"github.com/db-journey/migrate/driver"
	"github.com/db-journey/migrate/file"
	"github.com/lib/pq"
)

var _ driver.Driver = (*Driver)(nil)

// Driver is the postgres driver for journey.
type Driver struct {
	db *sql.DB
}

const tableName = "schema_migrations"
const txDisabledOption = "disable_ddl_transaction"

// make sure our driver still implements the driver.Driver interface
var _ driver.Driver = (*Driver)(nil)

// Initialize opens and verifies the database handle.
func (driver *Driver) Initialize(url string) error {
	db, err := sql.Open("postgres", url)
	if err != nil {
		return err
	}
	if err := db.Ping(); err != nil {
		return err
	}
	driver.db = db

	return driver.ensureVersionTableExists()
}

// SetDB replaces the current database handle.
func (driver *Driver) SetDB(db *sql.DB) {
	driver.db = db
}

// Close closes the database handle.
func (driver *Driver) Close() error {
	return driver.db.Close()
}

func (driver *Driver) ensureVersionTableExists() error {
	// avoid DDL statements if possible for BDR (see #23)
	var c int
	if err := driver.db.QueryRow("SELECT count(*) FROM information_schema.tables WHERE table_name = $1", tableName).Scan(&c); err != nil {
		return err
	}

	if c <= 0 {
		_, err := driver.db.Exec("CREATE TABLE IF NOT EXISTS " + tableName + " (version bigint not null primary key)")
		return err
	}

	// table schema_migrations already exists, check if the schema is correct, ie: version is a bigint

	var dataType string
	if err := driver.db.QueryRow("SELECT data_type FROM information_schema.columns where table_name = $1 and column_name = 'version'", tableName).Scan(&dataType); err != nil {
		return err
	}

	if dataType == "bigint" {
		return nil
	}

	_, err := driver.db.Exec("ALTER TABLE " + tableName + " ALTER COLUMN version TYPE bigint USING version::bigint")
	return err
}

// FilenameExtension returns "sql".
func (driver *Driver) FilenameExtension() string {
	return "sql"
}

// Migrate performs the migration of any one file.
func (driver *Driver) Migrate(f file.File, pipe chan interface{}) {
	defer close(pipe)
	pipe <- f

	tx, err := driver.db.Begin()
	if err != nil {
		pipe <- err
		return
	}

	if f.Direction == direction.Up {
		if _, err = tx.Exec("INSERT INTO "+tableName+" (version) VALUES ($1)", f.Version); err != nil {
			pipe <- err
			if err = tx.Rollback(); err != nil {
				pipe <- err
			}
			return
		}
	} else if f.Direction == direction.Down {
		if _, err = tx.Exec("DELETE FROM "+tableName+" WHERE version=$1", f.Version); err != nil {
			pipe <- err
			if err = tx.Rollback(); err != nil {
				pipe <- err
			}
			return
		}
	}

	if err = f.ReadContent(); err != nil {
		pipe <- err
		return
	}

	if txDisabled(fileOptions(f.Content)) {
		_, err = driver.db.Exec(string(f.Content))
	} else {
		_, err = tx.Exec(string(f.Content))
	}

	if err != nil {
		pqErr := err.(*pq.Error)
		offset, err := strconv.Atoi(pqErr.Position)
		if err == nil && offset >= 0 {
			lineNo, columnNo := file.LineColumnFromOffset(f.Content, offset-1)
			errorPart := file.LinesBeforeAndAfter(f.Content, lineNo, 5, 5, true)
			pipe <- fmt.Errorf("%s %v: %s in line %v, column %v:\n\n%s", pqErr.Severity, pqErr.Code, pqErr.Message, lineNo, columnNo, string(errorPart))
		} else {
			pipe <- fmt.Errorf("%s %v: %s", pqErr.Severity, pqErr.Code, pqErr.Message)
		}

		if err := tx.Rollback(); err != nil {
			pipe <- err
		}
		return
	}

	if err := tx.Commit(); err != nil {
		pipe <- err
		return
	}
}

// Version returns the current migration version.
func (driver *Driver) Version() (file.Version, error) {
	var version file.Version
	err := driver.db.QueryRow("SELECT version FROM " + tableName + " ORDER BY version DESC LIMIT 1").Scan(&version)
	if err == sql.ErrNoRows {
		return version, nil
	}

	return version, err
}

// Versions returns the list of applied migrations.
func (driver *Driver) Versions() (file.Versions, error) {
	rows, err := driver.db.Query("SELECT version FROM " + tableName + " ORDER BY version DESC")
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	versions := file.Versions{}
	for rows.Next() {
		var version file.Version
		if err = rows.Scan(&version); err != nil {
			return nil, err
		}
		versions = append(versions, version)
	}

	if err = rows.Err(); err != nil {
		return nil, err
	}

	return versions, err
}

// Execute a SQL statement
func (driver *Driver) Execute(statement string) error {
	_, err := driver.db.Exec(statement)
	return err
}

// fileOptions returns the list of options extracted from the first line of the file content.
// Format: "-- <option1> <option2> <...>"
func fileOptions(content []byte) []string {
	firstLine := strings.SplitN(string(content), "\n", 2)[0]
	if !strings.HasPrefix(firstLine, "-- ") {
		return []string{}
	}
	opts := strings.TrimPrefix(firstLine, "-- ")
	return strings.Split(opts, " ")
}

func txDisabled(opts []string) bool {
	for _, v := range opts {
		if v == txDisabledOption {
			return true
		}
	}
	return false
}

func init() {
	// According to the PostgreSQL documentation (section 32.1.1.2), postgres
	// library supports two URI schemes: postgresql:// and postgres://
	// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
	drv := Driver{}
	driver.RegisterDriver("postgres", &drv)
	driver.RegisterDriver("postgresql", &drv)
}
<|endoftext|>


<commit_before>package eventmaster

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"strings"

	sq "github.com/Masterminds/squirrel"
	"github.com/lib/pq"
	_ "github.com/lib/pq"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
	eventmaster "github.com/wish/eventmaster/proto"
)

// implements datastore
type PostgresStore struct {
	db *sql.DB
}

type PostgresConfig struct {
	Addr        string `json:"addr"`
	Port        int    `json:"port"`
	Database    string `json:"database"`
	ServiceName string `json:"service_name"`
	Username    string `json:"username"`
	Password    string `json:"password"`
}

func NewPostgresStore(c PostgresConfig) (*PostgresStore, error) {
	var host string
	if c.ServiceName != "" {
		host = c.ServiceName + ".service.consul"
	} else {
		host = c.Addr
	}
	log.Infof("Connecting to postgres: %v", host)
	psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
		"password=%s dbname=%s sslmode=disable",
		host, c.Port, c.Username, c.Password, c.Database)
	db, err := sql.Open("postgres", psqlInfo)
	if err != nil {
		return nil, errors.Wrap(err, "Error creating postgres session")
	}
	err = db.Ping()
	if err != nil {
		return nil, errors.Wrap(err, "Error creating postgres session")
	}
	log.Infof("Successfully connected to postgres %s", host)

	return &PostgresStore{
		db: db,
	}, nil
}

func (p *PostgresStore) AddEvent(e *Event) error {
	data := "{}"
	if e.Data != nil {
		dataBytes, err := json.Marshal(e.Data)
		if err != nil {
			return err
		}
		data = string(dataBytes)
	}

	tx, err := p.db.Begin()
	if err != nil {
		return err
	}

	_, err = tx.Exec("INSERT INTO event "+
		"(event_id, parent_event_id, dc_id, topic_id, host, target_host_set, \"user\", event_time, tag_set, received_time)"+
		" VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
		e.EventID, e.ParentEventID, e.DCID, e.TopicID, strings.ToLower(e.Host), pq.Array(e.TargetHosts), strings.ToLower(e.User), e.EventTime, pq.Array(e.Tags), e.ReceivedTime)

	if err != nil {
		tx.Rollback()
		return err
	}

	_, err = tx.Exec("INSERT INTO event_metadata (event_id, data_json) VALUES ($1, $2)", e.EventID, data)
	if err != nil {
		tx.Rollback()
		return err
	}

	return tx.Commit()
}

func (p *PostgresStore) Find(q *eventmaster.Query, topicIDs []string, dcIDs []string) (Events, error) {
	// use sql builder for better readability
	psql := sq.StatementBuilder.PlaceholderFormat(sq.Dollar)
	columns := []string{"event_id", "parent_event_id", "dc_id", "topic_id", "host", "target_host_set", "user", "event_time", "tag_set", "received_time"}
	table := "event"
	base_query := psql.Select(columns...).From(table)

	wheres := sq.And{}
	// add time constarint
	wheres = append(wheres, sq.GtOrEq{"event_time": q.StartEventTime})
	wheres = append(wheres, sq.LtOrEq{"event_time": q.EndEventTime})
	// TODO: add other constraints

	query := base_query.Where(wheres)

	var events []*Event

	var event_id string
	var parent_event_id string
	var dc_id string
	var topic_id string
	var host string
	var target_host_set []string
	var user string
	var event_time int64
	var tag_set []string
	var received_time int64

	rows, err := query.RunWith(p.db).Query()
	for rows.Next() {
		err = rows.Scan(&event_id, &parent_event_id, &dc_id, &topic_id, &host, pq.Array(&target_host_set), &user, &event_time, pq.Array(&tag_set), &received_time)
		if err != nil {
			return nil, err
		}
		events = append(events, &Event{
			EventID:       event_id,
			ParentEventID: parent_event_id,
			DCID:          dc_id,
			TopicID:       topic_id,
			Host:          host,
			TargetHosts:   target_host_set,
			User:          user,
			EventTime:     event_time,
			Tags:          tag_set,
			ReceivedTime:  received_time,
		})
	}

	return events, nil
}

func (p *PostgresStore) FindByID(string, bool) (*Event, error) {
	// TODO: implement this function
	return nil, nil
}

func (p *PostgresStore) FindIDs(*eventmaster.TimeQuery, HandleEvent) error {
	// TODO: implement this function
	return nil
}

func (p *PostgresStore) GetTopics() ([]Topic, error) {
	rows, err := p.db.Query("SELECT topic_id, topic_name, data_schema FROM event_topic")
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var topics []Topic
	var id string
	var name, schema string

	for rows.Next() {
		err = rows.Scan(&id, &name, &schema)
		if err != nil {
			return nil, err
		}
		var s map[string]interface{}
		err := json.Unmarshal([]byte(schema), &s)
		if err != nil {
			return nil, err
		}
		topics = append(topics, Topic{
			ID:     id,
			Name:   name,
			Schema: s,
		})
	}

	return topics, nil
}

func (p *PostgresStore) AddTopic(t RawTopic) error {
	_, err := p.db.Exec("INSERT INTO event_topic (topic_id, topic_name, data_schema) VALUES ($1, $2, $3)", t.ID, t.Name, t.Schema)
	return err
}

func (p *PostgresStore) UpdateTopic(t RawTopic) error {
	_, err := p.db.Exec("UPDATE event_topic SET topic_name=$1, data_schema=$2 WHERE topic_id=$3", t.Name, t.Schema, t.ID)
	return err
}

func (p *PostgresStore) DeleteTopic(id string) error {
	_, err := p.db.Exec("DELETE FROM event_topic WHERE topic_id=$1", id)
	return err
}

func (p *PostgresStore) GetDCs() ([]DC, error) {
	rows, err := p.db.Query("SELECT dc_id, dc from event_dc")
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var dcs []DC

	var dc_id, dc string
	for rows.Next() {
		err = rows.Scan(&dc_id, &dc)
		if err != nil {
			return nil, err
		}
		dcs = append(dcs, DC{
			ID:   dc_id,
			Name: dc,
		})
	}
	return dcs, nil
}

func (p *PostgresStore) AddDC(dc DC) error {
	stmt, err := p.db.Prepare("INSERT INTO event_dc (dc_id, dc) VALUES ($1, $2)")
	if err != nil {
		return errors.Wrap(err, "Failed preparing insert DC statement")
	}
	_, err = stmt.Exec(dc.ID, dc.Name)
	if err != nil {
		return err
	}

	return nil
}

func (p *PostgresStore) UpdateDC(id string, newName string) error {
	_, err := p.db.Exec("UPDATE event_dc SET dc=$1 WHERE dc_id=$2", newName, id)
	return err
}

func (p *PostgresStore) CloseSession() {
	p.db.Close()
}
<commit_msg>Implement Find<commit_after>package eventmaster

import (
	"database/sql"
	"encoding/json"
	"fmt"

	sq "github.com/Masterminds/squirrel"
	"github.com/lib/pq"
	_ "github.com/lib/pq"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
	eventmaster "github.com/wish/eventmaster/proto"
)

// implements datastore
type PostgresStore struct {
	db *sql.DB
}

type PostgresConfig struct {
	Addr        string `json:"addr"`
	Port        int    `json:"port"`
	Database    string `json:"database"`
	ServiceName string `json:"service_name"`
	Username    string `json:"username"`
	Password    string `json:"password"`
}

func NewPostgresStore(c PostgresConfig) (*PostgresStore, error) {
	var host string
	if c.ServiceName != "" {
		host = c.ServiceName + ".service.consul"
	} else {
		host = c.Addr
	}
	log.Infof("Connecting to postgres: %v", host)
	psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
		"password=%s dbname=%s sslmode=disable",
		host, c.Port, c.Username, c.Password, c.Database)
	db, err := sql.Open("postgres", psqlInfo)
	if err != nil {
		return nil, errors.Wrap(err, "Error creating postgres session")
	}
	err = db.Ping()
	if err != nil {
		return nil, errors.Wrap(err, "Error creating postgres session")
	}
	log.Infof("Successfully connected to postgres %s", host)

	return &PostgresStore{
		db: db,
	}, nil
}

func (p *PostgresStore) AddEvent(e *Event) error {
	data := "{}"
	if e.Data != nil {
		dataBytes, err := json.Marshal(e.Data)
		if err != nil {
			return err
		}
		data = string(dataBytes)
	}

	tx, err := p.db.Begin()
	if err != nil {
		return err
	}

	_, err = tx.Exec("INSERT INTO event "+
		"(event_id, parent_event_id, dc_id, topic_id, host, target_host_set, \"user\", event_time, tag_set, received_time)"+
		" VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
		e.EventID, e.ParentEventID, e.DCID, e.TopicID, e.Host, pq.Array(e.TargetHosts), e.User, e.EventTime, pq.Array(e.Tags), e.ReceivedTime)

	if err != nil {
		tx.Rollback()
		return err
	}

	_, err = tx.Exec("INSERT INTO event_metadata (event_id, data_json) VALUES ($1, $2)", e.EventID, data)
	if err != nil {
		tx.Rollback()
		return err
	}

	return tx.Commit()
}

func getPlaceHolderArray(slice []string) string {
	expr := "ARRAY["
	first := true
	for i := 0; i < len(slice); i++ {
		if !first {
			expr += ", "
		}
		expr += "?"
		first = false
	}
	expr += "]"
	return expr
}

func stringsToIfaces(in []string) []interface{} {
	new := make([]interface{}, len(in))
	for i, v := range in {
		new[i] = v
	}
	return new
}

func (p *PostgresStore) Find(q *eventmaster.Query, topicIDs []string, dcIDs []string) (Events, error) {
	// use sql builder for better readability
	psql := sq.StatementBuilder.PlaceholderFormat(sq.Dollar)
	columns := []string{"event_id", "parent_event_id", "dc_id", "topic_id", "host", "target_host_set", "user", "event_time", "tag_set", "received_time"}
	table := "event"
	base_query := psql.Select(columns...).From(table)

	wheres := sq.And{}

	// add constarints
	wheres = append(wheres, sq.GtOrEq{"event_time": q.StartEventTime})
	wheres = append(wheres, sq.LtOrEq{"event_time": q.EndEventTime})

	if len(q.User) > 0 {
		wheres = append(wheres, sq.Eq{"user": q.User})
	}

	if len(q.ParentEventID) > 0 {
		wheres = append(wheres, sq.Eq{"parent_event_id": q.ParentEventID})
	}

	if len(q.Host) > 0 {
		wheres = append(wheres, sq.Eq{"host": q.Host})
	}

	if len(topicIDs) > 0 {
		wheres = append(wheres, sq.Eq{"topic_id": topicIDs})
	}

	if len(dcIDs) > 0 {
		wheres = append(wheres, sq.Eq{"dc_id": dcIDs})
	}

	if len(q.TargetHostSet) > 0 {
		// intersection of two arrays
		// squirrel doesn't support Postgres' array and repective methods. Building new SQL template manually
		// make identical slice of []interface{} type to pass to varadic functions
		clause := "target_host_set && " + getPlaceHolderArray((q.TargetHostSet))
		wheres = append(wheres, sq.Expr(clause, stringsToIfaces(q.TargetHostSet)...))
	}

	if len(q.TagSet) > 0 {
		var op string
		if q.TagAndOperator {
			// LHS array is the superset of RHS array
			op = ">&"
		} else {
			// LHS array has interestion with RHS array
			op = "&&"
		}
		clause := fmt.Sprintf("tag_set %s %s", op, getPlaceHolderArray(q.TagSet))
		wheres = append(wheres, sq.Expr(clause, stringsToIfaces(q.TagSet)...))
	}

	if len(q.ExcludeTagSet) > 0 {
		clause := fmt.Sprintf("NOT (tag_set && %s)", getPlaceHolderArray((q.ExcludeTagSet)))
		wheres = append(wheres, sq.Expr(clause, stringsToIfaces(q.ExcludeTagSet)...))
	}

	query := base_query.Where(wheres)

	var events []*Event

	var event_id string
	var parent_event_id string
	var dc_id string
	var topic_id string
	var host string
	var target_host_set []string
	var user string
	var event_time int64
	var tag_set []string
	var received_time int64

	rows, err := query.RunWith(p.db).Query()
	for rows.Next() {
		err = rows.Scan(&event_id, &parent_event_id, &dc_id, &topic_id, &host, pq.Array(&target_host_set), &user, &event_time, pq.Array(&tag_set), &received_time)
		if err != nil {
			return nil, err
		}
		events = append(events, &Event{
			EventID:       event_id,
			ParentEventID: parent_event_id,
			DCID:          dc_id,
			TopicID:       topic_id,
			Host:          host,
			TargetHosts:   target_host_set,
			User:          user,
			EventTime:     event_time,
			Tags:          tag_set,
			ReceivedTime:  received_time,
		})
	}

	return events, nil
}

func (p *PostgresStore) FindByID(string, bool) (*Event, error) {
	// TODO: implement this function
	return nil, nil
}

func (p *PostgresStore) FindIDs(*eventmaster.TimeQuery, HandleEvent) error {
	// TODO: implement this function
	return nil
}

func (p *PostgresStore) GetTopics() ([]Topic, error) {
	rows, err := p.db.Query("SELECT topic_id, topic_name, data_schema FROM event_topic")
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var topics []Topic
	var id string
	var name, schema string

	for rows.Next() {
		err = rows.Scan(&id, &name, &schema)
		if err != nil {
			return nil, err
		}
		var s map[string]interface{}
		err := json.Unmarshal([]byte(schema), &s)
		if err != nil {
			return nil, err
		}
		topics = append(topics, Topic{
			ID:     id,
			Name:   name,
			Schema: s,
		})
	}

	return topics, nil
}

func (p *PostgresStore) AddTopic(t RawTopic) error {
	_, err := p.db.Exec("INSERT INTO event_topic (topic_id, topic_name, data_schema) VALUES ($1, $2, $3)", t.ID, t.Name, t.Schema)
	return err
}

func (p *PostgresStore) UpdateTopic(t RawTopic) error {
	_, err := p.db.Exec("UPDATE event_topic SET topic_name=$1, data_schema=$2 WHERE topic_id=$3", t.Name, t.Schema, t.ID)
	return err
}

func (p *PostgresStore) DeleteTopic(id string) error {
	_, err := p.db.Exec("DELETE FROM event_topic WHERE topic_id=$1", id)
	return err
}

func (p *PostgresStore) GetDCs() ([]DC, error) {
	rows, err := p.db.Query("SELECT dc_id, dc from event_dc")
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var dcs []DC

	var dc_id, dc string
	for rows.Next() {
		err = rows.Scan(&dc_id, &dc)
		if err != nil {
			return nil, err
		}
		dcs = append(dcs, DC{
			ID:   dc_id,
			Name: dc,
		})
	}
	return dcs, nil
}

func (p *PostgresStore) AddDC(dc DC) error {
	stmt, err := p.db.Prepare("INSERT INTO event_dc (dc_id, dc) VALUES ($1, $2)")
	if err != nil {
		return errors.Wrap(err, "Failed preparing insert DC statement")
	}
	_, err = stmt.Exec(dc.ID, dc.Name)
	if err != nil {
		return err
	}

	return nil
}

func (p *PostgresStore) UpdateDC(id string, newName string) error {
	_, err := p.db.Exec("UPDATE event_dc SET dc=$1 WHERE dc_id=$2", newName, id)
	return err
}

func (p *PostgresStore) CloseSession() {
	p.db.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"

	// "database/sql"

	_ "github.com/lib/pq"

	"github.com/DimShadoWWW/power-pg/proxy"
	"github.com/parnurzeal/gorequest"
)

var (
	localHost     = flag.String("l", ":5432", "puerto local para escuchar")
	dbHostname    = flag.String("h", "localhost", "hostname del servidor PostgreSQL")
	dbPort        = flag.String("r", "5432", "puerto del servidor PostgreSQL")
	dbName        = flag.String("d", "dbname", "nombre de la base de datos")
	dbUsername    = flag.String("u", "username", "usuario para acceder al servidor PostgreSQL")
	dbPassword    = flag.String("p", "password", "password para acceder al servidor PostgreSQL")
	remoteService = flag.String("s", "", "http://localhost:8080/query")
	// messages      = []string{}
)

type msgStruct struct {
	Type    string
	Content string
}

var (
	db *sql.DB
)

func main() {
	flag.Parse()
	// dbinfo := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable",
	// 	*dbUsername, *dbPassword, *dbName)
	// db, err := sql.Open("postgres", dbinfo)

	msgs := make(chan string)
	msgCh := make(chan proxy.Pkg)
	msgOut := make(chan msgStruct)
	if *remoteService != "" {
		go func() {
			time.Sleep(time.Second * 3)
			inFile, _ := os.Open("canales_list.txt")
			defer inFile.Close()
			scanner := bufio.NewScanner(inFile)
			scanner.Split(bufio.ScanLines)

			for scanner.Scan() {
				time.Sleep(time.Second * 1)
				// messages = []string{}
				// fmt.Println(scanner.Text())
				// msgOut <- fmt.Sprintf("# %s\n", scanner.Text())
				msgOut <- msgStruct{Type: "C", Content: scanner.Text()}
				_, _, errs := gorequest.New().Get(fmt.Sprintf("%s%s", *remoteService, scanner.Text())).End()
				if errs != nil {
					log.Fatalf("log failed: %v", errs)
				}
			}
			log.Println("done")
			os.Exit(0)
		}()
	}

	go func() {
		f, err := os.OpenFile("/all.txt", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		for msg := range msgs {
			// fmt.Println(msg)
			_, err := f.WriteString(fmt.Sprintf("%s\n", msg))
			if err != nil {
				log.Fatalf("log failed: %v", err)
			}
		}
	}()

	go func() {
		f, err := os.OpenFile("/reports/report.md", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
		// c := 0
		spaces := regexp.MustCompile("[\t]+")
		multipleSpaces := regexp.MustCompile("    ")
		for {
			// select {
			// case msg1 := <-msgOut:
			msg := <-msgOut
			if msg.Type == "C" {
				// c = 0
				f.Close()
				f, err = os.OpenFile(fmt.Sprintf("/reports/report-%s.md", msg.Content), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				// c = 0
				if err != nil {
					panic(err)
				}
				_, err := f.WriteString(fmt.Sprintf("# %s\n", msg.Content))
				if err != nil {
					log.Fatalf("log failed: %v", err)
				}
			} else {
				// case msg2 := <-msgOut:
				// c = c + 1
				m := spaces.ReplaceAll([]byte(msg.Content), []byte{' '})
				m = multipleSpaces.ReplaceAll(m, []byte{' '})
				_, err := f.WriteString(fmt.Sprintf("\n```sql\n%s\n```\n", string(m)))
				if err != nil {
					log.Fatalf("log failed: %v", err)
				}
			}
		}
	}()

	go func() {
		temp := ""
		for msg := range msgCh {
			if msg.Type == 'P' {
				if strings.Contains(string(msg.Content), "$1") {
					var newMsg proxy.ReadBuf
					newMsg = msg.Content
					_ = newMsg.Int32()

					// The name of the destination portal (an empty string selects the unnamed portal).
					p := bytes.Index(newMsg, []byte{0})
					// remove first string
					msgs <- fmt.Sprintf("msg ---->%#v\n", newMsg)
					msgs <- fmt.Sprintf("msg ---->%s\n", string(newMsg))
					msgs <- fmt.Sprintf("first string ---->%#v\n", newMsg[:p+1])
					msgs <- fmt.Sprintf("first string ---->%s\n", string(newMsg[:p+1]))
					newMsg = newMsg[p+1:]
					fmt.Printf("0 newMsg   ----->%#v\n", newMsg)

					// The name of the source prepared statement (an empty string selects the unnamed prepared statement).
					p = bytes.Index(newMsg, []byte{0})
					// remove second string
					msgs <- fmt.Sprintf("second string: message ---->%#v\n", newMsg[:p])
					temp = string(newMsg[:p])
					msgs <- fmt.Sprintf("second string: message temp ---->%s\n", temp)

				} else {
					temp = ""
					selectIdx := strings.Index(string(msg.Content), string([]byte{83, 69, 76, 69, 67, 84, 32}))
					if selectIdx == -1 {
						selectIdx = 0
					}
					sepIdx := strings.Index(string(msg.Content), string([]byte{0, 0, 1, 0, 0}))
					if sepIdx == -1 || sepIdx+5 > len(msg.Content) {
						sepIdx := strings.Index(string(msg.Content), string([]byte{0, 1, 0, 0}))
						if sepIdx == -1 || sepIdx+4 > len(msg.Content) {
							sepIdx = len(msg.Content) - 4
						} else {
							sepIdx = len(msg.Content) - 1
						}
					}
					if sepIdx == -1 {
						sepIdx = len(msg.Content)
					}
					if selectIdx == -1 {
						selectIdx = 0
					}

					fmt.Printf("SEP index ----->%v\n", sepIdx)
					fmt.Printf("SEP len   ----->%v\n", len(msg.Content))
					fmt.Printf("SEP CONT  ----->%v\n", msg.Content)
					// messages = append(messages, string(bytes.Trim(msg.Content[selectIdx:sepIdx], "\x00")))
					msgOut <- msgStruct{Type: "M", Content: string(bytes.Trim(msg.Content[selectIdx:sepIdx], "\x00"))}
				}
			} else {
				if msg.Type == 'B' && temp != "" {
					var newMsg proxy.ReadBuf
					newMsg = msg.Content

					// The name of the destination portal (an empty string selects the unnamed portal).
					p := bytes.Index(newMsg, []byte{0})
					// remove first string
					msgs <- fmt.Sprintf("msg ---->%#v\n", newMsg)
					msgs <- fmt.Sprintf("first string ---->%#v\n", newMsg[:p+1])
					newMsg = newMsg[p+1:]
					fmt.Printf("0 newMsg   ----->%#v\n", newMsg)

					// The name of the source prepared statement (an empty string selects the unnamed prepared statement).
					p = bytes.Index(newMsg, []byte{0})
					// remove second string
					msgs <- fmt.Sprintf("second string ---->%#v\n", newMsg[:p+1])
					newMsg = newMsg[p+1:]
					fmt.Printf("1 newMsg   ----->%#v\n", newMsg)

					t := newMsg.Int16()
					msgs <- fmt.Sprintf("vars types numbers ---->%#v\n", t)
					for i := 0; i < t; i++ {
						t = newMsg.Int16()
						msgs <- fmt.Sprintf("22 newMsg   ----->%#v\n", newMsg)
					}

					msgs <- fmt.Sprintf("23 newMsg   ----->%#v\n", newMsg)
					totalVar := newMsg.Int16()
					vars := make(map[int]string)
					var varsIdx []int
					for i := 0; i < totalVar; i++ {
						msgs <- fmt.Sprintf("2 newMsg   ----->%#v\n", newMsg)
						varLen := newMsg.Int32()
						// aa := newMsg.Next(4)
						// fmt.Printf("aa   -----> %#v\n", aa)
						// fmt.Printf("aa bits ----->%8b\n", aa[len(aa)-1])
						// varLen := int(binary.BigEndian.Uint32(aa))
						msgs <- fmt.Sprintf("varLen ----->%v\n", varLen)
						msgs <- fmt.Sprintf("newMsg   ----->%#v\n", newMsg)
						if varLen > len(newMsg) {
							varLen = len(newMsg) - 4
						}
						vars[i] = string(newMsg.Next(varLen))
						msgs <- fmt.Sprintf("vars   ----->%#v\n", vars)
						varsIdx = append(varsIdx, i)
						msgs <- fmt.Sprintf("varIdx  ----->%#v\n", varsIdx)

					}

					sort.Sort(sort.Reverse(sort.IntSlice(varsIdx)))
					for _, k := range varsIdx {
						// messages = append(messages, strings.Replace(temp, fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", string(newMsg[k+1])), -1))
						temp = strings.Replace(temp, fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", string(vars[k])), -1)
						msgs <- fmt.Sprintf("message subst k ----->%v\n", k)
						msgs <- fmt.Sprintf("message subst newMsg ----->%#v\n", newMsg)
						msgs <- fmt.Sprintf("message subst msg ----->%v\n", vars[k+1])
						msgs <- fmt.Sprintf("message subst temp ----->%v\n", temp)
						msgs <- fmt.Sprintf("message subst param %s ----->%v\n", fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", vars[k]))
					}
					msgs <- fmt.Sprintf("end message  ----->%v\n", temp)
					msgOut <- msgStruct{Type: "M", Content: temp}
				}
				temp = ""
			}
		}
	}()

	proxy.Start(localHost, dbHostname, dbPort, getQueryModificada, msgs, msgCh)
}

func getQueryModificada(queryOriginal string) string {
	// log.Println("aa")
	// if queryOriginal[:5] != "power" {
	// 	return queryOriginal
	// }

	// log.Println(queryOriginal)
	fmt.Println(queryOriginal)
	return queryOriginal
}
<commit_msg>Update<commit_after>package main

import (
	"bufio"
	"bytes"
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"

	// "database/sql"

	_ "github.com/lib/pq"

	"github.com/DimShadoWWW/power-pg/proxy"
	"github.com/parnurzeal/gorequest"
)

var (
	localHost     = flag.String("l", ":5432", "puerto local para escuchar")
	dbHostname    = flag.String("h", "localhost", "hostname del servidor PostgreSQL")
	dbPort        = flag.String("r", "5432", "puerto del servidor PostgreSQL")
	dbName        = flag.String("d", "dbname", "nombre de la base de datos")
	dbUsername    = flag.String("u", "username", "usuario para acceder al servidor PostgreSQL")
	dbPassword    = flag.String("p", "password", "password para acceder al servidor PostgreSQL")
	remoteService = flag.String("s", "", "http://localhost:8080/query")
	// messages      = []string{}
)

type msgStruct struct {
	Type    string
	Content string
}

var (
	db *sql.DB
)

func main() {
	flag.Parse()
	// dbinfo := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable",
	// 	*dbUsername, *dbPassword, *dbName)
	// db, err := sql.Open("postgres", dbinfo)

	msgs := make(chan string)
	msgCh := make(chan proxy.Pkg)
	msgOut := make(chan msgStruct)
	if *remoteService != "" {
		go func() {
			time.Sleep(time.Second * 3)
			inFile, _ := os.Open("canales_list.txt")
			defer inFile.Close()
			scanner := bufio.NewScanner(inFile)
			scanner.Split(bufio.ScanLines)

			for scanner.Scan() {
				time.Sleep(time.Second * 1)
				// messages = []string{}
				// fmt.Println(scanner.Text())
				// msgOut <- fmt.Sprintf("# %s\n", scanner.Text())
				msgOut <- msgStruct{Type: "C", Content: scanner.Text()}
				_, _, errs := gorequest.New().Get(fmt.Sprintf("%s%s", *remoteService, scanner.Text())).End()
				if errs != nil {
					log.Fatalf("log failed: %v", errs)
				}
			}
			log.Println("done")
			os.Exit(0)
		}()
	}

	go func() {
		f, err := os.OpenFile("/all.txt", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		for msg := range msgs {
			// fmt.Println(msg)
			_, err := f.WriteString(fmt.Sprintf("%s\n", msg))
			if err != nil {
				log.Fatalf("log failed: %v", err)
			}
		}
	}()

	go func() {
		f, err := os.OpenFile("/reports/report.md", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
		// c := 0
		spaces := regexp.MustCompile("[\t]+")
		multipleSpaces := regexp.MustCompile("    ")
		for {
			// select {
			// case msg1 := <-msgOut:
			msg := <-msgOut
			if msg.Type == "C" {
				// c = 0
				f.Close()
				f, err = os.OpenFile(fmt.Sprintf("/reports/report-%s.md", msg.Content), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				// c = 0
				if err != nil {
					panic(err)
				}
				_, err := f.WriteString(fmt.Sprintf("# %s\n", msg.Content))
				if err != nil {
					log.Fatalf("log failed: %v", err)
				}
			} else {
				// case msg2 := <-msgOut:
				// c = c + 1
				m := spaces.ReplaceAll([]byte(msg.Content), []byte{' '})
				m = multipleSpaces.ReplaceAll(m, []byte{' '})
				_, err := f.WriteString(fmt.Sprintf("\n```sql\n%s\n```\n", string(m)))
				if err != nil {
					log.Fatalf("log failed: %v", err)
				}
			}
		}
	}()

	go func() {
		temp := ""
		for msg := range msgCh {
			if msg.Type == 'P' {
				if strings.Contains(string(msg.Content), "$1") {
					var newMsg proxy.ReadBuf
					newMsg = msg.Content
					_ = newMsg.Int32()

					// The name of the destination portal (an empty string selects the unnamed portal).
					p := bytes.Index(newMsg, []byte{0})
					// remove first string
					msgs <- fmt.Sprintf("msg ---->%#v\n", newMsg)
					msgs <- fmt.Sprintf("msg ---->%s\n", string(newMsg))
					msgs <- fmt.Sprintf("first string ---->%#v\n", newMsg[:p+1])
					msgs <- fmt.Sprintf("first string ---->%s\n", string(newMsg[:p+1]))
					newMsg = newMsg[p+1:]
					fmt.Printf("0 newMsg   ----->%#v\n", newMsg)

					// The name of the source prepared statement (an empty string selects the unnamed prepared statement).
					p = bytes.Index(newMsg, []byte{0})
					// remove second string
					msgs <- fmt.Sprintf("second string: message ---->%#v\n", newMsg[:p])
					temp = string(newMsg[:p])
					msgs <- fmt.Sprintf("second string: message temp ---->%s\n", temp)

				} else {
					temp = ""
					selectIdx := strings.Index(string(msg.Content), string([]byte{83, 69, 76, 69, 67, 84, 32}))
					if selectIdx == -1 {
						selectIdx = 0
					}
					sepIdx := strings.Index(string(msg.Content), string([]byte{0, 0, 1, 0, 0}))
					if sepIdx == -1 || sepIdx+5 > len(msg.Content) {
						sepIdx := strings.Index(string(msg.Content), string([]byte{0, 1, 0, 0}))
						if sepIdx == -1 || sepIdx+4 > len(msg.Content) {
							sepIdx = len(msg.Content) - 4
						} else {
							sepIdx = len(msg.Content) - 1
						}
					}
					if sepIdx == -1 {
						sepIdx = len(msg.Content)
					}
					if selectIdx == -1 {
						selectIdx = 0
					}

					fmt.Printf("SEP index ----->%v\n", sepIdx)
					fmt.Printf("SEP len   ----->%v\n", len(msg.Content))
					fmt.Printf("SEP CONT  ----->%v\n", msg.Content)
					// messages = append(messages, string(bytes.Trim(msg.Content[selectIdx:sepIdx], "\x00")))
					msgOut <- msgStruct{Type: "M", Content: string(bytes.Trim(msg.Content[selectIdx:sepIdx], "\x00"))}
				}
			} else {
				if msg.Type == 'B' && temp != "" {
					var newMsg proxy.ReadBuf
					newMsg = msg.Content

					// The name of the destination portal (an empty string selects the unnamed portal).
					p := bytes.Index(newMsg, []byte{0})
					// remove first string
					msgs <- fmt.Sprintf("msg ---->%#v\n", newMsg)
					msgs <- fmt.Sprintf("first string ---->%#v\n", newMsg[:p+1])
					newMsg = newMsg[p+1:]
					fmt.Printf("0 newMsg   ----->%#v\n", newMsg)

					// The name of the source prepared statement (an empty string selects the unnamed prepared statement).
					p = bytes.Index(newMsg, []byte{0})
					// remove second string
					msgs <- fmt.Sprintf("second string ---->%#v\n", newMsg[:p+1])
					newMsg = newMsg[p+1:]
					fmt.Printf("1 newMsg   ----->%#v\n", newMsg)

					t := newMsg.Int16()
					msgs <- fmt.Sprintf("vars types numbers ---->%#v\n", t)
					for i := 0; i < t; i++ {
						t = newMsg.Int16()
						msgs <- fmt.Sprintf("22 newMsg   ----->%#v\n", newMsg)
					}

					msgs <- fmt.Sprintf("23 newMsg   ----->%#v\n", newMsg)
					totalVar := newMsg.Int16()
					vars := make(map[int]string)
					var varsIdx []int
					if totalVar == 0 && len(newMsg) > 4 {
						totalVar = newMsg.Int16()
					}
					msgs <- fmt.Sprintf("totalVar   ----->%d\n", totalVar)
					for i := 0; i < totalVar; i++ {
						msgs <- fmt.Sprintf("2 newMsg   ----->%#v\n", newMsg)
						varLen := newMsg.Int32()
						// aa := newMsg.Next(4)
						// fmt.Printf("aa   -----> %#v\n", aa)
						// fmt.Printf("aa bits ----->%8b\n", aa[len(aa)-1])
						// varLen := int(binary.BigEndian.Uint32(aa))
						msgs <- fmt.Sprintf("varLen ----->%v\n", varLen)
						msgs <- fmt.Sprintf("newMsg   ----->%#v\n", newMsg)
						if varLen > len(newMsg) {
							varLen = len(newMsg) - 4
						}
						vars[i] = string(newMsg.Next(varLen))
						msgs <- fmt.Sprintf("vars   ----->%#v\n", vars)
						varsIdx = append(varsIdx, i)
						msgs <- fmt.Sprintf("varIdx  ----->%#v\n", varsIdx)
					}

					sort.Sort(sort.Reverse(sort.IntSlice(varsIdx)))
					for _, k := range varsIdx {
						// messages = append(messages, strings.Replace(temp, fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", string(newMsg[k+1])), -1))
						temp = strings.Replace(temp, fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", string(vars[k])), -1)
						msgs <- fmt.Sprintf("message subst k ----->%v\n", k)
						msgs <- fmt.Sprintf("message subst newMsg ----->%#v\n", newMsg)
						msgs <- fmt.Sprintf("message subst msg ----->%v\n", vars[k+1])
						msgs <- fmt.Sprintf("message subst temp ----->%v\n", temp)
						msgs <- fmt.Sprintf("message subst param %s ----->%v\n", fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", vars[k]))
					}
					msgs <- fmt.Sprintf("end message  ----->%v\n", temp)
					msgOut <- msgStruct{Type: "M", Content: temp}
				}
				temp = ""
			}
		}
	}()

	proxy.Start(localHost, dbHostname, dbPort, getQueryModificada, msgs, msgCh)
}

func getQueryModificada(queryOriginal string) string {
	// log.Println("aa")
	// if queryOriginal[:5] != "power" {
	// 	return queryOriginal
	// }

	// log.Println(queryOriginal)
	fmt.Println(queryOriginal)
	return queryOriginal
}
<|endoftext|>


<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"fmt"
	avro "github.com/stealthly/go-avro"
	"hash/fnv"
	"time"
)

var TimingField = &avro.SchemaField{
	Name: "timings",
	Doc: "Timings",
	Default: "null",
	Type: &avro.ArraySchema{
		Items: &avro.LongSchema{},
	},
}

// MirrorMakerConfig defines configuration options for MirrorMaker
type MirrorMakerConfig struct {
	// Whitelist of topics to mirror. Exactly one whitelist or blacklist is allowed.
	Whitelist string

	// Blacklist of topics to mirror. Exactly one whitelist or blacklist is allowed.
	Blacklist string

	// Consumer configurations to consume from a source cluster.
	ConsumerConfigs []string

	// Embedded producer config.
	ProducerConfig string

	// Number of producer instances.
	NumProducers int

	// Number of consumption streams.
	NumStreams int

	// Flag to preserve partition number. E.g. if message was read from partition 5 it'll be written to partition 5. Note that this can affect performance.
	PreservePartitions bool

	// Flag to preserve message order. E.g. message sequence 1, 2, 3, 4, 5 will remain 1, 2, 3, 4, 5 in destination topic. Note that this can affect performance.
	PreserveOrder bool

	// Destination topic prefix. E.g. if message was read from topic "test" and prefix is "dc1_" it'll be written to topic "dc1_test".
	TopicPrefix string

	// Number of messages that are buffered between the consumer and producer.
	ChannelSize int

	// Message keys encoder for producer
	KeyEncoder Encoder

	// Message values encoder for producer
	ValueEncoder Encoder

	// Message keys decoder for consumer
	KeyDecoder Decoder

	// Message values decoder for consumer
	ValueDecoder Decoder

	// Function that generates producer instances
	ProducerConstructor ProducerConstructor

	// Path to producer configuration, that is responsible for logging timings
	// Defines whether add timings to message or not.
	// Note: used only for avro encoded messages
	TimingsProducerConfig string
}

// Creates an empty MirrorMakerConfig.
func NewMirrorMakerConfig() *MirrorMakerConfig {
	return &MirrorMakerConfig{
		KeyEncoder:          &ByteEncoder{},
		ValueEncoder:        &ByteEncoder{},
		KeyDecoder:          &ByteDecoder{},
		ValueDecoder:        &ByteDecoder{},
		ProducerConstructor: NewSaramaProducer,
	}
}

// MirrorMaker is a tool to mirror source Kafka cluster into a target (mirror) Kafka cluster.
// It uses a Kafka consumer to consume messages from the source cluster, and re-publishes those messages to the target cluster.
type MirrorMaker struct {
	config          *MirrorMakerConfig
	consumers       []*Consumer
	producers       []Producer
	messageChannels []chan *Message
	timingsProducer Producer
	newSchema		*avro.RecordSchema
}

// Creates a new MirrorMaker using given MirrorMakerConfig.
func NewMirrorMaker(config *MirrorMakerConfig) *MirrorMaker {
	return &MirrorMaker{
		config: config,
	}
}

// Starts the MirrorMaker. This method is blocking and should probably be run in a separate goroutine.
func (this *MirrorMaker) Start() {
	this.initializeMessageChannels()
	this.startConsumers()
	this.startProducers()
}

// Gracefully stops the MirrorMaker.
func (this *MirrorMaker) Stop() {
	consumerCloseChannels := make([]<-chan bool, 0)
	for _, consumer := range this.consumers {
		consumerCloseChannels = append(consumerCloseChannels, consumer.Close())
	}

	for _, ch := range consumerCloseChannels {
		<-ch
	}

	for _, ch := range this.messageChannels {
		close(ch)
	}

	//TODO maybe drain message channel first?
	for _, producer := range this.producers {
		producer.Close()
	}
}

func (this *MirrorMaker) startConsumers() {
	for _, consumerConfigFile := range this.config.ConsumerConfigs {
		config, err := ConsumerConfigFromFile(consumerConfigFile)
		if err != nil {
			panic(err)
		}
		config.KeyDecoder = this.config.KeyDecoder
		config.ValueDecoder = this.config.ValueDecoder

		zkConfig, err := ZookeeperConfigFromFile(consumerConfigFile)
		if err != nil {
			panic(err)
		}
		config.NumWorkers = 1
		config.AutoOffsetReset = SmallestOffset
		config.Coordinator = NewZookeeperCoordinator(zkConfig)
		config.WorkerFailureCallback = func(_ *WorkerManager) FailedDecision {
			return CommitOffsetAndContinue
		}
		config.WorkerFailedAttemptCallback = func(_ *Task, _ WorkerResult) FailedDecision {
			return CommitOffsetAndContinue
		}
		if this.config.PreserveOrder {
			numProducers := this.config.NumProducers
			config.Strategy = func(_ *Worker, msg *Message, id TaskId) WorkerResult {
				if this.config.TimingsProducerConfig != "" {
					if record, ok := msg.DecodedValue.(*avro.GenericRecord); ok {
						this.addTiming(record)
						this.messageChannels[topicPartitionHash(msg)%numProducers] <- msg
					} else {
						return NewProcessingFailedResult(id)
					}
				}

				return NewSuccessfulResult(id)
			}
		} else {
			config.Strategy = func(_ *Worker, msg *Message, id TaskId) WorkerResult {
				this.messageChannels[0] <- msg

				return NewSuccessfulResult(id)
			}
		}

		consumer := NewConsumer(config)
		this.consumers = append(this.consumers, consumer)
		if this.config.Whitelist != "" {
			go consumer.StartWildcard(NewWhiteList(this.config.Whitelist), this.config.NumStreams)
		} else if this.config.Blacklist != "" {
			go consumer.StartWildcard(NewBlackList(this.config.Blacklist), this.config.NumStreams)
		} else {
			panic("Consume pattern not specified")
		}
	}
}

func (this *MirrorMaker) initializeMessageChannels() {
	if this.config.PreserveOrder {
		for i := 0; i < this.config.NumProducers; i++ {
			this.messageChannels = append(this.messageChannels, make(chan *Message, this.config.ChannelSize))
		}
	} else {
		this.messageChannels = append(this.messageChannels, make(chan *Message, this.config.ChannelSize))
	}
}

func (this *MirrorMaker) startProducers() {
	if this.config.TimingsProducerConfig != "" {
		conf, err := ProducerConfigFromFile(this.config.TimingsProducerConfig)
		if err != nil {
			panic(err)
		}
		if this.config.PreservePartitions {
			conf.Partitioner = NewFixedPartitioner
		} else {
			conf.Partitioner = NewRandomPartitioner
		}
		conf.KeyEncoder = this.config.KeyEncoder
		conf.ValueEncoder = this.config.ValueEncoder
		this.timingsProducer = this.config.ProducerConstructor(conf)
		go this.failedRoutine(this.timingsProducer)
	}

	for i := 0; i < this.config.NumProducers; i++ {
		conf, err := ProducerConfigFromFile(this.config.ProducerConfig)
		if err != nil {
			panic(err)
		}
		if this.config.PreservePartitions {
			conf.Partitioner = NewFixedPartitioner
		} else {
			conf.Partitioner = NewRandomPartitioner
		}
		conf.KeyEncoder = this.config.KeyEncoder
		conf.ValueEncoder = this.config.ValueEncoder
		producer := this.config.ProducerConstructor(conf)
		this.producers = append(this.producers, producer)
		if this.config.TimingsProducerConfig != "" {
			go this.timingsRoutine(producer)
		}
		go this.failedRoutine(producer)
		if this.config.PreserveOrder {
			go this.produceRoutine(producer, i)
		} else {
			go this.produceRoutine(producer, 0)
		}
	}
}

func (this *MirrorMaker) produceRoutine(producer Producer, channelIndex int) {
	partitionEncoder := &Int32Encoder{}
	for msg := range this.messageChannels[channelIndex] {
		if this.config.PreservePartitions {
			producer.Input() <- &ProducerMessage{Topic: this.config.TopicPrefix + msg.Topic, Key: uint32(msg.Partition), Value: msg.DecodedValue, KeyEncoder: partitionEncoder}
		} else {
			producer.Input() <- &ProducerMessage{Topic: this.config.TopicPrefix + msg.Topic, Key: msg.Key, Value: msg.DecodedValue}
		}
	}
}

func (this *MirrorMaker) timingsRoutine(producer Producer) {
	partitionEncoder := &Int32Encoder{}
	partitionDecoder := &Int32Decoder{}
	for msg := range producer.Successes() {
		decodedKey, err := partitionDecoder.Decode(msg.Key.([]byte))
		if err != nil {
			Errorf(this, "Failed to decode %v", msg.Key)
		}
		decodedValue, err := this.config.ValueDecoder.Decode(msg.Value.([]byte))
		if err != nil {
			Errorf(this, "Failed to decode %v", msg.Value)
		}

		if record, ok := decodedValue.(*avro.GenericRecord); ok {
			this.addTiming(record)
			this.timingsProducer.Input() <- &ProducerMessage{Topic: "timings_" + msg.Topic, Key: decodedKey.(uint32),
				Value: record, KeyEncoder: partitionEncoder}
		} else {
			Errorf(this, "Invalid avro schema type %s", decodedValue)
		}
	}
}

func (this *MirrorMaker) failedRoutine(producer Producer) {
	for msg := range producer.Errors() {
		Error("mirrormaker", msg.err)
	}
}

func (this *MirrorMaker) addTiming(record *avro.GenericRecord) {
	now := time.Now().Unix()
	if this.newSchema == nil {
		schema := *record.Schema().(*avro.RecordSchema)
		this.newSchema = &schema
		this.newSchema.Fields = append(this.newSchema.Fields, TimingField)
	}
	var timings []int64
	if record.Get("timings") == nil {
		timings = make([]int64, 0)
		record.Set("timings", timings)
	}
	timings = append(timings, now)
}

func topicPartitionHash(msg *Message) int {
	h := fnv.New32a()
	h.Write([]byte(fmt.Sprintf("%s%d", msg.Topic, msg.Partition)))
	return int(h.Sum32())
}
<commit_msg>re #23 changed way addTimings works<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"fmt"
	avro "github.com/stealthly/go-avro"
	"hash/fnv"
	"time"
)

var TimingField = &avro.SchemaField{
	Name: "timings",
	Doc: "Timings",
	Default: "null",
	Type: &avro.ArraySchema{
		Items: &avro.LongSchema{},
	},
}

// MirrorMakerConfig defines configuration options for MirrorMaker
type MirrorMakerConfig struct {
	// Whitelist of topics to mirror. Exactly one whitelist or blacklist is allowed.
	Whitelist string

	// Blacklist of topics to mirror. Exactly one whitelist or blacklist is allowed.
	Blacklist string

	// Consumer configurations to consume from a source cluster.
	ConsumerConfigs []string

	// Embedded producer config.
	ProducerConfig string

	// Number of producer instances.
	NumProducers int

	// Number of consumption streams.
	NumStreams int

	// Flag to preserve partition number. E.g. if message was read from partition 5 it'll be written to partition 5. Note that this can affect performance.
	PreservePartitions bool

	// Flag to preserve message order. E.g. message sequence 1, 2, 3, 4, 5 will remain 1, 2, 3, 4, 5 in destination topic. Note that this can affect performance.
	PreserveOrder bool

	// Destination topic prefix. E.g. if message was read from topic "test" and prefix is "dc1_" it'll be written to topic "dc1_test".
	TopicPrefix string

	// Number of messages that are buffered between the consumer and producer.
	ChannelSize int

	// Message keys encoder for producer
	KeyEncoder Encoder

	// Message values encoder for producer
	ValueEncoder Encoder

	// Message keys decoder for consumer
	KeyDecoder Decoder

	// Message values decoder for consumer
	ValueDecoder Decoder

	// Function that generates producer instances
	ProducerConstructor ProducerConstructor

	// Path to producer configuration, that is responsible for logging timings
	// Defines whether add timings to message or not.
	// Note: used only for avro encoded messages
	TimingsProducerConfig string
}

// Creates an empty MirrorMakerConfig.
func NewMirrorMakerConfig() *MirrorMakerConfig {
	return &MirrorMakerConfig{
		KeyEncoder:          &ByteEncoder{},
		ValueEncoder:        &ByteEncoder{},
		KeyDecoder:          &ByteDecoder{},
		ValueDecoder:        &ByteDecoder{},
		ProducerConstructor: NewSaramaProducer,
	}
}

// MirrorMaker is a tool to mirror source Kafka cluster into a target (mirror) Kafka cluster.
// It uses a Kafka consumer to consume messages from the source cluster, and re-publishes those messages to the target cluster.
type MirrorMaker struct {
	config          *MirrorMakerConfig
	consumers       []*Consumer
	producers       []Producer
	messageChannels []chan *Message
	timingsProducer Producer
	newSchema		*avro.RecordSchema
}

// Creates a new MirrorMaker using given MirrorMakerConfig.
func NewMirrorMaker(config *MirrorMakerConfig) *MirrorMaker {
	return &MirrorMaker{
		config: config,
	}
}

// Starts the MirrorMaker. This method is blocking and should probably be run in a separate goroutine.
func (this *MirrorMaker) Start() {
	this.initializeMessageChannels()
	this.startConsumers()
	this.startProducers()
}

// Gracefully stops the MirrorMaker.
func (this *MirrorMaker) Stop() {
	consumerCloseChannels := make([]<-chan bool, 0)
	for _, consumer := range this.consumers {
		consumerCloseChannels = append(consumerCloseChannels, consumer.Close())
	}

	for _, ch := range consumerCloseChannels {
		<-ch
	}

	for _, ch := range this.messageChannels {
		close(ch)
	}

	//TODO maybe drain message channel first?
	for _, producer := range this.producers {
		producer.Close()
	}
}

func (this *MirrorMaker) startConsumers() {
	for _, consumerConfigFile := range this.config.ConsumerConfigs {
		config, err := ConsumerConfigFromFile(consumerConfigFile)
		if err != nil {
			panic(err)
		}
		config.KeyDecoder = this.config.KeyDecoder
		config.ValueDecoder = this.config.ValueDecoder

		zkConfig, err := ZookeeperConfigFromFile(consumerConfigFile)
		if err != nil {
			panic(err)
		}
		config.NumWorkers = 1
		config.AutoOffsetReset = SmallestOffset
		config.Coordinator = NewZookeeperCoordinator(zkConfig)
		config.WorkerFailureCallback = func(_ *WorkerManager) FailedDecision {
			return CommitOffsetAndContinue
		}
		config.WorkerFailedAttemptCallback = func(_ *Task, _ WorkerResult) FailedDecision {
			return CommitOffsetAndContinue
		}
		if this.config.PreserveOrder {
			numProducers := this.config.NumProducers
			config.Strategy = func(_ *Worker, msg *Message, id TaskId) WorkerResult {
				if this.config.TimingsProducerConfig != "" {
					if record, ok := msg.DecodedValue.(*avro.GenericRecord); ok {
						record = this.addTiming(record)
						this.messageChannels[topicPartitionHash(msg)%numProducers] <- msg
					} else {
						return NewProcessingFailedResult(id)
					}
				}

				return NewSuccessfulResult(id)
			}
		} else {
			config.Strategy = func(_ *Worker, msg *Message, id TaskId) WorkerResult {
				this.messageChannels[0] <- msg

				return NewSuccessfulResult(id)
			}
		}

		consumer := NewConsumer(config)
		this.consumers = append(this.consumers, consumer)
		if this.config.Whitelist != "" {
			go consumer.StartWildcard(NewWhiteList(this.config.Whitelist), this.config.NumStreams)
		} else if this.config.Blacklist != "" {
			go consumer.StartWildcard(NewBlackList(this.config.Blacklist), this.config.NumStreams)
		} else {
			panic("Consume pattern not specified")
		}
	}
}

func (this *MirrorMaker) initializeMessageChannels() {
	if this.config.PreserveOrder {
		for i := 0; i < this.config.NumProducers; i++ {
			this.messageChannels = append(this.messageChannels, make(chan *Message, this.config.ChannelSize))
		}
	} else {
		this.messageChannels = append(this.messageChannels, make(chan *Message, this.config.ChannelSize))
	}
}

func (this *MirrorMaker) startProducers() {
	if this.config.TimingsProducerConfig != "" {
		conf, err := ProducerConfigFromFile(this.config.TimingsProducerConfig)
		if err != nil {
			panic(err)
		}
		if this.config.PreservePartitions {
			conf.Partitioner = NewFixedPartitioner
		} else {
			conf.Partitioner = NewRandomPartitioner
		}
		conf.KeyEncoder = this.config.KeyEncoder
		conf.ValueEncoder = this.config.ValueEncoder
		this.timingsProducer = this.config.ProducerConstructor(conf)
		go this.failedRoutine(this.timingsProducer)
	}

	for i := 0; i < this.config.NumProducers; i++ {
		conf, err := ProducerConfigFromFile(this.config.ProducerConfig)
		if err != nil {
			panic(err)
		}
		if this.config.PreservePartitions {
			conf.Partitioner = NewFixedPartitioner
		} else {
			conf.Partitioner = NewRandomPartitioner
		}
		conf.KeyEncoder = this.config.KeyEncoder
		conf.ValueEncoder = this.config.ValueEncoder
		producer := this.config.ProducerConstructor(conf)
		this.producers = append(this.producers, producer)
		if this.config.TimingsProducerConfig != "" {
			go this.timingsRoutine(producer)
		}
		go this.failedRoutine(producer)
		if this.config.PreserveOrder {
			go this.produceRoutine(producer, i)
		} else {
			go this.produceRoutine(producer, 0)
		}
	}
}

func (this *MirrorMaker) produceRoutine(producer Producer, channelIndex int) {
	partitionEncoder := &Int32Encoder{}
	for msg := range this.messageChannels[channelIndex] {
		if this.config.PreservePartitions {
			producer.Input() <- &ProducerMessage{Topic: this.config.TopicPrefix + msg.Topic, Key: uint32(msg.Partition), Value: msg.DecodedValue, KeyEncoder: partitionEncoder}
		} else {
			producer.Input() <- &ProducerMessage{Topic: this.config.TopicPrefix + msg.Topic, Key: msg.Key, Value: msg.DecodedValue}
		}
	}
}

func (this *MirrorMaker) timingsRoutine(producer Producer) {
	partitionEncoder := &Int32Encoder{}
	partitionDecoder := &Int32Decoder{}
	for msg := range producer.Successes() {
		decodedKey, err := partitionDecoder.Decode(msg.Key.([]byte))
		if err != nil {
			Errorf(this, "Failed to decode %v", msg.Key)
		}
		decodedValue, err := this.config.ValueDecoder.Decode(msg.Value.([]byte))
		if err != nil {
			Errorf(this, "Failed to decode %v", msg.Value)
		}

		if record, ok := decodedValue.(*avro.GenericRecord); ok {
			record = this.addTiming(record)
			this.timingsProducer.Input() <- &ProducerMessage{Topic: "timings_" + msg.Topic, Key: decodedKey.(uint32),
				Value: record, KeyEncoder: partitionEncoder}
		} else {
			Errorf(this, "Invalid avro schema type %s", decodedValue)
		}
	}
}

func (this *MirrorMaker) failedRoutine(producer Producer) {
	for msg := range producer.Errors() {
		Error("mirrormaker", msg.err)
	}
}

func (this *MirrorMaker) addTiming(record *avro.GenericRecord) *avro.GenericRecord {
	now := time.Now().Unix()
	if this.newSchema == nil {
		schema := *record.Schema().(*avro.RecordSchema)
		this.newSchema = &schema
		this.newSchema.Fields = append(this.newSchema.Fields, TimingField)
	}
	var timings []int64
	if record.Get("timings") == nil {
		timings = make([]int64, 0)
		newRecord := avro.NewGenericRecord(this.newSchema)
		for _, field := range this.newSchema.Fields {
			newRecord.Set(field.Name, record.Get(field.Name))
		}
		record = newRecord
	} else {
		timings = record.Get("timings").([]int64)
	}
	timings = append(timings, now)
	record.Set("timings", timings)

	return record
}

func topicPartitionHash(msg *Message) int {
	h := fnv.New32a()
	h.Write([]byte(fmt.Sprintf("%s%d", msg.Topic, msg.Partition)))
	return int(h.Sum32())
}
<|endoftext|>


<commit_before>/*
Package sdk is the gRPC implementation of the SDK gRPC server
Copyright 2018 Portworx

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sdk

import (
	"context"

	"github.com/libopenstorage/openstorage/api"
	ost_errors "github.com/libopenstorage/openstorage/api/errors"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// Start a volume migration
func (s *VolumeServer) Start(
	ctx context.Context,
	req *api.SdkCloudMigrateStartRequest,
) (*api.SdkCloudMigrateStartResponse, error) {

	if volume := req.GetVolume(); volume != nil {
		// Check ownership
		if err := checkAccessFromDriverForVolumeId(ctx, s.driver(ctx), volume.GetVolumeId(), api.Ownership_Read); err != nil {
			return nil, err
		}

		//migrate volume
		return s.volumeMigrate(ctx, req, volume)
	} else if volumeGroup := req.GetVolumeGroup(); volumeGroup != nil {

		labels := make(map[string]string, 0)
		labels["group"] = volumeGroup.GetGroupId()
		if !s.haveOwnership(ctx, labels) {
			return nil, status.Error(codes.PermissionDenied, "Volume Operation not Permitted.")
		}
		//migrate volume groups
		return s.volumeGroupMigrate(ctx, req, volumeGroup)
	} else if allVolumes := req.GetAllVolumes(); allVolumes != nil {
		// migrate all volumes
		if !s.haveOwnership(ctx, nil) {
			return nil, status.Error(codes.PermissionDenied, "Volume Operation not Permitted.")
		}
		return s.allVolumesMigrate(ctx, req, allVolumes)
	}
	return nil, status.Error(codes.InvalidArgument, "Unknown operation request")
}

func (s *VolumeServer) haveOwnership(ctx context.Context, labels map[string]string) bool {
	vols, err := s.driver(ctx).Enumerate(nil, labels)
	if err != nil {
		return false
	}
	for _, vol := range vols {
		// Check ownership
		if !vol.IsPermitted(ctx, api.Ownership_Read) {
			return false
		}
	}

	return true
}

func (s *VolumeServer) volumeGroupMigrate(
	ctx context.Context,
	req *api.SdkCloudMigrateStartRequest,
	volumeGroup *api.SdkCloudMigrateStartRequest_MigrateVolumeGroup,
) (*api.SdkCloudMigrateStartResponse, error) {
	//Create a request object with operation as Migrate volume
	request := &api.CloudMigrateStartRequest{
		Operation: api.CloudMigrate_MigrateVolumeGroup,
		ClusterId: req.GetClusterId(),
		TargetId:  volumeGroup.GetGroupId(),
		TaskId:    req.GetTaskId(), // optional will be "" if not passed
	}
	resp, err := s.driver(ctx).CloudMigrateStart(request)
	if err != nil {
		if _, ok := err.(*ost_errors.ErrExists); ok {
			return nil, status.Errorf(codes.AlreadyExists, "Cannot start migration for %s : %v", req.GetClusterId(), err)
		}
		// if errExist return codes.
		return nil, status.Errorf(codes.Internal, "Cannot start migration for %s : %v", req.GetClusterId(), err)
	}
	return &api.SdkCloudMigrateStartResponse{
		Result: resp,
	}, nil
}

func (s *VolumeServer) allVolumesMigrate(
	ctx context.Context,
	req *api.SdkCloudMigrateStartRequest,
	allVolume *api.SdkCloudMigrateStartRequest_MigrateAllVolumes,
) (*api.SdkCloudMigrateStartResponse, error) {
	//Create a request object with operation as Migrate volume
	request := &api.CloudMigrateStartRequest{
		Operation: api.CloudMigrate_MigrateCluster,
		ClusterId: req.GetClusterId(),
		TaskId:    req.GetTaskId(),
	}
	resp, err := s.driver(ctx).CloudMigrateStart(request)
	if err != nil {
		if _, ok := err.(*ost_errors.ErrExists); ok {
			return nil, status.Errorf(codes.AlreadyExists, "Cannot start migration for %s : %v", req.GetClusterId(), err)
		}
		return nil, status.Errorf(codes.Internal, "Cannot start migration for %s : %v", req.GetClusterId(), err)
	}
	return &api.SdkCloudMigrateStartResponse{
		Result: resp,
	}, nil
}

func (s *VolumeServer) volumeMigrate(
	ctx context.Context,
	req *api.SdkCloudMigrateStartRequest,
	volume *api.SdkCloudMigrateStartRequest_MigrateVolume,
) (*api.SdkCloudMigrateStartResponse, error) {
	//Create a request object with operation as Migrate volume
	request := &api.CloudMigrateStartRequest{
		Operation: api.CloudMigrate_MigrateVolume,
		ClusterId: req.GetClusterId(),
		TargetId:  volume.GetVolumeId(),
		TaskId:    req.GetTaskId(),
	}
	resp, err := s.driver(ctx).CloudMigrateStart(request)
	if err != nil {
		if _, ok := err.(*ost_errors.ErrExists); ok {
			return nil, status.Errorf(codes.AlreadyExists, "Cannot start migration for %s : %v", req.GetClusterId(), err)
		}
		return nil, status.Errorf(codes.Internal, "Cannot start migration for %s : %v", req.GetClusterId(), err)
	}
	return &api.SdkCloudMigrateStartResponse{
		Result: resp,
	}, nil
}

// Cancel or stop a ongoing migration
func (s *VolumeServer) Cancel(
	ctx context.Context,
	req *api.SdkCloudMigrateCancelRequest,
) (*api.SdkCloudMigrateCancelResponse, error) {

	if req.GetRequest() == nil {
		return nil, status.Errorf(codes.InvalidArgument, "Must supply valid request")
	} else if len(req.GetRequest().GetTaskId()) == 0 {
		return nil, status.Errorf(codes.InvalidArgument, "Must supply valid Task ID")
	}
	err := s.driver(ctx).CloudMigrateCancel(req.GetRequest())
	if err != nil {
		return nil, status.Errorf(codes.Internal, "Cannot stop migration for %s : %v",
			req.GetRequest().GetTaskId(), err)
	}
	return &api.SdkCloudMigrateCancelResponse{}, nil
}

// Status of ongoing migration
func (s *VolumeServer) Status(
	ctx context.Context,
	req *api.SdkCloudMigrateStatusRequest,
) (*api.SdkCloudMigrateStatusResponse, error) {

	resp, err := s.driver(ctx).CloudMigrateStatus(req.GetRequest())
	if err != nil {
		return nil, status.Errorf(codes.Internal, "Cannot get status of migration : %v", err)
	}
	return &api.SdkCloudMigrateStatusResponse{
		Result: resp,
	}, nil

}
<commit_msg>Add driver check to Migration calls (#972)<commit_after>/*
Package sdk is the gRPC implementation of the SDK gRPC server
Copyright 2018 Portworx

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sdk

import (
	"context"

	"github.com/libopenstorage/openstorage/api"
	ost_errors "github.com/libopenstorage/openstorage/api/errors"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// Start a volume migration
func (s *VolumeServer) Start(
	ctx context.Context,
	req *api.SdkCloudMigrateStartRequest,
) (*api.SdkCloudMigrateStartResponse, error) {
	if s.cluster() == nil || s.driver(ctx) == nil {
		return nil, status.Error(codes.Unavailable, "Resource has not been initialized")
	}

	if volume := req.GetVolume(); volume != nil {
		// Check ownership
		if err := checkAccessFromDriverForVolumeId(ctx, s.driver(ctx), volume.GetVolumeId(), api.Ownership_Read); err != nil {
			return nil, err
		}

		//migrate volume
		return s.volumeMigrate(ctx, req, volume)
	} else if volumeGroup := req.GetVolumeGroup(); volumeGroup != nil {

		labels := make(map[string]string, 0)
		labels["group"] = volumeGroup.GetGroupId()
		if !s.haveOwnership(ctx, labels) {
			return nil, status.Error(codes.PermissionDenied, "Volume Operation not Permitted.")
		}
		//migrate volume groups
		return s.volumeGroupMigrate(ctx, req, volumeGroup)
	} else if allVolumes := req.GetAllVolumes(); allVolumes != nil {
		// migrate all volumes
		if !s.haveOwnership(ctx, nil) {
			return nil, status.Error(codes.PermissionDenied, "Volume Operation not Permitted.")
		}
		return s.allVolumesMigrate(ctx, req, allVolumes)
	}
	return nil, status.Error(codes.InvalidArgument, "Unknown operation request")
}

func (s *VolumeServer) haveOwnership(ctx context.Context, labels map[string]string) bool {
	// checking if driver is initialized happens in Start
	vols, err := s.driver(ctx).Enumerate(nil, labels)
	if err != nil {
		return false
	}
	for _, vol := range vols {
		// Check ownership
		if !vol.IsPermitted(ctx, api.Ownership_Read) {
			return false
		}
	}

	return true
}

func (s *VolumeServer) volumeGroupMigrate(
	ctx context.Context,
	req *api.SdkCloudMigrateStartRequest,
	volumeGroup *api.SdkCloudMigrateStartRequest_MigrateVolumeGroup,
) (*api.SdkCloudMigrateStartResponse, error) {
	if s.cluster() == nil || s.driver(ctx) == nil {
		return nil, status.Error(codes.Unavailable, "Resource has not been initialized")
	}

	//Create a request object with operation as Migrate volume
	request := &api.CloudMigrateStartRequest{
		Operation: api.CloudMigrate_MigrateVolumeGroup,
		ClusterId: req.GetClusterId(),
		TargetId:  volumeGroup.GetGroupId(),
		TaskId:    req.GetTaskId(), // optional will be "" if not passed
	}
	resp, err := s.driver(ctx).CloudMigrateStart(request)
	if err != nil {
		if _, ok := err.(*ost_errors.ErrExists); ok {
			return nil, status.Errorf(codes.AlreadyExists, "Cannot start migration for %s : %v", req.GetClusterId(), err)
		}
		// if errExist return codes.
		return nil, status.Errorf(codes.Internal, "Cannot start migration for %s : %v", req.GetClusterId(), err)
	}
	return &api.SdkCloudMigrateStartResponse{
		Result: resp,
	}, nil
}

func (s *VolumeServer) allVolumesMigrate(
	ctx context.Context,
	req *api.SdkCloudMigrateStartRequest,
	allVolume *api.SdkCloudMigrateStartRequest_MigrateAllVolumes,
) (*api.SdkCloudMigrateStartResponse, error) {
	if s.cluster() == nil || s.driver(ctx) == nil {
		return nil, status.Error(codes.Unavailable, "Resource has not been initialized")
	}

	//Create a request object with operation as Migrate volume
	request := &api.CloudMigrateStartRequest{
		Operation: api.CloudMigrate_MigrateCluster,
		ClusterId: req.GetClusterId(),
		TaskId:    req.GetTaskId(),
	}
	resp, err := s.driver(ctx).CloudMigrateStart(request)
	if err != nil {
		if _, ok := err.(*ost_errors.ErrExists); ok {
			return nil, status.Errorf(codes.AlreadyExists, "Cannot start migration for %s : %v", req.GetClusterId(), err)
		}
		return nil, status.Errorf(codes.Internal, "Cannot start migration for %s : %v", req.GetClusterId(), err)
	}
	return &api.SdkCloudMigrateStartResponse{
		Result: resp,
	}, nil
}

func (s *VolumeServer) volumeMigrate(
	ctx context.Context,
	req *api.SdkCloudMigrateStartRequest,
	volume *api.SdkCloudMigrateStartRequest_MigrateVolume,
) (*api.SdkCloudMigrateStartResponse, error) {
	if s.cluster() == nil || s.driver(ctx) == nil {
		return nil, status.Error(codes.Unavailable, "Resource has not been initialized")
	}

	//Create a request object with operation as Migrate volume
	request := &api.CloudMigrateStartRequest{
		Operation: api.CloudMigrate_MigrateVolume,
		ClusterId: req.GetClusterId(),
		TargetId:  volume.GetVolumeId(),
		TaskId:    req.GetTaskId(),
	}
	resp, err := s.driver(ctx).CloudMigrateStart(request)
	if err != nil {
		if _, ok := err.(*ost_errors.ErrExists); ok {
			return nil, status.Errorf(codes.AlreadyExists, "Cannot start migration for %s : %v", req.GetClusterId(), err)
		}
		return nil, status.Errorf(codes.Internal, "Cannot start migration for %s : %v", req.GetClusterId(), err)
	}
	return &api.SdkCloudMigrateStartResponse{
		Result: resp,
	}, nil
}

// Cancel or stop a ongoing migration
func (s *VolumeServer) Cancel(
	ctx context.Context,
	req *api.SdkCloudMigrateCancelRequest,
) (*api.SdkCloudMigrateCancelResponse, error) {
	if s.cluster() == nil || s.driver(ctx) == nil {
		return nil, status.Error(codes.Unavailable, "Resource has not been initialized")
	}

	if req.GetRequest() == nil {
		return nil, status.Errorf(codes.InvalidArgument, "Must supply valid request")
	} else if len(req.GetRequest().GetTaskId()) == 0 {
		return nil, status.Errorf(codes.InvalidArgument, "Must supply valid Task ID")
	}
	err := s.driver(ctx).CloudMigrateCancel(req.GetRequest())
	if err != nil {
		return nil, status.Errorf(codes.Internal, "Cannot stop migration for %s : %v",
			req.GetRequest().GetTaskId(), err)
	}
	return &api.SdkCloudMigrateCancelResponse{}, nil
}

// Status of ongoing migration
func (s *VolumeServer) Status(
	ctx context.Context,
	req *api.SdkCloudMigrateStatusRequest,
) (*api.SdkCloudMigrateStatusResponse, error) {
	if s.cluster() == nil || s.driver(ctx) == nil {
		return nil, status.Error(codes.Unavailable, "Resource has not been initialized")
	}

	resp, err := s.driver(ctx).CloudMigrateStatus(req.GetRequest())
	if err != nil {
		return nil, status.Errorf(codes.Internal, "Cannot get status of migration : %v", err)
	}
	return &api.SdkCloudMigrateStatusResponse{
		Result: resp,
	}, nil

}
<|endoftext|>