<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
	"strings"
	"sync"
	"time"
)

var conn dbox.IConnection
var count int

// var mwg sync.WaitGroup

var (
	t0                                 time.Time
	custgroup, prodgroup, plcode, ref  string
	value, fiscalyear, iscount, scount int
	globalgross                        float64
	mapsperiod                         map[string]float64
	mapkeysvalue                       map[string]float64
	masters                            toolkit.M
	mwg                                sync.WaitGroup
)

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, _ := gdrj.Find(fnModel(), filter, nil)
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmaster() {
	masters = toolkit.M{}

	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))

	ccs := buildmap(map[string]*gdrj.CostCenter{},
		func() orm.IModel {
			return new(gdrj.CostCenter)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.CostCenter)
			o := obj.(*gdrj.CostCenter)
			h[o.ID] = o
		}).(map[string]*gdrj.CostCenter)
	masters.Set("costcenter", ccs)
}

func main() {
	t0 = time.Now()
	mapkeysvalue = make(map[string]float64)
	mapsperiod = make(map[string]float64)
	flag.IntVar(&fiscalyear, "year", 2015, "YYYY representation of godrej fiscal year. Default is 2015")
	flag.StringVar(&ref, "ref", "", "Reference from document or other. Default is blank")
	flag.Parse()

	eperiode := time.Date(fiscalyear, 4, 1, 0, 0, 0, 0, time.UTC)
	speriode := eperiode.AddDate(-1, 0, 0)

	// if plcode == "" || value == 0 {
	// 	toolkit.Println("PLCode and Value are mandatory to fill")
	// 	os.Exit(1)
	// }

	setinitialconnection()
	defer gdrj.CloseDb()

	toolkit.Println("Get Data Master...")
	prepmaster()

	toolkit.Println("Count SGA Data Process...")
	// conn, _ = modules.GetDboxIConnection("db_godrej")
	conn, _ := modules.GetDboxIConnection("db_godrej")
	csga, _ := conn.NewQuery().Select().From("tmpsgaallocs").Cursor(nil)
	defer csga.Close()
	defer conn.Close()

	i := 0
	ssga := csga.Count()
	ccs := masters.Get("costcenter").(map[string]*gdrj.CostCenter)
	for {

		i++
		tsga := toolkit.M{}
		e := csga.Fetch(&tsga, 1, false)
		if e != nil {
			break
		}

		date := time.Date(tsga.Get("y", 0).(int), time.Month(tsga.Get("p", 0).(int)), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 4, 0)
		pval := toolkit.Sprintf("%d_%d", date.Year(), int(date.Month()))

		group := ""
		tcc, exist := ccs[tsga.Get("ccid", "").(string)]
		if exist {
			group = tcc.CostGroup01
		}

		if group == "" {
			group = "Other"
		}

		kval := toolkit.Sprintf("%v_%v", pval, group)
		mapkeysvalue[kval] += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)
		mapsperiod[pval] += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)

		if ssga%500 == 0 {
			toolkit.Printfn("Prepare sga master %d of %d in %s",
				i, ssga,
				time.Since(t0).String())
		}

	}

	toolkit.Println("Start Data Process...")
	filter := dbox.And(dbox.Gte("date.date", speriode), dbox.Lt("date.date", eperiode), dbox.Gt("skuid_vdist", ""))
	// filter = dbox.Eq("_id", "RK/IMN/15000001_1")
	c, _ := gdrj.Find(new(gdrj.SalesPL), filter, nil)
	defer c.Close()

	scount = c.Count()
	iscount = 0
	step := scount / 100
	i = 0

	jobs := make(chan *gdrj.SalesPL, count)
	toolkit.Println("Prepare Worker")
	for wi := 0; wi < 20; wi++ {
		mwg.Add(1)
		go worker(wi, jobs)
	}
	// ====================================
	for {
		i++

		spl := new(gdrj.SalesPL)
		e := c.Fetch(spl, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		globalgross += spl.GrossAmount

		if i > step {
			step += scount / 100
			toolkit.Printfn("Preparing %d of %d in %s", i, scount,
				time.Since(t0).String())
		}
	}

	i = 0
	c.ResetFetch()
	step = scount / 100
	// ===========================
	for {
		i++

		spl := new(gdrj.SalesPL)
		e := c.Fetch(spl, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		aplmodel := spl.PLDatas
		for k, _ := range aplmodel {
			if strings.Contains(k, "PL33") || strings.Contains(k, "PL34") || strings.Contains(k, "PL35") {
				// aplmodel[k] = nil
				delete(aplmodel, k)
			}
		}

		spl.PLDatas = aplmodel

		jobs <- spl
		if i > step {
			step += scount / 100
			toolkit.Printfn("Processing %d of %d in %s", i, scount,
				time.Since(t0).String())
		}

	}

	close(jobs)
	mwg.Wait()

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func worker(wi int, jobs <-chan *gdrj.SalesPL) {
	workerConn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerConn.Close()
	defer mwg.Done()

	var j *gdrj.SalesPL
	table := j.TableName()
	plmodels := masters.Get("plmodel").(map[string]*gdrj.PLModel)

	for j = range jobs {

		key := toolkit.Sprintf("%d_%d", j.Date.Year, int(j.Date.Month))

		ratio := j.GrossAmount / globalgross
		totsgaperiod, _ := mapsperiod[key]
		totsgaline := ratio * totsgaperiod
		// toolkit.Println(key, " : ", totsgaperiod)

		for k, v := range mapkeysvalue {
			skey := strings.Split(k, "_")
			if toolkit.ToInt(skey[0], toolkit.RoundingAuto) != j.Date.Year || toolkit.ToInt(skey[1], toolkit.RoundingAuto) != int(j.Date.Month) {
				continue
			}

			j.AddDataCC("PL33", (-totsgaline*v/totsgaperiod)*0.7, skey[2], plmodels)
			j.AddDataCC("PL34", (-totsgaline*v/totsgaperiod)*0.26, skey[2], plmodels)
			j.AddDataCC("PL35", (-totsgaline*v/totsgaperiod)*0.04, skey[2], plmodels)

		}

		j.CalcSum(masters)

		e := workerConn.NewQuery().From(table).
			Save().Exec(toolkit.M{}.Set("data", j))

		if e != nil {
			toolkit.Printfn("Unable to save %s = %s",
				j.ID, e.Error())
		}

		iscount++
		if iscount%1000 == 0 {
			toolkit.Printfn("Saved %d of %d in %s",
				iscount, scount,
				time.Since(t0).String())
		}
	}
}
<commit_msg>fix formula<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
	"strings"
	"sync"
	"time"
)

var conn dbox.IConnection
var count int

// var mwg sync.WaitGroup

var (
	t0                                                time.Time
	custgroup, prodgroup, plcode, ref                 string
	value, fiscalyear, iscount, gscount, scount, step int
	globalgross, globalsga                            float64
	mapsperiod                                        map[string]float64
	mapkeysvalue                                      map[string]float64
	masters                                           toolkit.M
	mwg                                               sync.WaitGroup
)

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func buildmap(holder interface{},
	fnModel func() orm.IModel,
	filter *dbox.Filter,
	fnIter func(holder interface{}, obj interface{})) interface{} {
	crx, _ := gdrj.Find(fnModel(), filter, nil)
	defer crx.Close()
	for {
		s := fnModel()
		e := crx.Fetch(s, 1, false)
		if e != nil {
			break
		}
		fnIter(holder, s)
	}
	return holder
}

func prepmaster() {
	masters = toolkit.M{}

	masters.Set("plmodel", buildmap(map[string]*gdrj.PLModel{},
		func() orm.IModel {
			return new(gdrj.PLModel)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.PLModel)
			o := obj.(*gdrj.PLModel)
			h[o.ID] = o
		}).(map[string]*gdrj.PLModel))

	ccs := buildmap(map[string]*gdrj.CostCenter{},
		func() orm.IModel {
			return new(gdrj.CostCenter)
		},
		nil,
		func(holder, obj interface{}) {
			h := holder.(map[string]*gdrj.CostCenter)
			o := obj.(*gdrj.CostCenter)
			h[o.ID] = o
		}).(map[string]*gdrj.CostCenter)
	masters.Set("costcenter", ccs)
}

func main() {
	t0 = time.Now()
	mapkeysvalue = make(map[string]float64)
	mapsperiod = make(map[string]float64)
	flag.IntVar(&fiscalyear, "year", 2015, "YYYY representation of godrej fiscal year. Default is 2015")
	flag.StringVar(&ref, "ref", "", "Reference from document or other. Default is blank")
	flag.Parse()

	eperiode := time.Date(fiscalyear, 4, 1, 0, 0, 0, 0, time.UTC)
	speriode := eperiode.AddDate(-1, 0, 0)

	// if plcode == "" || value == 0 {
	// 	toolkit.Println("PLCode and Value are mandatory to fill")
	// 	os.Exit(1)
	// }

	setinitialconnection()
	defer gdrj.CloseDb()

	toolkit.Println("Get Data Master...")
	prepmaster()

	toolkit.Println("Count SGA Data Process...")
	// conn, _ = modules.GetDboxIConnection("db_godrej")
	conn, _ := modules.GetDboxIConnection("db_godrej")
	csga, _ := conn.NewQuery().Select().From("tmpsgaallocs").Cursor(nil)
	defer csga.Close()
	defer conn.Close()

	i := 0
	ssga := csga.Count()
	ccs := masters.Get("costcenter").(map[string]*gdrj.CostCenter)
	for {

		i++
		tsga := toolkit.M{}
		e := csga.Fetch(&tsga, 1, false)
		if e != nil {
			break
		}

		date := time.Date(tsga.Get("y", 0).(int), time.Month(tsga.Get("p", 0).(int)), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 4, 0)
		pval := toolkit.Sprintf("%d_%d", date.Year(), int(date.Month()))

		group := ""
		tcc, exist := ccs[tsga.Get("ccid", "").(string)]
		if exist {
			group = tcc.CostGroup01
		}

		if group == "" {
			group = "Other"
		}

		kval := toolkit.Sprintf("%v_%v", pval, group)
		mapkeysvalue[kval] += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)
		mapsperiod[pval] += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)
		globalsga += toolkit.ToFloat64(tsga.Get("amount", 0), 6, toolkit.RoundingAuto)

		if ssga%500 == 0 {
			toolkit.Printfn("Prepare sga master %d of %d in %s",
				i, ssga,
				time.Since(t0).String())
		}

	}

	toolkit.Println("Start Data Process...")
	filter := dbox.And(dbox.Gte("date.date", speriode), dbox.Lt("date.date", eperiode), dbox.Gt("skuid_vdist", ""))
	// filter = dbox.Eq("_id", "RK/IMN/15000001_1")
	c, _ := gdrj.Find(new(gdrj.SalesPL), filter, nil)
	defer c.Close()

	scount = c.Count()
	iscount = 0
	gscount = 0
	step = scount / 100

	jobs := make(chan *gdrj.SalesPL, count)
	toolkit.Println("Prepare Worker")
	for wi := 0; wi < 50; wi++ {
		mwg.Add(1)
		go worker(wi, jobs)
	}
	// ====================================
	for {
		iscount++

		spl := new(gdrj.SalesPL)
		e := c.Fetch(spl, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		globalgross += spl.GrossAmount

		if iscount%step == 0 {
			toolkit.Printfn("Preparing %d of %d (%d) in %s", iscount, scount, iscount/step,
				time.Since(t0).String())
		}
	}

	c.ResetFetch()
	iscount = 0
	// ===========================
	for {
		iscount++

		spl := new(gdrj.SalesPL)
		e := c.Fetch(spl, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- spl
		if iscount%step == 0 {
			toolkit.Printfn("Processing %d of %d (%d) in %s", iscount, scount, iscount/step,
				time.Since(t0).String())
		}

	}

	close(jobs)
	mwg.Wait()

	toolkit.Printfn("Saved %d of %d (%d) in %s",
		gscount, scount, gscount/step,
		time.Since(t0).String())

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func worker(wi int, jobs <-chan *gdrj.SalesPL) {
	workerConn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerConn.Close()
	defer mwg.Done()

	var j *gdrj.SalesPL
	table := j.TableName()
	plmodels := masters.Get("plmodel").(map[string]*gdrj.PLModel)

	for j = range jobs {

		gscount++

		aplmodel := j.PLDatas
		for k, _ := range aplmodel {
			if strings.Contains(k, "PL33") || strings.Contains(k, "PL34") || strings.Contains(k, "PL35") {
				delete(aplmodel, k)
			}
		}

		j.PLDatas = aplmodel

		key := toolkit.Sprintf("%d_%d", j.Date.Year, int(j.Date.Month))

		ratio := j.GrossAmount / globalgross
		totsgaperiod, _ := mapsperiod[key]
		totsgaline := ratio * globalsga

		if totsgaperiod == 0 {
			continue
		}

		for k, v := range mapkeysvalue {
			skey := strings.Split(k, "_")
			if toolkit.ToInt(skey[0], toolkit.RoundingAuto) != j.Date.Year || toolkit.ToInt(skey[1], toolkit.RoundingAuto) != int(j.Date.Month) {
				continue
			}

			j.AddDataCC("PL33", (-totsgaline*v/totsgaperiod)*0.7, skey[2], plmodels)
			j.AddDataCC("PL34", (-totsgaline*v/totsgaperiod)*0.26, skey[2], plmodels)
			j.AddDataCC("PL35", (-totsgaline*v/totsgaperiod)*0.04, skey[2], plmodels)

		}

		j.CalcSum(masters)

		e := workerConn.NewQuery().From(table).
			Save().Exec(toolkit.M{}.Set("data", j))

		if e != nil {
			toolkit.Printfn("Unable to save %s = %s",
				j.ID, e.Error())
		}

		if gscount%step == 0 {
			toolkit.Printfn("Saved %d of %d (%d) in %s",
				gscount, scount, gscount/step,
				time.Since(t0).String())
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fractal

import (
	"bytes"
	"code.google.com/p/lzma"
	"compress/zlib"
	"fmt"
	"github.com/nfnt/resize"
	"github.com/pointlander/compress"
	"image"
	"image/color"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"syscall"
	"testing"
)

func TestDCT(t *testing.T) {
	in := [8][8]uint8 {
		{255, 0, 255, 0, 255, 0, 255, 0},
		{0, 255, 0, 255, 0, 255, 0, 255},
		{255, 0, 255, 0, 255, 0, 255, 0},
		{0, 255, 0, 255, 0, 255, 0, 255},
		{255, 0, 255, 0, 255, 0, 255, 0},
		{0, 255, 0, 255, 0, 255, 0, 255},
		{255, 0, 255, 0, 255, 0, 255, 0},
		{0, 255, 0, 255, 0, 255, 0, 255},
	}
	dct, idct := [8][8]int {}, [8][8]int {}
	ForwardDCT(&in, &dct)
	InverseDCT(&dct, &idct)
	for i := 0; i < N; i++ {
		for j := 0; j < N; j++ {
			if int(in[i][j]) != idct[i][j] {
				t.Errorf("input does not equal idct(dct(input))")
			}
		}
	}
}

func TestImageDCT(t *testing.T) {
	file, err := os.Open("images/lenna.png")
	if err != nil {
		log.Fatal(err)
	}

	input, _, err := image.Decode(file)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	input = Gray(input)
	output := DCTCoder(input)
	//output = DCTMap(output)
	//output = Paeth8(output)

	file, err = os.Create("lenna_dct.png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, output)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	data := make([]byte, len(output.Pix))
	copy(data, output.Pix)
	in, compressed := make(chan []byte, 1), &bytes.Buffer{}
	in <- data
	close(in)
	compress.BijectiveBurrowsWheelerCoder(in).MoveToFrontRunLengthCoder().AdaptiveCoder().Code(compressed)
	fmt.Printf("%.3f%% %7vb\n", 100 * float64(compressed.Len())/float64(len(output.Pix)), compressed.Len())

	//output = IPaeth8(output)
	//idct := DCTIMap(output)
	idct := DCTDecoder(output)

	file, err = os.Create("lenna_idct.png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, idct)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()
}

func TestFractal(t *testing.T) {
	runtime.GOMAXPROCS(64)

	fcpBuffer := &bytes.Buffer{}
	file, err := os.Open("images/lenna.png")
	if err != nil {
		log.Fatal(err)
	}

	info, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}
	name := info.Name()
	name = name[:strings.Index(name, ".")]

	input, _, err := image.Decode(file)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	width, height, scale := input.Bounds().Max.X, input.Bounds().Max.Y, 1
	width, height = width/scale, height/scale
	input = resize.Resize(uint(width), uint(height), input, resize.NearestNeighbor)

	gray := Gray(input)
	FractalCoder(gray, 4, fcpBuffer)
	fcp := fcpBuffer.Bytes()
	fcpBufferCopy := bytes.NewBuffer(fcp)

	file, err = os.Create(name + ".png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, input)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	file, err = os.Create(name + ".fcp")
	if err != nil {
		log.Fatal(err)
	}

	file.Write(fcp)
	file.Close()

	decoded := FractalDecoder(fcpBuffer, 2)

	file, err = os.Create(name + "_decoded.png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, decoded)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	decoded = FractalDecoder(fcpBufferCopy, 8)

	file, err = os.Create(name + "_decodedx2.png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, decoded)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	{
		tileSize := 2
		r, g, b := splitImage(input)
		buffer := &bytes.Buffer{}
		FractalCoder(r, tileSize, buffer)
		FractalCoder(g, tileSize, buffer)
		FractalCoder(b, tileSize, buffer)
		data := make([]byte, len(buffer.Bytes()))
		copy(data, buffer.Bytes())
		in, output := make(chan []byte, 1), &bytes.Buffer{}
		in <- data
		close(in)
		compress.BijectiveBurrowsWheelerCoder(in).MoveToFrontRunLengthCoder().AdaptiveCoder().Code(output)
		fmt.Printf("%.3f%% %7vb\n", 100 * float64(output.Len())/float64(buffer.Len()), output.Len())
		red := FractalDecoder(buffer, tileSize)
		green := FractalDecoder(buffer, tileSize)
		blue := FractalDecoder(buffer, tileSize)
		decoded := image.NewRGBA(input.Bounds())
		width, height := input.Bounds().Max.X, input.Bounds().Max.Y
		for x := 0; x < width; x++ {
			for y := 0; y < height; y++ {
				r, _, _, _ := red.At(x, y).RGBA()
				g, _, _, _ := green.At(x, y).RGBA()
				b, _, _, _ := blue.At(x, y).RGBA()
				decoded.Set(x, y, color.RGBA{
					R:uint8(r >> 8),
					G:uint8(g >> 8),
					B:uint8(b >> 8),
					A:0xFF})
			}
		}

		file, err = os.Create(name + "_color.png")
		if err != nil {
			log.Fatal(err)
		}

		err = png.Encode(file, decoded)
		if err != nil {
			log.Fatal(err)
		}
		file.Close()
	}

	fmt.Printf("%vb fcp file size\n", len(fcp))

	zpaq_test := func() (int, string) {
		zpaqFile := "fifo.zpaq"
		syscall.Mkfifo(zpaqFile, 0600)
		cmd := exec.Command("zp", "c3", zpaqFile, name+".fcp")
		cmd.Start()
		buffer, err := ioutil.ReadFile(zpaqFile)
		if err != nil {
			log.Fatal(err)
		}
		cmd.Wait()
		return len(buffer), "zp (command)"
	}

	compress_test := func() (int, string) {
		data := make([]byte, len(fcp))
		copy(data, fcp)
		in, buffer := make(chan []byte, 1), &bytes.Buffer{}
		in <- data
		close(in)
		compress.BijectiveBurrowsWheelerCoder(in).MoveToFrontRunLengthCoder().AdaptiveCoder().Code(buffer)
		return buffer.Len(), "github.com/pointlander/compress"
	}

	bzip2_test := func() (int, string) {
		cmd, buffer := exec.Command("bzip2", "--best"), &bytes.Buffer{}
		stdout, _ := cmd.StdoutPipe()
		stdin, _ := cmd.StdinPipe()
		cmd.Start()
		go func() {
			io.Copy(buffer, stdout)
		}()
		stdin.Write(fcp)
		stdin.Close()
		cmd.Wait()
		return buffer.Len(), "bzip2 (command)"
	}

	lzma_test := func() (int, string) {
		buffer := &bytes.Buffer{}
		writer := lzma.NewWriterLevel(buffer, lzma.BestCompression)
		writer.Write(fcp)
		writer.Close()
		return buffer.Len(), "code.google.com/p/lzma"
	}

	zlib_test := func() (int, string) {
		buffer := &bytes.Buffer{}
		writer, _ := zlib.NewWriterLevel(buffer, zlib.BestCompression)
		writer.Write(fcp)
		writer.Close()
		return buffer.Len(), "compress/zlib"
	}

	jpg_test := func() (int, string) {
		buffer := &bytes.Buffer{}
		jpeg.Encode(buffer, gray, nil)
		return buffer.Len(), "image/jpeg"
	}

	tests := []func() (int, string){zpaq_test, compress_test, bzip2_test, lzma_test, zlib_test, jpg_test}
	results := make([]struct {
		size int
		name string
	}, len(tests))
	for i, test := range tests {
		results[i].size, results[i].name = test()
		if i < 5 {
			fmt.Printf("%.3f%% %7vb %v\n", 100*float64(results[i].size)/float64(len(fcp)), results[i].size, results[i].name)
		}
	}

	fmt.Println("\n image compression comparisons")
	for _, result := range results {
		fmt.Printf("%.3f%% %7vb %v\n", 100*float64(result.size)/float64(width*height), result.size, result.name)
	}
}
<commit_msg>Use lzma library from github<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fractal

import (
	"bytes"
	"compress/zlib"
	"fmt"
	"image"
	"image/color"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"syscall"
	"testing"

	"github.com/kjk/lzma"
	"github.com/nfnt/resize"
	"github.com/pointlander/compress"
)

func TestDCT(t *testing.T) {
	in := [8][8]uint8{
		{255, 0, 255, 0, 255, 0, 255, 0},
		{0, 255, 0, 255, 0, 255, 0, 255},
		{255, 0, 255, 0, 255, 0, 255, 0},
		{0, 255, 0, 255, 0, 255, 0, 255},
		{255, 0, 255, 0, 255, 0, 255, 0},
		{0, 255, 0, 255, 0, 255, 0, 255},
		{255, 0, 255, 0, 255, 0, 255, 0},
		{0, 255, 0, 255, 0, 255, 0, 255},
	}
	dct, idct := [8][8]int{}, [8][8]int{}
	ForwardDCT(&in, &dct)
	InverseDCT(&dct, &idct)
	for i := 0; i < N; i++ {
		for j := 0; j < N; j++ {
			if int(in[i][j]) != idct[i][j] {
				t.Errorf("input does not equal idct(dct(input))")
			}
		}
	}
}

func TestImageDCT(t *testing.T) {
	file, err := os.Open("images/lenna.png")
	if err != nil {
		log.Fatal(err)
	}

	input, _, err := image.Decode(file)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	input = Gray(input)
	output := DCTCoder(input)
	//output = DCTMap(output)
	//output = Paeth8(output)

	file, err = os.Create("lenna_dct.png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, output)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	data := make([]byte, len(output.Pix))
	copy(data, output.Pix)
	in, compressed := make(chan []byte, 1), &bytes.Buffer{}
	in <- data
	close(in)
	compress.BijectiveBurrowsWheelerCoder(in).MoveToFrontRunLengthCoder().AdaptiveCoder().Code(compressed)
	fmt.Printf("%.3f%% %7vb\n", 100*float64(compressed.Len())/float64(len(output.Pix)), compressed.Len())

	//output = IPaeth8(output)
	//idct := DCTIMap(output)
	idct := DCTDecoder(output)

	file, err = os.Create("lenna_idct.png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, idct)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()
}

func TestFractal(t *testing.T) {
	runtime.GOMAXPROCS(64)

	fcpBuffer := &bytes.Buffer{}
	file, err := os.Open("images/lenna.png")
	if err != nil {
		log.Fatal(err)
	}

	info, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}
	name := info.Name()
	name = name[:strings.Index(name, ".")]

	input, _, err := image.Decode(file)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	width, height, scale := input.Bounds().Max.X, input.Bounds().Max.Y, 1
	width, height = width/scale, height/scale
	input = resize.Resize(uint(width), uint(height), input, resize.NearestNeighbor)

	gray := Gray(input)
	FractalCoder(gray, 4, fcpBuffer)
	fcp := fcpBuffer.Bytes()
	fcpBufferCopy := bytes.NewBuffer(fcp)

	file, err = os.Create(name + ".png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, input)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	file, err = os.Create(name + ".fcp")
	if err != nil {
		log.Fatal(err)
	}

	file.Write(fcp)
	file.Close()

	decoded := FractalDecoder(fcpBuffer, 2)

	file, err = os.Create(name + "_decoded.png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, decoded)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	decoded = FractalDecoder(fcpBufferCopy, 8)

	file, err = os.Create(name + "_decodedx2.png")
	if err != nil {
		log.Fatal(err)
	}

	err = png.Encode(file, decoded)
	if err != nil {
		log.Fatal(err)
	}
	file.Close()

	{
		tileSize := 2
		r, g, b := splitImage(input)
		buffer := &bytes.Buffer{}
		FractalCoder(r, tileSize, buffer)
		FractalCoder(g, tileSize, buffer)
		FractalCoder(b, tileSize, buffer)
		data := make([]byte, len(buffer.Bytes()))
		copy(data, buffer.Bytes())
		in, output := make(chan []byte, 1), &bytes.Buffer{}
		in <- data
		close(in)
		compress.BijectiveBurrowsWheelerCoder(in).MoveToFrontRunLengthCoder().AdaptiveCoder().Code(output)
		fmt.Printf("%.3f%% %7vb\n", 100*float64(output.Len())/float64(buffer.Len()), output.Len())
		red := FractalDecoder(buffer, tileSize)
		green := FractalDecoder(buffer, tileSize)
		blue := FractalDecoder(buffer, tileSize)
		decoded := image.NewRGBA(input.Bounds())
		width, height := input.Bounds().Max.X, input.Bounds().Max.Y
		for x := 0; x < width; x++ {
			for y := 0; y < height; y++ {
				r, _, _, _ := red.At(x, y).RGBA()
				g, _, _, _ := green.At(x, y).RGBA()
				b, _, _, _ := blue.At(x, y).RGBA()
				decoded.Set(x, y, color.RGBA{
					R: uint8(r >> 8),
					G: uint8(g >> 8),
					B: uint8(b >> 8),
					A: 0xFF})
			}
		}

		file, err = os.Create(name + "_color.png")
		if err != nil {
			log.Fatal(err)
		}

		err = png.Encode(file, decoded)
		if err != nil {
			log.Fatal(err)
		}
		file.Close()
	}

	fmt.Printf("%vb fcp file size\n", len(fcp))

	zpaq_test := func() (int, string) {
		zpaqFile := "fifo.zpaq"
		syscall.Mkfifo(zpaqFile, 0600)
		cmd := exec.Command("zp", "c3", zpaqFile, name+".fcp")
		cmd.Start()
		buffer, err := ioutil.ReadFile(zpaqFile)
		if err != nil {
			log.Fatal(err)
		}
		cmd.Wait()
		return len(buffer), "zp (command)"
	}

	compress_test := func() (int, string) {
		data := make([]byte, len(fcp))
		copy(data, fcp)
		in, buffer := make(chan []byte, 1), &bytes.Buffer{}
		in <- data
		close(in)
		compress.BijectiveBurrowsWheelerCoder(in).MoveToFrontRunLengthCoder().AdaptiveCoder().Code(buffer)
		return buffer.Len(), "github.com/pointlander/compress"
	}

	bzip2_test := func() (int, string) {
		cmd, buffer := exec.Command("bzip2", "--best"), &bytes.Buffer{}
		stdout, _ := cmd.StdoutPipe()
		stdin, _ := cmd.StdinPipe()
		cmd.Start()
		go func() {
			io.Copy(buffer, stdout)
		}()
		stdin.Write(fcp)
		stdin.Close()
		cmd.Wait()
		return buffer.Len(), "bzip2 (command)"
	}

	lzma_test := func() (int, string) {
		buffer := &bytes.Buffer{}
		writer := lzma.NewWriterLevel(buffer, lzma.BestCompression)
		writer.Write(fcp)
		writer.Close()
		return buffer.Len(), "github.com/kjk/lzma"
	}

	zlib_test := func() (int, string) {
		buffer := &bytes.Buffer{}
		writer, _ := zlib.NewWriterLevel(buffer, zlib.BestCompression)
		writer.Write(fcp)
		writer.Close()
		return buffer.Len(), "compress/zlib"
	}

	jpg_test := func() (int, string) {
		buffer := &bytes.Buffer{}
		jpeg.Encode(buffer, gray, nil)
		return buffer.Len(), "image/jpeg"
	}

	tests := []func() (int, string){zpaq_test, compress_test, bzip2_test, lzma_test, zlib_test, jpg_test}
	results := make([]struct {
		size int
		name string
	}, len(tests))
	for i, test := range tests {
		results[i].size, results[i].name = test()
		if i < 5 {
			fmt.Printf("%.3f%% %7vb %v\n", 100*float64(results[i].size)/float64(len(fcp)), results[i].size, results[i].name)
		}
	}

	fmt.Println("\n image compression comparisons")
	for _, result := range results {
		fmt.Printf("%.3f%% %7vb %v\n", 100*float64(result.size)/float64(width*height), result.size, result.name)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dashboard provides testing of the grafana dashboards used in Istio
// to provide mesh monitoring capabilities.
package dashboard

import (
	"bufio"
	"context"
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/prometheus/client_golang/api"
	"github.com/prometheus/client_golang/api/prometheus/v1"
	"github.com/prometheus/common/model"

	"istio.io/fortio/fhttp"
	"istio.io/fortio/periodic"
	"istio.io/istio/pkg/log"
	"istio.io/istio/tests/e2e/framework"
	"istio.io/istio/tests/util"
)

const (
	istioDashboard = "addons/grafana/dashboards/istio-dashboard.json"
	mixerDashboard = "addons/grafana/dashboards/mixer-dashboard.json"
	pilotDashboard = "addons/grafana/dashboards/pilot-dashboard.json"
	fortioYaml     = "tests/e2e/tests/dashboard/fortio-rules.yaml"
	netcatYaml     = "tests/e2e/tests/dashboard/netcat-rules.yaml"

	prometheusPort = "9090"
)

var (
	replacer = strings.NewReplacer(
		"$source_version", "unknown",
		"$source", "istio-ingress.*",
		"$http_destination", "echosrv.*",
		"$destination_version", "v1.*",
		"$adapter", "kubernetesenv",
		`connection_mtls=\"true\"`, "",
		`connection_mtls=\"false\"`, "",
		`\`, "",
	)

	tcpReplacer = strings.NewReplacer(
		"$tcp_destination", "netcat-srv.*",
		"istio-ingress", "netcat-client",
		"unknown", "v1",
	)

	tc *testConfig
)

func TestMain(m *testing.M) {
	flag.Parse()
	check(framework.InitLogging(), "cannot setup logging")
	check(setTestConfig(), "could not create TestConfig")
	tc.Cleanup.RegisterCleanable(tc)
	os.Exit(tc.RunTest(m))
}

func TestDashboards(t *testing.T) {
	cases := []struct {
		name      string
		dashboard string
		filter    func([]string) []string
	}{
		{"Istio", istioDashboard, func(queries []string) []string { return queries }},
		{"Mixer", mixerDashboard, mixerQueryFilterFn},
		{"Pilot", pilotDashboard, pilotQueryFilterFn},
	}

	for _, testCase := range cases {
		t.Run(testCase.name, func(t *testing.T) {
			queries, err := extractQueries(testCase.dashboard)
			if err != nil {
				t.Fatalf("Could not extract queries from dashboard: %v", err)
			}

			queries = testCase.filter(queries)

			t.Logf("For %s, checking %d queries.", testCase.name, len(queries))

			for _, query := range queries {
				modified := replaceGrafanaTemplates(query)
				value, err := tc.promAPI.Query(context.Background(), modified, time.Now())
				if err != nil {
					t.Errorf("Failure executing query (%s): %v", modified, err)
				}
				if value == nil {
					t.Errorf("Returned value should not be nil for '%s'", modified)
					continue
				}
				numSamples := 0
				switch v := value.(type) {
				case model.Vector:
					numSamples = v.Len()
				case *model.Scalar:
					numSamples = 1
				default:
					t.Logf("unknown metric value type: %T", v)
				}
				if numSamples == 0 {
					t.Errorf("Expected a metric value for '%s', found no samples: %#v", modified, value)
				}
			}
		})
	}
}

func sendTrafficToCluster(gateway string) (*fhttp.HTTPRunnerResults, error) {
	opts := fhttp.HTTPRunnerOptions{
		RunnerOptions: periodic.RunnerOptions{
			QPS:        10,
			Exactly:    60,
			NumThreads: 5,
			Out:        os.Stderr,
		},
		HTTPOptions: fhttp.HTTPOptions{
			URL: gateway + "/fortio/?status=404:10,503:15&size=1024:10,512:5",
		},
		AllowInitialErrors: true,
	}
	return fhttp.RunHTTPTest(&opts)
}

func generateTCPInCluster() error {
	ns := tc.Kube.Namespace
	ncPods, err := getPodList(ns, "app=netcat-client")
	if err != nil {
		return fmt.Errorf("could not get nc client pods: %v", err)
	}
	if len(ncPods) != 1 {
		return fmt.Errorf("bad number of pods returned for netcat client: %d", len(ncPods))
	}
	clientPod := ncPods[0]
	_, err = util.Shell("kubectl -n %s exec %s -c netcat -- sh -c \"echo 'test' | nc -vv -w5 -vv netcat-srv 44444\"", ns, clientPod)
	return err
}

func extractQueries(dashFile string) ([]string, error) {
	var queries []string
	dash, err := os.Open(util.GetResourcePath(dashFile))
	if err != nil {
		return queries, fmt.Errorf("could not open dashboard file: %v", err)
	}
	scanner := bufio.NewScanner(dash)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "expr") {
			query := strings.Trim(strings.TrimPrefix(strings.TrimSpace(line), "\"expr\": "), `",`)
			queries = append(queries, query)
		}
	}
	return queries, nil
}

func replaceGrafanaTemplates(orig string) string {
	out := replacer.Replace(orig)
	if strings.Contains(orig, "istio_tcp") {
		out = tcpReplacer.Replace(out)
	}
	return out
}

// There currently is no good way to inject failures into a running Mixer,
// nor to cause Mixers to become unhealthy and drop out of cluster membership.
// For now, we will filter out the queries related to those metrics.
//
// Issue: https://github.com/istio/istio/issues/4070
func mixerQueryFilterFn(queries []string) []string {
	filtered := make([]string, 0, len(queries))
	for _, query := range queries {
		if strings.Contains(query, "_rq_5xx") {
			continue
		}
		if strings.Contains(query, "_rq_4xx") {
			continue
		}
		if strings.Contains(query, "ejections") {
			continue
		}
		if strings.Contains(query, "retry") {
			continue
		}
		if strings.Contains(query, "DataLoss") {
			continue
		}
		if strings.Contains(query, "grpc_code!=") {
			continue
		}
		filtered = append(filtered, query)
	}
	return filtered
}

// There currently is no good way to inject failures into a running Pilot,
// nor to cause Pilots to become unhealthy and drop out of cluster membership.
// For now, we will filter out the queries related to those metrics.
//
// Issue: https://github.com/istio/istio/issues/4155
func pilotQueryFilterFn(queries []string) []string {
	filtered := make([]string, 0, len(queries))
	for _, query := range queries {
		if strings.Contains(query, "_rq_5xx") {
			continue
		}
		if strings.Contains(query, "_rq_4xx") {
			continue
		}
		if strings.Contains(query, "_membership_") {
			continue
		}
		filtered = append(filtered, query)
	}
	return filtered
}

func promAPI() (v1.API, error) {
	client, err := api.NewClient(api.Config{Address: fmt.Sprintf("http://localhost:%s", prometheusPort)})
	if err != nil {
		return nil, err
	}
	return v1.NewAPI(client), nil
}

type testConfig struct {
	*framework.CommonConfig
	promAPI v1.API
}

func (t *testConfig) Teardown() error {
	return nil
}

func setTestConfig() error {
	cc, err := framework.NewCommonConfig("dashboard_test")
	if err != nil {
		return err
	}
	tc = new(testConfig)
	tc.CommonConfig = cc
	hub := os.Getenv("FORTIO_HUB")
	tag := os.Getenv("FORTIO_TAG")
	image := hub + "/fortio:" + tag
	if hub == "" || tag == "" {
		image = "istio/fortio:latest" // TODO: change
	}
	log.Infof("Fortio hub %s tag %s -> image %s", hub, tag, image)
	services := []framework.App{
		{
			KubeInject:      true,
			AppYamlTemplate: util.GetResourcePath(fortioYaml),
			Template: &fortioTemplate{
				FortioImage: image,
			},
		},
		{
			KubeInject:      true,
			AppYamlTemplate: util.GetResourcePath(netcatYaml),
		},
	}
	for i := range services {
		tc.Kube.AppManager.AddApp(&services[i])
	}
	mp := newPromProxy(tc.Kube.Namespace)
	tc.Cleanup.RegisterCleanable(mp)
	return nil
}

func (t *testConfig) Setup() (err error) {
	if !util.CheckPodsRunning(tc.Kube.Namespace) {
		return fmt.Errorf("could not get all pods running")
	}

	pAPI, err := promAPI()
	if err != nil {
		return fmt.Errorf("could not build prometheus API client: %v", err)
	}
	t.promAPI = pAPI

	if err := generateTCPInCluster(); err != nil {
		return fmt.Errorf("generating TCP traffic failed: %v", err)
	}

	gateway, errGw := tc.Kube.Ingress()
	if errGw != nil {
		return errGw
	}
	if _, err := sendTrafficToCluster(gateway); err != nil {
		return fmt.Errorf("generating HTTP traffic failed: %v", err)
	}

	allowPrometheusSync()

	return nil
}

type promProxy struct {
	namespace        string
	portFwdProcesses []*os.Process
}

func newPromProxy(namespace string) *promProxy {
	return &promProxy{
		namespace: namespace,
	}
}

func (p *promProxy) portForward(labelSelector string, localPort string, remotePort string) error {
	var pod string
	var err error
	var proc *os.Process

	getName := fmt.Sprintf("kubectl -n %s get pod -l %s -o jsonpath='{.items[0].metadata.name}'", p.namespace, labelSelector)
	pod, err = util.Shell(getName)
	if err != nil {
		return err
	}
	log.Infof("%s pod name: %s", labelSelector, pod)

	log.Infof("Setting up %s proxy", labelSelector)
	portFwdCmd := fmt.Sprintf("kubectl port-forward %s %s:%s -n %s", strings.Trim(pod, "'"), localPort, remotePort, p.namespace)
	log.Info(portFwdCmd)
	if proc, err = util.RunBackground(portFwdCmd); err != nil {
		log.Errorf("Failed to port forward: %s", err)
		return err
	}
	p.portFwdProcesses = append(p.portFwdProcesses, proc)
	log.Infof("running %s port-forward in background, pid = %d", labelSelector, proc.Pid)
	return nil
}

func (p *promProxy) Setup() error {
	if !util.CheckPodsRunning(tc.Kube.Namespace) {
		return errors.New("could not establish port-forward to prometheus: pods not running")
	}

	return p.portForward("app=prometheus", prometheusPort, prometheusPort)
}

func (p *promProxy) Teardown() (err error) {
	log.Info("Cleaning up mixer proxy")
	for _, proc := range p.portFwdProcesses {
		err := proc.Kill()
		if err != nil {
			log.Errorf("Failed to kill port-forward process, pid: %d", proc.Pid)
		}
	}
	return
}

func check(err error, msg string) {
	if err != nil {
		log.Errorf("%s. Error %s", msg, err)
		os.Exit(-1)
	}
}

type fortioTemplate struct {
	FortioImage string
}

func getPodList(namespace string, selector string) ([]string, error) {
	pods, err := util.Shell("kubectl get pods -n %s -l %s -o jsonpath={.items[*].metadata.name}", namespace, selector)
	if err != nil {
		return nil, err
	}
	return strings.Split(pods, " "), nil
}

func allowPrometheusSync() {
	time.Sleep(1 * time.Minute)
}
<commit_msg>Address e2e test issues (#4212)<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dashboard provides testing of the grafana dashboards used in Istio
// to provide mesh monitoring capabilities.
package dashboard

import (
	"bufio"
	"context"
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/prometheus/client_golang/api"
	"github.com/prometheus/client_golang/api/prometheus/v1"
	"github.com/prometheus/common/model"

	"istio.io/fortio/fhttp"
	"istio.io/fortio/periodic"
	"istio.io/istio/pkg/log"
	"istio.io/istio/tests/e2e/framework"
	"istio.io/istio/tests/util"
)

const (
	istioDashboard = "addons/grafana/dashboards/istio-dashboard.json"
	mixerDashboard = "addons/grafana/dashboards/mixer-dashboard.json"
	pilotDashboard = "addons/grafana/dashboards/pilot-dashboard.json"
	fortioYaml     = "tests/e2e/tests/dashboard/fortio-rules.yaml"
	netcatYaml     = "tests/e2e/tests/dashboard/netcat-rules.yaml"

	prometheusPort = "9090"
)

var (
	replacer = strings.NewReplacer(
		"$source_version", "unknown",
		"$source", "istio-ingress.*",
		"$http_destination", "echosrv.*",
		"$destination_version", "v1.*",
		"$adapter", "kubernetesenv",
		`connection_mtls=\"true\"`, "",
		`connection_mtls=\"false\"`, "",
		`\`, "",
	)

	tcpReplacer = strings.NewReplacer(
		"$tcp_destination", "netcat-srv.*",
		"istio-ingress", "netcat-client",
		"unknown", "v1.*",
	)

	tc *testConfig
)

func TestMain(m *testing.M) {
	flag.Parse()
	check(framework.InitLogging(), "cannot setup logging")
	check(setTestConfig(), "could not create TestConfig")
	tc.Cleanup.RegisterCleanable(tc)
	os.Exit(tc.RunTest(m))
}

func TestDashboards(t *testing.T) {
	cases := []struct {
		name      string
		dashboard string
		filter    func([]string) []string
	}{
		{"Istio", istioDashboard, func(queries []string) []string { return queries }},
		{"Mixer", mixerDashboard, mixerQueryFilterFn},
		{"Pilot", pilotDashboard, pilotQueryFilterFn},
	}

	for _, testCase := range cases {
		t.Run(testCase.name, func(t *testing.T) {
			queries, err := extractQueries(testCase.dashboard)
			if err != nil {
				t.Fatalf("Could not extract queries from dashboard: %v", err)
			}

			queries = testCase.filter(queries)

			t.Logf("For %s, checking %d queries.", testCase.name, len(queries))

			for _, query := range queries {
				modified := replaceGrafanaTemplates(query)
				value, err := tc.promAPI.Query(context.Background(), modified, time.Now())
				if err != nil {
					t.Errorf("Failure executing query (%s): %v", modified, err)
				}
				if value == nil {
					t.Errorf("Returned value should not be nil for '%s'", modified)
					continue
				}
				numSamples := 0
				switch v := value.(type) {
				case model.Vector:
					numSamples = v.Len()
				case *model.Scalar:
					numSamples = 1
				default:
					t.Logf("unknown metric value type: %T", v)
				}
				if numSamples == 0 {
					t.Errorf("Expected a metric value for '%s', found no samples: %#v", modified, value)
				}
			}
		})
	}
}

func sendTrafficToCluster(gateway string) (*fhttp.HTTPRunnerResults, error) {
	opts := fhttp.HTTPRunnerOptions{
		RunnerOptions: periodic.RunnerOptions{
			QPS:        10,
			Exactly:    60,
			NumThreads: 5,
			Out:        os.Stderr,
		},
		HTTPOptions: fhttp.HTTPOptions{
			URL: gateway + "/fortio/?status=404:10,503:15&size=1024:10,512:5",
		},
		AllowInitialErrors: true,
	}
	return fhttp.RunHTTPTest(&opts)
}

func generateTCPInCluster() error {
	ns := tc.Kube.Namespace
	ncPods, err := getPodList(ns, "app=netcat-client")
	if err != nil {
		return fmt.Errorf("could not get nc client pods: %v", err)
	}
	if len(ncPods) != 1 {
		return fmt.Errorf("bad number of pods returned for netcat client: %d", len(ncPods))
	}
	clientPod := ncPods[0]
	_, err = util.Shell("kubectl -n %s exec %s -c netcat -- sh -c \"echo 'test' | nc -vv -w5 -vv netcat-srv 44444\"", ns, clientPod)
	return err
}

func extractQueries(dashFile string) ([]string, error) {
	var queries []string
	dash, err := os.Open(util.GetResourcePath(dashFile))
	if err != nil {
		return queries, fmt.Errorf("could not open dashboard file: %v", err)
	}
	scanner := bufio.NewScanner(dash)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "expr") {
			query := strings.Trim(strings.TrimPrefix(strings.TrimSpace(line), "\"expr\": "), `",`)
			queries = append(queries, query)
		}
	}
	return queries, nil
}

func replaceGrafanaTemplates(orig string) string {
	out := replacer.Replace(orig)
	if strings.Contains(orig, "istio_tcp") {
		out = tcpReplacer.Replace(out)
	}
	return out
}

// There currently is no good way to inject failures into a running Mixer,
// nor to cause Mixers to become unhealthy and drop out of cluster membership.
// For now, we will filter out the queries related to those metrics.
//
// Issue: https://github.com/istio/istio/issues/4070
func mixerQueryFilterFn(queries []string) []string {
	filtered := make([]string, 0, len(queries))
	for _, query := range queries {
		if strings.Contains(query, "_rq_5xx") {
			continue
		}
		if strings.Contains(query, "_rq_4xx") {
			continue
		}
		if strings.Contains(query, "ejections") {
			continue
		}
		if strings.Contains(query, "retry") {
			continue
		}
		if strings.Contains(query, "DataLoss") {
			continue
		}
		if strings.Contains(query, "grpc_code!=") {
			continue
		}
		filtered = append(filtered, query)
	}
	return filtered
}

// There currently is no good way to inject failures into a running Pilot,
// nor to cause Pilots to become unhealthy and drop out of cluster membership.
// For now, we will filter out the queries related to those metrics.
//
// Issue: https://github.com/istio/istio/issues/4155
func pilotQueryFilterFn(queries []string) []string {
	filtered := make([]string, 0, len(queries))
	for _, query := range queries {
		if strings.Contains(query, "_rq_5xx") {
			continue
		}
		if strings.Contains(query, "_rq_4xx") {
			continue
		}
		if strings.Contains(query, "_membership_") {
			continue
		}
		if strings.Contains(query, "discovery_errors") {
			continue
		}
		if strings.Contains(query, "update_failure") {
			continue
		}
		filtered = append(filtered, query)
	}
	return filtered
}

func promAPI() (v1.API, error) {
	client, err := api.NewClient(api.Config{Address: fmt.Sprintf("http://localhost:%s", prometheusPort)})
	if err != nil {
		return nil, err
	}
	return v1.NewAPI(client), nil
}

type testConfig struct {
	*framework.CommonConfig
	promAPI v1.API
}

func (t *testConfig) Teardown() error {
	return nil
}

func setTestConfig() error {
	cc, err := framework.NewCommonConfig("dashboard_test")
	if err != nil {
		return err
	}
	tc = new(testConfig)
	tc.CommonConfig = cc
	hub := os.Getenv("FORTIO_HUB")
	tag := os.Getenv("FORTIO_TAG")
	image := hub + "/fortio:" + tag
	if hub == "" || tag == "" {
		image = "istio/fortio:latest" // TODO: change
	}
	log.Infof("Fortio hub %s tag %s -> image %s", hub, tag, image)
	services := []framework.App{
		{
			KubeInject:      true,
			AppYamlTemplate: util.GetResourcePath(fortioYaml),
			Template: &fortioTemplate{
				FortioImage: image,
			},
		},
		{
			KubeInject:      true,
			AppYamlTemplate: util.GetResourcePath(netcatYaml),
		},
	}
	for i := range services {
		tc.Kube.AppManager.AddApp(&services[i])
	}
	mp := newPromProxy(tc.Kube.Namespace)
	tc.Cleanup.RegisterCleanable(mp)
	return nil
}

func (t *testConfig) Setup() (err error) {
	if !util.CheckPodsRunning(tc.Kube.Namespace) {
		return fmt.Errorf("could not get all pods running")
	}

	pAPI, err := promAPI()
	if err != nil {
		return fmt.Errorf("could not build prometheus API client: %v", err)
	}
	t.promAPI = pAPI

	gateway, errGw := tc.Kube.Ingress()
	if errGw != nil {
		return errGw
	}
	if _, err := sendTrafficToCluster(gateway); err != nil {
		return fmt.Errorf("generating HTTP traffic failed: %v", err)
	}

	if err := generateTCPInCluster(); err != nil {
		return fmt.Errorf("generating TCP traffic failed: %v", err)
	}

	allowPrometheusSync()

	return nil
}

type promProxy struct {
	namespace        string
	portFwdProcesses []*os.Process
}

func newPromProxy(namespace string) *promProxy {
	return &promProxy{
		namespace: namespace,
	}
}

func (p *promProxy) portForward(labelSelector string, localPort string, remotePort string) error {
	var pod string
	var err error
	var proc *os.Process

	getName := fmt.Sprintf("kubectl -n %s get pod -l %s -o jsonpath='{.items[0].metadata.name}'", p.namespace, labelSelector)
	pod, err = util.Shell(getName)
	if err != nil {
		return err
	}
	log.Infof("%s pod name: %s", labelSelector, pod)

	log.Infof("Setting up %s proxy", labelSelector)
	portFwdCmd := fmt.Sprintf("kubectl port-forward %s %s:%s -n %s", strings.Trim(pod, "'"), localPort, remotePort, p.namespace)
	log.Info(portFwdCmd)
	if proc, err = util.RunBackground(portFwdCmd); err != nil {
		log.Errorf("Failed to port forward: %s", err)
		return err
	}
	p.portFwdProcesses = append(p.portFwdProcesses, proc)
	log.Infof("running %s port-forward in background, pid = %d", labelSelector, proc.Pid)
	return nil
}

func (p *promProxy) Setup() error {
	if !util.CheckPodsRunning(tc.Kube.Namespace) {
		return errors.New("could not establish port-forward to prometheus: pods not running")
	}

	return p.portForward("app=prometheus", prometheusPort, prometheusPort)
}

func (p *promProxy) Teardown() (err error) {
	log.Info("Cleaning up mixer proxy")
	for _, proc := range p.portFwdProcesses {
		err := proc.Kill()
		if err != nil {
			log.Errorf("Failed to kill port-forward process, pid: %d", proc.Pid)
		}
	}
	return
}

func check(err error, msg string) {
	if err != nil {
		log.Errorf("%s. Error %s", msg, err)
		os.Exit(-1)
	}
}

type fortioTemplate struct {
	FortioImage string
}

func getPodList(namespace string, selector string) ([]string, error) {
	pods, err := util.Shell("kubectl get pods -n %s -l %s -o jsonpath={.items[*].metadata.name}", namespace, selector)
	if err != nil {
		return nil, err
	}
	return strings.Split(pods, " "), nil
}

func allowPrometheusSync() {
	time.Sleep(1 * time.Minute)
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package ws

import (
	"context"
	"errors"
	"net"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/dop251/goja"
	"github.com/gorilla/websocket"
	"github.com/loadimpact/k6/js/common"
	"github.com/loadimpact/k6/lib/metrics"
	"github.com/loadimpact/k6/lib/netext"
	"github.com/loadimpact/k6/stats"
)

type WS struct{}

type Socket struct {
	ctx           context.Context
	conn          *websocket.Conn
	eventHandlers map[string][]goja.Callable
	scheduled     chan goja.Callable
	done          chan struct{}
	shutdownOnce  sync.Once

	msgsSent, msgsReceived float64

	pingCounter       int
	pingStart         time.Time
	totalPingCount    float64
	totalPingDuration time.Duration
}

const writeWait = 10 * time.Second

var (
	newline = []byte{'\n'}
	space   = []byte{' '}
)

func (*WS) Connect(ctx context.Context, url string, args ...goja.Value) *http.Response {

	rt := common.GetRuntime(ctx)
	state := common.GetState(ctx)

	setupFn, userTags, header, err := parseArgs(rt, args)
	if err != nil {
		// Argument parsing errors should be passed to the user
		common.Throw(rt, err)
		return nil
	}

	tags := map[string]string{
		"url":   url,
		"group": state.Group.Path,
	}
	// Merge with the user-provided tags
	for k, v := range userTags {
		tags[k] = v
	}

	var bytesRead, bytesWritten int64
	// Pass a custom net.Dial function to websocket.Dialer that will substitute
	// the underlying net.Conn with our own tracked netext.Conn
	netDial := func(network, address string) (net.Conn, error) {
		var err error
		var conn net.Conn

		dialer := state.Dialer
		if conn, err = dialer.DialContext(ctx, network, address); err != nil {
			return nil, err
		}

		trackedConn := netext.TrackConn(conn, &bytesRead, &bytesWritten)

		return trackedConn, nil
	}

	wsd := websocket.Dialer{
		NetDial: netDial,
		Proxy:   http.ProxyFromEnvironment,
	}

	start := time.Now()
	conn, response, connErr := wsd.Dial(url, header)
	connectionEnd := time.Now()
	connectionDuration := stats.D(connectionEnd.Sub(start))

	socket := Socket{
		ctx:           ctx,
		conn:          conn,
		eventHandlers: make(map[string][]goja.Callable),
		scheduled:     make(chan goja.Callable),
		done:          make(chan struct{}),
	}

	// Run the user-provided set up function
	setupFn(goja.Undefined(), rt.ToValue(&socket))

	if connErr != nil {
		// Pass the error to the user script before exiting immediately
		socket.handleEvent("error", rt.ToValue(connErr))
		return response
	}
	defer conn.Close()

	tags["status"] = strconv.Itoa(response.StatusCode)
	tags["subprotocol"] = response.Header.Get("Sec-WebSocket-Protocol")

	// The connection is now open, emit the event
	socket.handleEvent("open")

	// Pass ping/pong events through the main control loop
	pingChan := make(chan interface{})
	pongChan := make(chan interface{})
	conn.SetPingHandler(func(string) error { pingChan <- "ping"; return nil })
	conn.SetPongHandler(func(string) error { pongChan <- "pong"; return nil })

	readDataChan := make(chan []byte)
	readErrChan := make(chan error)

	// Wraps a couple of channels around conn.ReadMessage
	go readPump(conn, readDataChan, readErrChan)

	// This is the main control loop. All JS code (including error handlers)
	// should only be executed by this thread to avoid race conditions
	for {
		select {
		case <-pingChan:
			// Handle pings received from the server
			socket.handleEvent("ping")

		case <-pongChan:
			// Handle pong responses to our pings
			socket.decPingCounter()
			socket.handleEvent("pong")

		case readData := <-readDataChan:
			socket.msgsReceived++
			socket.handleEvent("message", rt.ToValue(string(readData)))

		case readErr := <-readErrChan:
			socket.handleEvent("error", rt.ToValue(readErr))

		case scheduledFn := <-socket.scheduled:
			scheduledFn(goja.Undefined())

		case <-ctx.Done():
			// This means that K6 is shutting down (e.g., during an interrupt)
			socket.handleEvent("close", rt.ToValue("Interrupt"))
			socket.closeConnection(websocket.CloseGoingAway)

		case <-socket.done:
			// This is the final exit point normally triggered by closeConnection
			end := time.Now()
			sessionDuration := stats.D(end.Sub(start))

			avgPingRoundtrip := float64(socket.totalPingDuration) / socket.totalPingCount
			pingMetric := stats.D(time.Duration(avgPingRoundtrip))

			samples := []stats.Sample{
				{Metric: metrics.WSSessions, Time: end, Tags: tags, Value: 1},
				{Metric: metrics.WSMessagesSent, Time: end, Tags: tags, Value: socket.msgsReceived},
				{Metric: metrics.WSMessagesReceived, Time: end, Tags: tags, Value: socket.msgsSent},
				{Metric: metrics.WSConnecting, Time: end, Tags: tags, Value: connectionDuration},
				{Metric: metrics.WSSessionDuration, Time: end, Tags: tags, Value: sessionDuration},
				{Metric: metrics.WSPing, Time: end, Tags: tags, Value: pingMetric},
				{Metric: metrics.DataReceived, Time: end, Tags: tags, Value: float64(bytesRead)},
				{Metric: metrics.DataSent, Time: end, Tags: tags, Value: float64(bytesWritten)},
			}
			state.Samples = append(state.Samples, samples...)

			return response
		}
	}
}

func (s *Socket) On(event string, handler goja.Value) {
	if handler, ok := goja.AssertFunction(handler); ok {
		s.eventHandlers[event] = append(s.eventHandlers[event], handler)
	}
}

func (s *Socket) handleEvent(event string, args ...goja.Value) {
	if handlers, ok := s.eventHandlers[event]; ok {
		for _, handler := range handlers {
			handler(goja.Undefined(), args...)
		}
	}
}

func (s *Socket) Send(message string) {
	// NOTE: No binary message support for the time being since goja doesn't
	// support typed arrays.
	rt := common.GetRuntime(s.ctx)

	writeData := []byte(message)
	if err := s.conn.WriteMessage(websocket.TextMessage, writeData); err != nil {
		s.handleEvent("error", rt.ToValue(err))
	}

	s.msgsSent++
}

func (s *Socket) Ping() {
	rt := common.GetRuntime(s.ctx)
	deadline := time.Now().Add(writeWait)

	err := s.conn.WriteControl(websocket.PingMessage, []byte{}, deadline)
	if err != nil {
		s.handleEvent("error", rt.ToValue(err))
		return
	}

	s.incPingCounter()
}

func (s *Socket) incPingCounter() {
	s.pingCounter++
	s.totalPingCount++

	if s.pingCounter == 1 {
		s.pingStart = time.Now()
	}
}

func (s *Socket) decPingCounter() {
	s.pingCounter--

	if s.pingCounter == 0 {
		pingEnd := time.Now()
		s.totalPingDuration += pingEnd.Sub(s.pingStart)
	}
}

func (s *Socket) SetTimeout(fn goja.Callable, timeoutMs int) {
	// Starts a goroutine, blocks once on the timeout and pushes the callable
	// back to the main loop through the scheduled channel
	go func() {
		select {
		case <-time.After(time.Duration(timeoutMs) * time.Millisecond):
			s.scheduled <- fn

		case <-s.done:
			return
		}
	}()
}

func (s *Socket) SetInterval(fn goja.Callable, intervalMs int) {
	// Starts a goroutine, blocks forever on the ticker and pushes the callable
	// back to the main loop through the scheduled channel
	go func() {
		ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
		defer ticker.Stop()

		for {
			select {
			case <-ticker.C:
				s.scheduled <- fn

			case <-s.done:
				return
			}
		}
	}()
}

func (s *Socket) Close(args ...goja.Value) {
	code := websocket.CloseGoingAway
	if len(args) > 0 {
		code = int(args[0].ToInteger())
	}

	s.closeConnection(code)
}

// Attempts to close the websocket gracefully
func (s *Socket) closeConnection(code int) error {

	var err error
	s.shutdownOnce.Do(func() {
		rt := common.GetRuntime(s.ctx)

		err := s.conn.WriteControl(websocket.CloseMessage,
			websocket.FormatCloseMessage(code, ""),
			time.Now().Add(writeWait),
		)
		if err != nil {
			s.handleEvent("error", rt.ToValue(err))
			// Just call the handler, we'll try to close the connection anyway
		}
		s.conn.Close()

		// Stops the main control loop
		close(s.done)
	})

	return err
}

// Wraps conn.ReadMessage in a channel
func readPump(conn *websocket.Conn, readChan chan []byte, errorChan chan error) {
	defer conn.Close()

	for {
		_, message, err := conn.ReadMessage()
		if err != nil {
			// Only emit the error if we didn't close the socket ourselves
			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) {
				errorChan <- err
			}

			return
		}

		readChan <- message
	}
}

func parseArgs(rt *goja.Runtime, args []goja.Value) (goja.Callable, map[string]string, http.Header, error) {
	var callableV goja.Value
	var paramsV goja.Value

	// The params argument is optional
	switch len(args) {
	case 2:
		paramsV = args[0]
		callableV = args[1]
	case 1:
		paramsV = goja.Undefined()
		callableV = args[0]
	default:
		return nil, nil, nil, errors.New("Invalid number of arguments to ws.connect")
	}

	// Get the callable (required)
	var callable goja.Callable
	var isFunc bool
	if callable, isFunc = goja.AssertFunction(callableV); !isFunc {
		return nil, nil, nil, errors.New("Last argument to ws.connect must be a function")
	}

	// Leave header to nil by default so we can pass it directly to the Dialer
	var header http.Header
	tags := map[string]string{
		"status":      "0",
		"subprotocol": "",
	}

	if !goja.IsUndefined(paramsV) && !goja.IsNull(paramsV) {
		params := paramsV.ToObject(rt)
		for _, k := range params.Keys() {
			switch k {
			case "headers":
				header = http.Header{}
				headersV := params.Get(k)
				if goja.IsUndefined(headersV) || goja.IsNull(headersV) {
					continue
				}
				headersObj := headersV.ToObject(rt)
				if headersObj == nil {
					continue
				}
				for _, key := range headersObj.Keys() {
					header.Set(key, headersObj.Get(key).String())
				}
			case "tags":
				tagsV := params.Get(k)
				if goja.IsUndefined(tagsV) || goja.IsNull(tagsV) {
					continue
				}
				tagObj := tagsV.ToObject(rt)
				if tagObj == nil {
					continue
				}
				for _, key := range tagObj.Keys() {
					tags[key] = tagObj.Get(key).String()
				}
			}
		}

	}

	return callable, tags, header, nil
}
<commit_msg>Fixes WSPing metric<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package ws

import (
	"context"
	"errors"
	"net"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/dop251/goja"
	"github.com/gorilla/websocket"
	"github.com/loadimpact/k6/js/common"
	"github.com/loadimpact/k6/lib/metrics"
	"github.com/loadimpact/k6/lib/netext"
	"github.com/loadimpact/k6/stats"
)

type WS struct{}

type Socket struct {
	ctx           context.Context
	conn          *websocket.Conn
	eventHandlers map[string][]goja.Callable
	scheduled     chan goja.Callable
	done          chan struct{}
	shutdownOnce  sync.Once

	msgsSent, msgsReceived float64

	pingSendTimestamps map[string]time.Time
	pingSendCounter    int
	totalPingCount     float64
	totalPingDuration  time.Duration
}

const writeWait = 10 * time.Second

var (
	newline = []byte{'\n'}
	space   = []byte{' '}
)

func (*WS) Connect(ctx context.Context, url string, args ...goja.Value) *http.Response {
	rt := common.GetRuntime(ctx)
	state := common.GetState(ctx)

	setupFn, userTags, header, err := parseArgs(rt, args)
	if err != nil {
		// Argument parsing errors should be passed to the user
		common.Throw(rt, err)
		return nil
	}

	tags := map[string]string{
		"url":   url,
		"group": state.Group.Path,
	}
	// Merge with the user-provided tags
	for k, v := range userTags {
		tags[k] = v
	}

	var bytesRead, bytesWritten int64
	// Pass a custom net.Dial function to websocket.Dialer that will substitute
	// the underlying net.Conn with our own tracked netext.Conn
	netDial := func(network, address string) (net.Conn, error) {
		var err error
		var conn net.Conn

		dialer := state.Dialer
		if conn, err = dialer.DialContext(ctx, network, address); err != nil {
			return nil, err
		}

		trackedConn := netext.TrackConn(conn, &bytesRead, &bytesWritten)

		return trackedConn, nil
	}

	wsd := websocket.Dialer{
		NetDial: netDial,
		Proxy:   http.ProxyFromEnvironment,
	}

	start := time.Now()
	conn, response, connErr := wsd.Dial(url, header)
	connectionEnd := time.Now()
	connectionDuration := stats.D(connectionEnd.Sub(start))

	socket := Socket{
		ctx:                ctx,
		conn:               conn,
		eventHandlers:      make(map[string][]goja.Callable),
		pingSendTimestamps: make(map[string]time.Time),
		scheduled:          make(chan goja.Callable),
		done:               make(chan struct{}),
	}

	// Run the user-provided set up function
	setupFn(goja.Undefined(), rt.ToValue(&socket))

	if connErr != nil {
		// Pass the error to the user script before exiting immediately
		socket.handleEvent("error", rt.ToValue(connErr))
		return response
	}
	defer conn.Close()

	tags["status"] = strconv.Itoa(response.StatusCode)
	tags["subprotocol"] = response.Header.Get("Sec-WebSocket-Protocol")

	// The connection is now open, emit the event
	socket.handleEvent("open")

	// Pass ping/pong events through the main control loop
	pingChan := make(chan string)
	pongChan := make(chan string)
	conn.SetPingHandler(func(msg string) error { pingChan <- msg; return nil })
	conn.SetPongHandler(func(pingID string) error { pongChan <- pingID; return nil })

	readDataChan := make(chan []byte)
	readErrChan := make(chan error)

	// Wraps a couple of channels around conn.ReadMessage
	go readPump(conn, readDataChan, readErrChan)

	// This is the main control loop. All JS code (including error handlers)
	// should only be executed by this thread to avoid race conditions
	for {
		select {
		case <-pingChan:
			// Handle pings received from the server
			socket.handleEvent("ping")

		case pingID := <-pongChan:
			// Handle pong responses to our pings
			socket.trackPong(pingID)
			socket.handleEvent("pong")

		case readData := <-readDataChan:
			socket.msgsReceived++
			socket.handleEvent("message", rt.ToValue(string(readData)))

		case readErr := <-readErrChan:
			socket.handleEvent("error", rt.ToValue(readErr))

		case scheduledFn := <-socket.scheduled:
			scheduledFn(goja.Undefined())

		case <-ctx.Done():
			// This means that K6 is shutting down (e.g., during an interrupt)
			socket.handleEvent("close", rt.ToValue("Interrupt"))
			socket.closeConnection(websocket.CloseGoingAway)

		case <-socket.done:
			// This is the final exit point normally triggered by closeConnection
			end := time.Now()
			sessionDuration := stats.D(end.Sub(start))

			avgPingRoundtrip := float64(socket.totalPingDuration) / socket.totalPingCount
			pingMetric := stats.D(time.Duration(avgPingRoundtrip))

			samples := []stats.Sample{
				{Metric: metrics.WSSessions, Time: end, Tags: tags, Value: 1},
				{Metric: metrics.WSMessagesSent, Time: end, Tags: tags, Value: socket.msgsReceived},
				{Metric: metrics.WSMessagesReceived, Time: end, Tags: tags, Value: socket.msgsSent},
				{Metric: metrics.WSConnecting, Time: end, Tags: tags, Value: connectionDuration},
				{Metric: metrics.WSSessionDuration, Time: end, Tags: tags, Value: sessionDuration},
				{Metric: metrics.WSPing, Time: end, Tags: tags, Value: pingMetric},
				{Metric: metrics.DataReceived, Time: end, Tags: tags, Value: float64(bytesRead)},
				{Metric: metrics.DataSent, Time: end, Tags: tags, Value: float64(bytesWritten)},
			}
			state.Samples = append(state.Samples, samples...)

			return response
		}
	}
}

func (s *Socket) On(event string, handler goja.Value) {
	if handler, ok := goja.AssertFunction(handler); ok {
		s.eventHandlers[event] = append(s.eventHandlers[event], handler)
	}
}

func (s *Socket) handleEvent(event string, args ...goja.Value) {
	if handlers, ok := s.eventHandlers[event]; ok {
		for _, handler := range handlers {
			handler(goja.Undefined(), args...)
		}
	}
}

func (s *Socket) Send(message string) {
	// NOTE: No binary message support for the time being since goja doesn't
	// support typed arrays.
	rt := common.GetRuntime(s.ctx)

	writeData := []byte(message)
	if err := s.conn.WriteMessage(websocket.TextMessage, writeData); err != nil {
		s.handleEvent("error", rt.ToValue(err))
	}

	s.msgsSent++
}

func (s *Socket) Ping() {
	rt := common.GetRuntime(s.ctx)
	deadline := time.Now().Add(writeWait)
	pingID := strconv.Itoa(s.pingSendCounter)
	data := []byte(pingID)

	err := s.conn.WriteControl(websocket.PingMessage, data, deadline)
	if err != nil {
		s.handleEvent("error", rt.ToValue(err))
		return
	}

	s.pingSendTimestamps[pingID] = time.Now()
	s.pingSendCounter++
}

func (s *Socket) trackPong(pingID string) {
	pongTimestamp := time.Now()

	if _, ok := s.pingSendTimestamps[pingID]; !ok {
		// We received a pong for a ping we didn't send; ignore
		// (this shouldn't happen with a compliant server)
		return
	}
	pingTimestamp := s.pingSendTimestamps[pingID]

	s.totalPingDuration += pongTimestamp.Sub(pingTimestamp)
	s.totalPingCount++
}

func (s *Socket) SetTimeout(fn goja.Callable, timeoutMs int) {
	// Starts a goroutine, blocks once on the timeout and pushes the callable
	// back to the main loop through the scheduled channel
	go func() {
		select {
		case <-time.After(time.Duration(timeoutMs) * time.Millisecond):
			s.scheduled <- fn

		case <-s.done:
			return
		}
	}()
}

func (s *Socket) SetInterval(fn goja.Callable, intervalMs int) {
	// Starts a goroutine, blocks forever on the ticker and pushes the callable
	// back to the main loop through the scheduled channel
	go func() {
		ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
		defer ticker.Stop()

		for {
			select {
			case <-ticker.C:
				s.scheduled <- fn

			case <-s.done:
				return
			}
		}
	}()
}

func (s *Socket) Close(args ...goja.Value) {
	code := websocket.CloseGoingAway
	if len(args) > 0 {
		code = int(args[0].ToInteger())
	}

	s.closeConnection(code)
}

// Attempts to close the websocket gracefully
func (s *Socket) closeConnection(code int) error {

	var err error
	s.shutdownOnce.Do(func() {
		rt := common.GetRuntime(s.ctx)

		err := s.conn.WriteControl(websocket.CloseMessage,
			websocket.FormatCloseMessage(code, ""),
			time.Now().Add(writeWait),
		)
		if err != nil {
			s.handleEvent("error", rt.ToValue(err))
			// Just call the handler, we'll try to close the connection anyway
		}
		s.conn.Close()

		// Stops the main control loop
		close(s.done)
	})

	return err
}

// Wraps conn.ReadMessage in a channel
func readPump(conn *websocket.Conn, readChan chan []byte, errorChan chan error) {
	defer conn.Close()

	for {
		_, message, err := conn.ReadMessage()
		if err != nil {
			// Only emit the error if we didn't close the socket ourselves
			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) {
				errorChan <- err
			}

			return
		}

		readChan <- message
	}
}

func parseArgs(rt *goja.Runtime, args []goja.Value) (goja.Callable, map[string]string, http.Header, error) {
	var callableV goja.Value
	var paramsV goja.Value

	// The params argument is optional
	switch len(args) {
	case 2:
		paramsV = args[0]
		callableV = args[1]
	case 1:
		paramsV = goja.Undefined()
		callableV = args[0]
	default:
		return nil, nil, nil, errors.New("Invalid number of arguments to ws.connect")
	}

	// Get the callable (required)
	var callable goja.Callable
	var isFunc bool
	if callable, isFunc = goja.AssertFunction(callableV); !isFunc {
		return nil, nil, nil, errors.New("Last argument to ws.connect must be a function")
	}

	// Leave header to nil by default so we can pass it directly to the Dialer
	var header http.Header
	tags := map[string]string{
		"status":      "0",
		"subprotocol": "",
	}

	if !goja.IsUndefined(paramsV) && !goja.IsNull(paramsV) {
		params := paramsV.ToObject(rt)
		for _, k := range params.Keys() {
			switch k {
			case "headers":
				header = http.Header{}
				headersV := params.Get(k)
				if goja.IsUndefined(headersV) || goja.IsNull(headersV) {
					continue
				}
				headersObj := headersV.ToObject(rt)
				if headersObj == nil {
					continue
				}
				for _, key := range headersObj.Keys() {
					header.Set(key, headersObj.Get(key).String())
				}
			case "tags":
				tagsV := params.Get(k)
				if goja.IsUndefined(tagsV) || goja.IsNull(tagsV) {
					continue
				}
				tagObj := tagsV.ToObject(rt)
				if tagObj == nil {
					continue
				}
				for _, key := range tagObj.Keys() {
					tags[key] = tagObj.Get(key).String()
				}
			}
		}

	}

	return callable, tags, header, nil
}
<|endoftext|>


<commit_before>package goreddit

import (
	"encoding/json"
	"net/url"
)

type Comment struct {
	Author      string      `json:"author"`
	Text        string      `json:"body"`
	Title       string      `json:"title"`
	HtmlText    string      `json:"body_html"`
	Fullname    string      `json:"name"`
	Downvotes   int         `json:"downs"`
	Upvotes     int         `json:"ups"`
	Score       int         `json:"score"`
	ScoreHidden bool        `json:"score_hidden"`
	Gilded      int         `json:"gilded"`
	Parent      string      `json:"parent_id"`
	Edited      interface{} `json:"edited"` // Unknown type (bug?) It can be a float or a bool
	Archived    bool        `json:"archived"`
	Created     float64     `json:"created"`
	Replies     Reply       `json:"replies"`
}

type Reply struct {
	Data struct {
		Comments []struct {
			Comment Comment `json:"data"`
		} `json:"children"`
	}
}

// Comment send the text message as a reply to the parent message. The parent string
// must be a Reddit's fullname identifier
func (r *Reddit) Comment(parent string, text string) (err error) {
	form := url.Values{
		"api_type": {"json"},
		"text":     {text},
		"thing_id": {parent},
	}
	request, err := r.Request("POST", "/api/comment", form)
	if err != nil {
		return
	}

	_, err = r.JsonResponse(request)
	return
}

// ListCommentsSub lists the comments from all the links on a sub
func (r *Reddit) ListCommentsSub(sub string, options ListingOpt) (comments Reply, err error) {
	return r.ListComments(sub, "", options)
}

// ListComments lists the comments from a particular link of the sub
func (r *Reddit) ListComments(sub string, idlink string, options ListingOpt) (comments Reply, err error) {
	form := options.Values()
	request, err := r.Request("GET", "/r/"+sub+"/comments/"+idlink, form)
	if err != nil {
		return
	}

	data, err := r.JsonResponse(request)
	if err != nil {
		return
	}

	err = json.Unmarshal(data, &comments)
	if err != nil {
		if _, ok := err.(*json.UnmarshalTypeError); ok {
			// Silently ignore a UnmarshalTypeError :)
			err = nil
		} else {
			return
		}
	}

	return
}
<commit_msg>Adding voting feature<commit_after>package goreddit

import (
	"encoding/json"
	"fmt"
	"net/url"
)

const (
	Upvote   = 1
	Downvote = -1
	Unvote   = 0
)

type Comment struct {
	Author      string      `json:"author"`
	Text        string      `json:"body"`
	Title       string      `json:"title"`
	HtmlText    string      `json:"body_html"`
	Fullname    string      `json:"name"`
	Downvotes   int         `json:"downs"`
	Upvotes     int         `json:"ups"`
	Score       int         `json:"score"`
	ScoreHidden bool        `json:"score_hidden"`
	Gilded      int         `json:"gilded"`
	Parent      string      `json:"parent_id"`
	Edited      interface{} `json:"edited"` // Unknown type (bug?) It can be a float or a bool
	Archived    bool        `json:"archived"`
	Created     float64     `json:"created"`
	Replies     Reply       `json:"replies"`
}

type Reply struct {
	Data struct {
		Comments []struct {
			Comment Comment `json:"data"`
		} `json:"children"`
	}
}

// Comment send the text message as a reply to the parent message. The parent string
// must be a Reddit's fullname identifier
func (r *Reddit) Comment(parent string, text string) (err error) {
	form := url.Values{
		"api_type": {"json"},
		"text":     {text},
		"thing_id": {parent},
	}
	request, err := r.Request("POST", "/api/comment", form)
	if err != nil {
		return
	}

	_, err = r.JsonResponse(request)
	return
}

// ListCommentsSub lists the comments from all the links on a sub
func (r *Reddit) ListCommentsSub(sub string, options ListingOpt) (comments Reply, err error) {
	return r.ListComments(sub, "", options)
}

// ListComments lists the comments from a particular link of the sub
func (r *Reddit) ListComments(sub string, idlink string, options ListingOpt) (comments Reply, err error) {
	form := options.Values()
	request, err := r.Request("GET", "/r/"+sub+"/comments/"+idlink, form)
	if err != nil {
		return
	}

	data, err := r.JsonResponse(request)
	if err != nil {
		return
	}

	err = json.Unmarshal(data, &comments)
	if err != nil {
		if _, ok := err.(*json.UnmarshalTypeError); ok {
			// Silently ignore a UnmarshalTypeError :)
			err = nil
		} else {
			return
		}
	}

	return
}

// Vote takes a fullname (it can be a link or a comment) and cast a vote on it.
// Votes can be goreddit.Upvote, goreddit.Downvote and goreddit.Unvote.
// NOTE from Reddit: votes must be cast by humans. That is, API clients proxying a
// human's action one-for-one are OK, but bots deciding how to vote on content or
// amplifying a human's vote are not. See the reddit rules for more details on what
// constitutes vote cheating.
func (r *Reddit) Vote(fullname string, vote int) (err error) {
	form := url.Values{
		"id":  {fullname},
		"dir": {fmt.Sprintf("%d", vote)},
	}
	request, err := r.Request("POST", "/api/vote", form)
	if err != nil {
		return
	}
	_, err = r.JsonResponse(request)
	return
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"

	"k8s.io/apimachinery/pkg/api/resource"
	apiValidation "k8s.io/apimachinery/pkg/api/validation"
	utilValidation "k8s.io/apimachinery/pkg/util/validation"
)

func validateAnnotations(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, _ := range m {
		errors := utilValidation.IsQualifiedName(strings.ToLower(k))
		if len(errors) > 0 {
			for _, e := range errors {
				es = append(es, fmt.Errorf("%s (%q) %s", key, k, e))
			}
		}

		if isInternalKey(k) {
			es = append(es, fmt.Errorf("%s: %q is internal Kubernetes annotation", key, k))
		}
	}
	return
}

func validateName(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)

	errors := apiValidation.NameIsDNSLabel(v, false)
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validateGenerateName(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)

	errors := apiValidation.NameIsDNSLabel(v, true)
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validateLabels(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, v := range m {
		for _, msg := range utilValidation.IsQualifiedName(k) {
			es = append(es, fmt.Errorf("%s (%q) %s", key, k, msg))
		}
		val, isString := v.(string)
		if !isString {
			es = append(es, fmt.Errorf("%s.%s (%#v): Expected value to be string", key, k, v))
			return
		}
		for _, msg := range utilValidation.IsValidLabelValue(val) {
			es = append(es, fmt.Errorf("%s (%q) %s", key, val, msg))
		}
	}
	return
}

func validatePortNum(value interface{}, key string) (ws []string, es []error) {
	errors := utilValidation.IsValidPortNum(value.(int))
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validatePortName(value interface{}, key string) (ws []string, es []error) {
	errors := utilValidation.IsValidPortName(value.(string))
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}
func validatePortNumOrName(value interface{}, key string) (ws []string, es []error) {
	switch value.(type) {
	case string:
		intVal, err := strconv.Atoi(value.(string))
		if err != nil {
			return validatePortName(value, key)
		}
		return validatePortNum(intVal, key)
	case int:
		return validatePortNum(value, key)

	default:
		es = append(es, fmt.Errorf("%s must be defined of type string or int on the schema", key))
		return
	}
}

func validateResourceList(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, value := range m {
		if _, ok := value.(int); ok {
			continue
		}

		if v, ok := value.(string); ok {
			_, err := resource.ParseQuantity(v)
			if err != nil {
				es = append(es, fmt.Errorf("%s.%s (%q): %s", key, k, v, err))
			}
			continue
		}

		err := "Value can be either string or int"
		es = append(es, fmt.Errorf("%s.%s (%#v): %s", key, k, value, err))
	}
	return
}

func validateResourceQuantity(value interface{}, key string) (ws []string, es []error) {
	if v, ok := value.(string); ok {
		_, err := resource.ParseQuantity(v)
		if err != nil {
			es = append(es, fmt.Errorf("%s.%s : %s", key, v, err))
		}
	}
	return
}

func validatePositiveInteger(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v <= 0 {
		es = append(es, fmt.Errorf("%s must be greater than 0", key))
	}
	return
}

func validateDNSPolicy(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)
	if v != "ClusterFirst" && v != "Default" {
		es = append(es, fmt.Errorf("%s must be either ClusterFirst or Default", key))
	}
	return
}

func validateRestartPolicy(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)
	switch v {
	case "Always", "OnFailure", "Never":
		return
	default:
		es = append(es, fmt.Errorf("%s must be one of Always, OnFailure or Never ", key))
	}
	return
}

func validateTerminationGracePeriodSeconds(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v < 0 {
		es = append(es, fmt.Errorf("%s must be greater than or equal to 0", key))
	}
	return
}

func validateModeBits(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v < 0 || v > 0777 {
		es = append(es, fmt.Errorf("%s (%#o) expects octal notation (a value between 0 and 0777)", key, v))
	}
	return
}

func validateAttributeValueDoesNotContain(searchString string) schema.SchemaValidateFunc {
	return func(v interface{}, k string) (ws []string, errors []error) {
		input := v.(string)
		if strings.Contains(input, searchString) {
			errors = append(errors, fmt.Errorf(
				"%q must not contain %q",
				k, searchString))
		}
		return
	}
}

func validateAttributeValueIsIn(validValues []string) schema.SchemaValidateFunc {
	return func(v interface{}, k string) (ws []string, errors []error) {
		input := v.(string)
		isValid := false
		for _, s := range validValues {
			if s == input {
				isValid = true
				break
			}
		}
		if !isValid {
			errors = append(errors, fmt.Errorf(
				"%q must contain a value from %#v, got %q",
				k, validValues, input))
		}
		return

	}
}
<commit_msg>change name validator to DNS1123 subdomain<commit_after>package kubernetes

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"

	"k8s.io/apimachinery/pkg/api/resource"
	apiValidation "k8s.io/apimachinery/pkg/api/validation"
	utilValidation "k8s.io/apimachinery/pkg/util/validation"
)

func validateAnnotations(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, _ := range m {
		errors := utilValidation.IsQualifiedName(strings.ToLower(k))
		if len(errors) > 0 {
			for _, e := range errors {
				es = append(es, fmt.Errorf("%s (%q) %s", key, k, e))
			}
		}

		if isInternalKey(k) {
			es = append(es, fmt.Errorf("%s: %q is internal Kubernetes annotation", key, k))
		}
	}
	return
}

func validateName(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)
	errors := apiValidation.NameIsDNSSubdomain(v, false)
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validateGenerateName(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)

	errors := apiValidation.NameIsDNSLabel(v, true)
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validateLabels(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, v := range m {
		for _, msg := range utilValidation.IsQualifiedName(k) {
			es = append(es, fmt.Errorf("%s (%q) %s", key, k, msg))
		}
		val, isString := v.(string)
		if !isString {
			es = append(es, fmt.Errorf("%s.%s (%#v): Expected value to be string", key, k, v))
			return
		}
		for _, msg := range utilValidation.IsValidLabelValue(val) {
			es = append(es, fmt.Errorf("%s (%q) %s", key, val, msg))
		}
	}
	return
}

func validatePortNum(value interface{}, key string) (ws []string, es []error) {
	errors := utilValidation.IsValidPortNum(value.(int))
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}

func validatePortName(value interface{}, key string) (ws []string, es []error) {
	errors := utilValidation.IsValidPortName(value.(string))
	if len(errors) > 0 {
		for _, err := range errors {
			es = append(es, fmt.Errorf("%s %s", key, err))
		}
	}
	return
}
func validatePortNumOrName(value interface{}, key string) (ws []string, es []error) {
	switch value.(type) {
	case string:
		intVal, err := strconv.Atoi(value.(string))
		if err != nil {
			return validatePortName(value, key)
		}
		return validatePortNum(intVal, key)
	case int:
		return validatePortNum(value, key)

	default:
		es = append(es, fmt.Errorf("%s must be defined of type string or int on the schema", key))
		return
	}
}

func validateResourceList(value interface{}, key string) (ws []string, es []error) {
	m := value.(map[string]interface{})
	for k, value := range m {
		if _, ok := value.(int); ok {
			continue
		}

		if v, ok := value.(string); ok {
			_, err := resource.ParseQuantity(v)
			if err != nil {
				es = append(es, fmt.Errorf("%s.%s (%q): %s", key, k, v, err))
			}
			continue
		}

		err := "Value can be either string or int"
		es = append(es, fmt.Errorf("%s.%s (%#v): %s", key, k, value, err))
	}
	return
}

func validateResourceQuantity(value interface{}, key string) (ws []string, es []error) {
	if v, ok := value.(string); ok {
		_, err := resource.ParseQuantity(v)
		if err != nil {
			es = append(es, fmt.Errorf("%s.%s : %s", key, v, err))
		}
	}
	return
}

func validatePositiveInteger(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v <= 0 {
		es = append(es, fmt.Errorf("%s must be greater than 0", key))
	}
	return
}

func validateDNSPolicy(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)
	if v != "ClusterFirst" && v != "Default" {
		es = append(es, fmt.Errorf("%s must be either ClusterFirst or Default", key))
	}
	return
}

func validateRestartPolicy(value interface{}, key string) (ws []string, es []error) {
	v := value.(string)
	switch v {
	case "Always", "OnFailure", "Never":
		return
	default:
		es = append(es, fmt.Errorf("%s must be one of Always, OnFailure or Never ", key))
	}
	return
}

func validateTerminationGracePeriodSeconds(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v < 0 {
		es = append(es, fmt.Errorf("%s must be greater than or equal to 0", key))
	}
	return
}

func validateModeBits(value interface{}, key string) (ws []string, es []error) {
	v := value.(int)
	if v < 0 || v > 0777 {
		es = append(es, fmt.Errorf("%s (%#o) expects octal notation (a value between 0 and 0777)", key, v))
	}
	return
}

func validateAttributeValueDoesNotContain(searchString string) schema.SchemaValidateFunc {
	return func(v interface{}, k string) (ws []string, errors []error) {
		input := v.(string)
		if strings.Contains(input, searchString) {
			errors = append(errors, fmt.Errorf(
				"%q must not contain %q",
				k, searchString))
		}
		return
	}
}

func validateAttributeValueIsIn(validValues []string) schema.SchemaValidateFunc {
	return func(v interface{}, k string) (ws []string, errors []error) {
		input := v.(string)
		isValid := false
		for _, s := range validValues {
			if s == input {
				isValid = true
				break
			}
		}
		if !isValid {
			errors = append(errors, fmt.Errorf(
				"%q must contain a value from %#v, got %q",
				k, validValues, input))
		}
		return

	}
}
<|endoftext|>


<commit_before>package main
import "fmt"
import "strings"
import "bytes"
import "errors"
import "container/list"
import "regexp"

var regsrch *regexp.Regexp
var regsrch2 *regexp.Regexp

func compile(code string) (string, string, error) {
    lines := strings.Split(code, "\n")
    processed := list.New()
    /*- create imports -*/
    imp, err := extract("<?goimp", lines, processed)
    if err != nil {
        return "", "", err
    }
    imports := ""
    if len(imp) != 0 && strings.TrimSpace(imp) != "" {
        imports = "import (\n"+imp+"\n)\n"
    }
    /*- create functions  -*/
    fn, err := extract("<?gofn", lines, processed)
    if err != nil {
        return "", "", err
    }
    funcs := processfnExtract(fn)
    /*- create use  -*/
    uses, err := extract("<?gouse", lines, processed)
    if err != nil {
        return "", "", err
    }
    /*- create renderer  -*/
    renderer := createRenderer(lines, processed)
    randName := "Render"+genRandName()
    fullcode := "package main\n"+imports+uses+funcs+"func GokRender"+randName+
                "(gok *Gok){\n"+renderer+"\n}"
    return fullcode, randName, err
}

func processfnExtract(s string) string {
    if (len(s) == 0) || (strings.TrimSpace(s) == "") {
        return ""
    }
    lns := strings.Split(s, "\n")
    lns[1] = "func "+lns[1]
    return strings.Join(lns, "\n")
}

type analyzed struct {
    ln int
    col1 int
    col2 int
}

func containsln(processed *list.List, ln int) (col1, col2 int) {
    for e := processed.Front(); e != nil; e = e.Next() {
        lnsAndCols := e.Value.(*analyzed)
        if lnsAndCols.ln == ln {
            return lnsAndCols.col1, lnsAndCols.col2
        }
    }
    return -1, -1
}

func extract(pattern string, lines []string, processed *list.List) (string, error) {
    gocode := new(bytes.Buffer)
    plen := len(pattern)
    inside := false
    for i, l := range lines {
        lenl := len(l)
        if (lenl == 0) || (strings.TrimSpace(l) == "") {
            continue
        }
        if !inside {
            indx := strings.Index(l, pattern)
            if indx == -1 {
                continue
            }
            inside = true
            lnsAndCols := new(analyzed)
            lnsAndCols.ln = i
            if (lenl == plen) || (strings.TrimSpace(l[indx+plen:]) == "") {
                lnsAndCols.col1 = 0
                lnsAndCols.col2 = lenl
                processed.PushBack(lnsAndCols)
                continue
            }
            fmt.Fprintf(gocode, "//%d\n%s\n",(i+1), strings.TrimSpace(l[indx+plen:]))
            lnsAndCols.col1 = indx+plen
            lnsAndCols.col2 = lenl
            processed.PushBack(lnsAndCols)
        } else {
            lnsAndCols := new(analyzed)
            lnsAndCols.ln = i
            indx := strings.Index(l, "?>")
            if indx == -1 {
                fmt.Fprintf(gocode, "//%d\n%s\n", (i+1), strings.TrimSpace(l))
                lnsAndCols.col1 = 0
                lnsAndCols.col2 = lenl
                processed.PushBack(lnsAndCols)
                continue
            }
            inside = false;
            if lenl == 2 {
                lnsAndCols.col1 = 0
                lnsAndCols.col2 = lenl
                processed.PushBack(lnsAndCols)
                continue
            }
            if (indx == 0) || (strings.TrimSpace(l[:indx]) == "") {
                lnsAndCols.col1 = 0
                lnsAndCols.col1 = lenl
                processed.PushBack(lnsAndCols)
                continue
            }
            fmt.Fprintf(gocode, "//%d\n%s\n", (i+1), strings.TrimSpace(l[:indx]))
            lnsAndCols.col1 = 0
            lnsAndCols.col2 = indx+2
            processed.PushBack(lnsAndCols)
        }
    }
    if inside {
        return "", errors.New("syntax error "+pattern+" incomplete")
    }
    return gocode.String(), nil
}

func createRenderer(lines []string, processed *list.List) string {
    gocode := new(bytes.Buffer)
    processlnStatic := false
    for i, l := range lines {
        lenl := len(l)
        col1, col2 := containsln(processed, i)
        var i1, i2 int
        if (col1 == 0) && (col2 == lenl) {
            continue
        } else if (col1 == -1) && (col2 == -1) {
            i1 = 0
            i2 = lenl
        } else if (col1 == 0) {
            i1 = col2
            i2 = lenl
        } else if (col2 == lenl) {
            i1 = 0
            i2 = col1
        }
        slice := l[i1:i2]
        gocode.Write(processln(slice, i+1, &processlnStatic))
    }
    return gocode.String()
}

func processln(s string, ln int, static *bool) []byte {
    if regsrch == nil {
        var err error
        regsrch, err = regexp.Compile("\\<\\?go\\s?[^(fn)(vars)(imp)]")
        if err != nil {
            return []byte("")
        }
    }
    gocode := new(bytes.Buffer)
    slen := len(s)
    for last := 0; last < slen; {
        slice := s[last:]
        indx1 := -1
        indx1End := 0
        temp := regsrch.FindIndex([]byte(slice))
        if temp != nil {
            indx1 = temp[0]
            indx1End = temp[1]
        }
        indx2 := strings.Index(slice, "?>")
        if (indx1 == -1) && (indx2 == -1) {
            /*- determine if this is html or go code -*/
            if (*static) {
                fmt.Fprintf(gocode, "//%d\n%s\n", ln, slice)
            } else {
                fmt.Fprintf(gocode, "//%d\ngok.Echo(\"%s\")\n", ln, buildStr(slice))
            }
            break
        } else if ((indx1 < indx2) && (indx1 != -1)) || (indx2 == -1) {
            if indx2 == -1 {
                indx2 = len(slice)
                *static = true
            }
            if strings.TrimSpace(slice[:indx1]) != "" {
                fmt.Fprintf(gocode, "//%d\ngok.Echo(\"%s\")\n", ln, buildStr(slice[:indx1]))
            }
            fmt.Fprintf(gocode, "//%d\n%s\n", ln, slice[indx1End:indx2])
            last += (indx2+2)
        } else {
            if indx1 == -1 {
                indx1 = len(slice)
                indx1End = 5
                (*static) = false
            }
            fmt.Fprintf(gocode, "//%d\n%s\n", ln, buildStr(slice[:indx2]))
            fmt.Fprintf(gocode, "//%d\ngok.Echo(\"%s\")\n", ln, slice[indx2+2:indx1])
            last += (indx1End)

        }
    }
    return gocode.Bytes()
}

func buildStr(s string) string {
    s = strings.Replace(s, "\n", "\\n", -1)
    s = strings.Replace(s, "\t", "\\t", -1)
    s = strings.Replace(s, "\r", "\\r", -1)
    s = strings.Replace(s, "\v", "\\v", -1)
    s = strings.Replace(s, "\f", "\\f", -1)
    s = strings.Replace(s, "\"", "\\\"", -1)
    return strings.TrimSpace(s)
}

<commit_msg>standard<commit_after>package main
import "fmt"
import "strings"
import "bytes"
import "errors"
import "container/list"
import "regexp"

var regsrch *regexp.Regexp
var regsrch2 *regexp.Regexp

func compile(code string) (string, string, error) {
    lines := strings.Split(code, "\n")
    processed := list.New()
    /*- create imports -*/
    imp, err := extract("<?goimp", lines, processed)
    if err != nil {
        return "", "", err
    }
    imports := ""
    if len(imp) != 0 && strings.TrimSpace(imp) != "" {
        imports = "import (\n"+imp+"\n)\n"
    }
    /*- create functions  -*/
    fn, err := extract("<?gofn", lines, processed)
    if err != nil {
        return "", "", err
    }
    funcs := processfnExtract(fn)
    /*- create use  -*/
    uses, err := extract("<?gouse", lines, processed)
    if err != nil {
        return "", "", err
    }
    /*- create renderer  -*/
    renderer := createRenderer(lines, processed)
    randName := "Render"+genRandName()
    fullcode := "package main\n"+imports+uses+funcs+"func "+randName+
                "(gok *Gok){\n"+renderer+"\n}"
    return fullcode, randName, err
}

func processfnExtract(s string) string {
    if (len(s) == 0) || (strings.TrimSpace(s) == "") {
        return ""
    }
    lns := strings.Split(s, "\n")
    lns[1] = "func "+lns[1]
    return strings.Join(lns, "\n")
}

type analyzed struct {
    ln int
    col1 int
    col2 int
}

func containsln(processed *list.List, ln int) (col1, col2 int) {
    for e := processed.Front(); e != nil; e = e.Next() {
        lnsAndCols := e.Value.(*analyzed)
        if lnsAndCols.ln == ln {
            return lnsAndCols.col1, lnsAndCols.col2
        }
    }
    return -1, -1
}

func extract(pattern string, lines []string, processed *list.List) (string, error) {
    gocode := new(bytes.Buffer)
    plen := len(pattern)
    inside := false
    for i, l := range lines {
        lenl := len(l)
        if (lenl == 0) || (strings.TrimSpace(l) == "") {
            continue
        }
        if !inside {
            indx := strings.Index(l, pattern)
            if indx == -1 {
                continue
            }
            inside = true
            lnsAndCols := new(analyzed)
            lnsAndCols.ln = i
            if (lenl == plen) || (strings.TrimSpace(l[indx+plen:]) == "") {
                lnsAndCols.col1 = 0
                lnsAndCols.col2 = lenl
                processed.PushBack(lnsAndCols)
                continue
            }
            fmt.Fprintf(gocode, "//%d\n%s\n",(i+1), strings.TrimSpace(l[indx+plen:]))
            lnsAndCols.col1 = indx+plen
            lnsAndCols.col2 = lenl
            processed.PushBack(lnsAndCols)
        } else {
            lnsAndCols := new(analyzed)
            lnsAndCols.ln = i
            indx := strings.Index(l, "?>")
            if indx == -1 {
                fmt.Fprintf(gocode, "//%d\n%s\n", (i+1), strings.TrimSpace(l))
                lnsAndCols.col1 = 0
                lnsAndCols.col2 = lenl
                processed.PushBack(lnsAndCols)
                continue
            }
            inside = false;
            if lenl == 2 {
                lnsAndCols.col1 = 0
                lnsAndCols.col2 = lenl
                processed.PushBack(lnsAndCols)
                continue
            }
            if (indx == 0) || (strings.TrimSpace(l[:indx]) == "") {
                lnsAndCols.col1 = 0
                lnsAndCols.col1 = lenl
                processed.PushBack(lnsAndCols)
                continue
            }
            fmt.Fprintf(gocode, "//%d\n%s\n", (i+1), strings.TrimSpace(l[:indx]))
            lnsAndCols.col1 = 0
            lnsAndCols.col2 = indx+2
            processed.PushBack(lnsAndCols)
        }
    }
    if inside {
        return "", errors.New("syntax error "+pattern+" incomplete")
    }
    return gocode.String(), nil
}

func createRenderer(lines []string, processed *list.List) string {
    gocode := new(bytes.Buffer)
    processlnStatic := false
    for i, l := range lines {
        lenl := len(l)
        col1, col2 := containsln(processed, i)
        var i1, i2 int
        if (col1 == 0) && (col2 == lenl) {
            continue
        } else if (col1 == -1) && (col2 == -1) {
            i1 = 0
            i2 = lenl
        } else if (col1 == 0) {
            i1 = col2
            i2 = lenl
        } else if (col2 == lenl) {
            i1 = 0
            i2 = col1
        }
        slice := l[i1:i2]
        gocode.Write(processln(slice, i+1, &processlnStatic))
    }
    return gocode.String()
}

func processln(s string, ln int, static *bool) []byte {
    if regsrch == nil {
        var err error
        regsrch, err = regexp.Compile("(\\<\\?go$|\\<\\?go\\s)")
        if err != nil {
            return []byte("")
        }
    }
    gocode := new(bytes.Buffer)
    slen := len(s)
    for last := 0; last < slen; {
        slice := s[last:]
        indx1 := -1
        indx1End := 0
        temp := regsrch.FindIndex([]byte(slice))
        if temp != nil {
            indx1 = temp[0]
            indx1End = temp[1]
        }
        indx2 := strings.Index(slice, "?>")
        if (indx1 == -1) && (indx2 == -1) {
            /*- determine if this is html or go code -*/
            if (*static) {
                fmt.Fprintf(gocode, "//%d\n%s\n", ln, slice)
            } else {
                fmt.Fprintf(gocode, "//%d\ngok.Echo(\"%s\")\n", ln, buildStr(slice))
            }
            break
        } else if ((indx1 < indx2) && (indx1 != -1)) || (indx2 == -1) {
            if indx2 == -1 {
                indx2 = len(slice)
                *static = true
            }
            if strings.TrimSpace(slice[:indx1]) != "" {
                fmt.Fprintf(gocode, "//%d\ngok.Echo(\"%s\")\n", ln, buildStr(slice[:indx1]))
            }
            fmt.Fprintf(gocode, "//%d\n%s\n", ln, slice[indx1End:indx2])
            last += (indx2+2)
        } else {
            if indx1 == -1 {
                indx1 = len(slice)
                indx1End = 5
                (*static) = false
            }
            fmt.Fprintf(gocode, "//%d\n%s\n", ln, buildStr(slice[:indx2]))
            fmt.Fprintf(gocode, "//%d\ngok.Echo(\"%s\")\n", ln, slice[indx2+2:indx1])
            last += (indx1End)

        }
    }
    return gocode.Bytes()
}

func buildStr(s string) string {
    s = strings.Replace(s, "\n", "\\n", -1)
    s = strings.Replace(s, "\t", "\\t", -1)
    s = strings.Replace(s, "\r", "\\r", -1)
    s = strings.Replace(s, "\v", "\\v", -1)
    s = strings.Replace(s, "\f", "\\f", -1)
    s = strings.Replace(s, "\"", "\\\"", -1)
    return strings.TrimSpace(s)
}

<|endoftext|>