<commit_before>package domain

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strings"
	"text/tabwriter"

	"github.com/spf13/cobra"
	"github.com/runabove/sail/internal"
)

var usageList = "Invalid usage. sail service domain list <application-name>/<service-name>. Please see sail domain list --help"

var cmdDomainList = &cobra.Command{
	Use:     "list",
	Short:   "List domains on the HTTP load balancer: sail service domain list <application-name>/<service-id>",
	Aliases: []string{"ls", "ps"},
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 1 {
			fmt.Fprintln(os.Stderr, usageList)
		} else {
			domainList(args[0])
		}
	},
}

func domainList(serviceID string) {

	t := strings.Split(serviceID, "/")
	if len(t) != 2 {
		fmt.Fprintln(os.Stderr, usageList)
		return
	}

	b := internal.ReqWant("GET", http.StatusOK, fmt.Sprintf("/applications/%s/services/%s/attached-routes", t[0], t[1]), nil)
	internal.FormatOutput(b, domainListFormatter)
}

func domainListFormatter(data []byte) {
	var routes []map[string]interface{}
	internal.Check(json.Unmarshal(data, &routes))

	w := tabwriter.NewWriter(os.Stdout, 30, 1, 3, ' ', 0)
	titles := []string{"DOMAIN", "METHOD", "PATTERN"}
	fmt.Fprintln(w, strings.Join(titles, "\t"))

	for _, route := range routes {
		fmt.Fprintf(w, "%s\t%s\t%s\n", route["domain"], route["method"], route["pattern"])
		w.Flush()
	}
}
<commit_msg>feat: list all domains in domain list<commit_after>package domain

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strings"
	"text/tabwriter"

	"github.com/runabove/sail/internal"
	"github.com/spf13/cobra"
)

var usageList = "Invalid usage. sail service domain list <application-name>/<service-name>. Please see sail domain list --help"
var domainHeadersDone = false

var cmdDomainList = &cobra.Command{
	Use:     "list",
	Short:   "List domains on the HTTP load balancer: sail service domain list [<application-name>[/<service-id>]]",
	Aliases: []string{"ls", "ps"},
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) > 1 {
			fmt.Fprintln(os.Stderr, usageList)
			os.Exit(1)
		}

		namespace := ""
		service := ""

		// Parse namespace / service
		if len(args) >= 1 {
			t := strings.Split(args[0], "/")
			namespace = t[0]
			if len(t) >= 2 {
				service = t[1]
			} else if len(t) > 2 {
				fmt.Fprintln(os.Stderr, usageList)
				return
			}
		}

		domainList(namespace, service)
	},
}

func domainList(namespace, service string) {
	var apps []string

	// TODO: rewrite whithout the m(n+1)+1... (needs API)
	if len(namespace) > 0 {
		apps = append(apps, namespace)
	} else {
		apps = internal.GetListApplications(nil)
	}

	for _, namespace := range apps {
		domainListNamespace(namespace, service)
	}
}

func domainListNamespace(namespace, service string) {
	var services []string

	// TODO: rewrite whithout the m(n+1)+1... (needs API)
	if len(service) > 0 {
		services = append(services, service)
	} else {
		b := internal.ReqWant("GET", http.StatusOK, fmt.Sprintf("/applications/%s/services", namespace), nil)
		internal.Check(json.Unmarshal(b, &services))
	}

	for _, service := range services {
		domainListService(namespace, service)
	}
}

func domainListService(namespace, service string) {
	b := internal.ReqWant("GET", http.StatusOK, fmt.Sprintf("/applications/%s/services/%s/attached-routes", namespace, service), nil)
	internal.FormatOutput(b, domainListFormatter)
}

func domainListFormatter(data []byte) {
	var routes []map[string]interface{}
	internal.Check(json.Unmarshal(data, &routes))

	w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)

	// below this: horrible hack. Do I feel ashamed: Yes.
	if !domainHeadersDone {
		titles := []string{"APP", "SERVICE", "DOMAIN", "METHOD", "PATTERN"}
		fmt.Fprintln(w, strings.Join(titles, "\t"))
		domainHeadersDone = true
	}

	for _, route := range routes {
		app := route["namespace"]
		service := route["service"]

		if app == nil {
			app = "-"
		}
		if service == nil {
			service = "-"
		}

		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", app, service, route["domain"], route["method"], route["pattern"])
		w.Flush()
	}
}
<|endoftext|>


<commit_before>package golgoquery

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/abhishekkr/gol/golhttpclient"
)

var (
	CacheDir = "/tmp/.tune.cli"
)

func createCache(cachePath string, cacheContent string) {
	cacheDir := path.Dir(cachePath)
	if _, err := os.Stat(cacheDir); os.IsNotExist(err) {
		log.Printf("[warn] cannot find default CacheDir(%s), creating cachedir", CacheDir)
		os.Mkdir(CacheDir, 0777)
	}

	err := ioutil.WriteFile(cachePath, []byte(cacheContent), 0644)
	if err != nil {
		log.Printf("[warn] cannot create cache (%s), creating cachedir", cachePath)
	}
}

func readCache(cachePath string) string {
	dat, err := ioutil.ReadFile(cachePath)
	if err != nil {
		log.Printf("[warn] missing cache file %s", cachePath)
	}
	return string(dat)
}

func urlToFilename(url string) string {
	var replacer = strings.NewReplacer(" ", "-",
		"\t", "-",
		"%", "-",
		"?", "-",
		"=", "-",
		"\\", "-",
		"/", "-",
		":", "-",
	)
	return replacer.Replace(url)
}

func CacheUrl(url string) (*goquery.Document, error) {
	urlFile := urlToFilename(url)
	cachePath := fmt.Sprintf("%s%s%s", CacheDir, string(filepath.Separator), urlFile)

	if _, err := os.Stat(cachePath); os.IsNotExist(err) {
		body, err := golhttpclient.HttpGet(url, map[string]string{}, map[string]string{})
		if err != nil {
			log.Fatalf("not able to fetch %s", url)
		}
		createCache(cachePath, body)
	}

	cache := readCache(cachePath)

	return goquery.NewDocumentFromReader(strings.NewReader(cache))
}
<commit_msg>golgoquery got cache<commit_after>package golgoquery

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/abhishekkr/gol/golhttpclient"
)

var (
	ReloadCache bool
	CacheDir    = "/tmp/.tune.cli"
	UserAgent   = "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0"
)

func createCache(cachePath string, cacheContent string) {
	cacheDir := path.Dir(cachePath)
	if _, err := os.Stat(cacheDir); os.IsNotExist(err) {
		log.Printf("[warn] cannot find default CacheDir(%s), creating cachedir", CacheDir)
		os.Mkdir(CacheDir, 0777)
	}

	err := ioutil.WriteFile(cachePath, []byte(cacheContent), 0644)
	if err != nil {
		log.Printf("[warn] cannot create cache (%s), creating cachedir", cachePath)
	}
}

func readCache(cachePath string) string {
	dat, err := ioutil.ReadFile(cachePath)
	if err != nil {
		log.Printf("[warn] missing cache file %s", cachePath)
	}
	return string(dat)
}

func cleanCache(cachePath string) {
	var err = os.Remove(cachePath)
	if err != nil {
		log.Printf("[warn] failed cleaning %s", cachePath)
	}
}

func urlToFilename(url string) string {
	var replacer = strings.NewReplacer(" ", "-",
		"\t", "-",
		"%", "-",
		"?", "-",
		"=", "-",
		"\\", "-",
		"/", "-",
		":", "-",
	)
	return replacer.Replace(url)
}

func CacheUrl(url string) (*goquery.Document, error) {
	urlFile := urlToFilename(url)
	cachePath := fmt.Sprintf("%s%s%s", CacheDir, string(filepath.Separator), urlFile)

	if ReloadCache {
		log.Println("[warn] cleaning cache ", cachePath)
		cleanCache(cachePath)
	}

	if _, err := os.Stat(cachePath); os.IsNotExist(err) {
		headers := map[string]string{
			"User-Agent": UserAgent,
		}
		log.Println("[info] fetching ", url)
		body, err := golhttpclient.HttpGet(url, map[string]string{}, headers)
		if err != nil {
			log.Fatalf("not able to fetch %s", url)
		}
		createCache(cachePath, body)
	}

	cache := readCache(cachePath)

	return goquery.NewDocumentFromReader(strings.NewReader(cache))
}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
    "time"
	"strings"
)

var conn dbox.IConnection

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

var (
    pcs = toolkit.M{}
    ccs = toolkit.M{}
    ledgers = toolkit.M{}
    prods = toolkit.M{}
    custs = toolkit.M{}
	plmodels = toolkit.M{}
	brands = toolkit.M{}
	ratios = map[string][]gdrj.SalesRatio{}
)

func getCursor(obj orm.IModel)dbox.ICursor{
    c, e := gdrj.Find(obj,nil,nil)
    if e!=nil{
        return nil
    }
    return c
}

func prepMaster(){
    pc:=new(gdrj.ProfitCenter)
    cc:=new(gdrj.CostCenter)
    prod:=new(gdrj.Product)
    ledger:=new(gdrj.LedgerMaster)

    cpc := getCursor(pc)
    defer cpc.Close()
    var e error
    for e=cpc.Fetch(pc,1,false);e==nil;{
        pcs.Set(pc.ID,pc)
        pc =new(gdrj.ProfitCenter)
        e=cpc.Fetch(pc,1,false)
    }

    ccc:=getCursor(cc)
    defer ccc.Close()
    for e=ccc.Fetch(cc,1,false);e==nil;{
        ccs.Set(cc.ID,cc)
        cc = new(gdrj.CostCenter)
        e=ccc.Fetch(cc,1,false)
    }

    cprod:=getCursor(prod)
    defer cprod.Close()
    for e=cprod.Fetch(prod,1,false);e==nil;{
        prods.Set(prod.ID,prod)
        prod=new(gdrj.Product)
        e=cprod.Fetch(prod,1,false)
    }

    cledger:=getCursor(ledger)
    defer cledger.Close()
    for e=cledger.Fetch(ledger,1,false);e==nil;{
        ledgers.Set(ledger.ID,ledger)
        ledger=new(gdrj.LedgerMaster)
        e=cledger.Fetch(ledger,1,false)
    }

    cust := new(gdrj.Customer)
    ccust:=getCursor(cust)
    defer ccust.Close()
    for e=ccust.Fetch(cust,1,false);e==nil;{
        custs.Set(cust.ID,cust)
        cust=new(gdrj.Customer)
        e=ccust.Fetch(cust,1,false)
    }

	plmodel := new(gdrj.PLModel)
	cplmodel := getCursor(plmodel)
	defer cplmodel.Close()
	for e=cplmodel.Fetch(plmodel,1,false);e==nil;{
		plmodels.Set(plmodel.ID,plmodel)
		plmodel=new(gdrj.PLModel)
		e=cplmodel.Fetch(plmodel,1,false)
	}

	toolkit.Println("--> Brand")
	brand := new(gdrj.HBrandCategory)
	cbrand := getCursor(plmodel)
	defer cbrand.Close()
	for e=cbrand.Fetch(brand,1,false);e==nil;{
		brands.Set(brand.ID,brand)
		brand=new(gdrj.HBrandCategory)
		e=cbrand.Fetch(brand,1,false)
	}

	toolkit.Println("--> Sales Ratio")
	ratio := new(gdrj.SalesRatio)
	cratios := getCursor(ratio)
	defer cratios.Close()
	for {
		efetch := cratios.Fetch(ratio, 1, false)
		if efetch != nil {
			break
		}
		ratioid := toolkit.Sprintf("%d_%d_%s", ratio.Year, ratio.Month, ratio.BranchID)
		a, exist := ratios[ratioid]
		if !exist {
			a = []gdrj.SalesRatio{}
		}
		a=append(a, *ratio)
		ratio = new(gdrj.SalesRatio)
		ratios[ratioid] = a
	}
}

func main() {
	//runtime.GOMAXPROCS(runtime.NumCPU())
	setinitialconnection()
	defer gdrj.CloseDb()

    toolkit.Println("Reading Master")
    prepMaster()

	pldm := new(gdrj.PLDataModel)
	toolkit.Println("Delete existing")
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","30052016SAP_EXPORT")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_FREIGHT")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_SUSEMI")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_APINTRA")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","30052016SAP_SGAPL")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_MEGASARI")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_SALESRD")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_DISC-RDJKT")).Delete().Exec(nil)

    toolkit.Println("START...")

	//for i, src := range arrstring {
	//dbf := dbox.Contains("src", src)
	crx, err := gdrj.Find(new(gdrj.RawDataPL), nil, toolkit.M{})
	if err != nil {
		toolkit.Println("Error Found : ", err.Error())
		os.Exit(1)
	}

    count := crx.Count()

	jobs := make(chan *gdrj.RawDataPL, count)
	result := make(chan string, count)

	for wi:=1;wi<10;wi++{
		go worker(wi, jobs, result)
	}

	t0 := time.Now()
	ci := 0
	iseof := false
	for !iseof {
		arrpl := []*gdrj.RawDataPL{}
		e := crx.Fetch(&arrpl, 1000, false)
		if e!=nil{
			iseof=true
			break
		}

		for _, v := range arrpl {
			jobs <- v
			ci++
		}

		toolkit.Printfn("Processing %d of %d in %s", ci, count, time.Since(t0).String())

		if len(arrpl) < 1000 {
			iseof = true
		}
	}

	toolkit.Println("Saving")
	step := count / 100
	limit := step
	for ri := 0; ri < count; ri++ {
		<-result
		if ri >= limit {
			toolkit.Printfn("Saving %d of %d (%dpct) in %s", ri, count, ri*100/count,
				time.Since(t0).String())
			limit += step
		}
	}
	toolkit.Printfn("Done %s", time.Since(t0).String())
}

var pldatas = map[string]*gdrj.PLDataModel{}

func worker(wi int, jobs <-chan *gdrj.RawDataPL, result chan<- string){
	workerconn, err := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

		for v:= range jobs{
			if v.Src=="31052016SAP_SALESRD" || v.Src=="31052016SAP_DISC-RDJKT" || v.Src==""{
				result <- "NOK"
				continue
			}

			tdate := time.Date(v.Year, time.Month(v.Period), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 3, 0)

			ls := new(gdrj.PLDataModel)
			ls.CompanyCode = v.EntityID
			//ls.LedgerAccount = v.Account

			ls.Year = tdate.Year()
			ls.Month = int(tdate.Month())
			ls.Date = gdrj.NewDate(ls.Year, ls.Month, 1)

			ls.PCID = v.PCID
			if v.PCID != "" && pcs.Has(v.PCID) {
				ls.PC = pcs.Get(v.PCID).(*gdrj.ProfitCenter)
			}

			ls.CCID = v.CCID
			if v.CCID != "" && ccs.Has(v.CCID) {
				ls.CC = ccs.Get(v.CCID).(*gdrj.CostCenter)
			}

			ls.OutletID = v.OutletID
			if v.OutletID != "" && custs.Has(v.OutletID) {
				ls.Customer = custs.Get(v.OutletID).(*gdrj.Customer)
				//ls.Customer = gdrj.CustomerGetByID(v.OutletID)
			} else {
				c := new(gdrj.Customer)
				c.Name = v.OutletName
				c.BranchID = v.BusA
				c.ChannelID = "I3"
				c.ChannelName = "MT"
				c.CustType = "EXP"
				c.CustomerGroup = "EXP"
				c.Zone = "EXP"
				c.Region = "EXP"
				c.National = "EXP"
				c.AreaName = "EXP"
				c.CustomerGroupName = "Export"
				ls.Customer = c
			}

			ls.SKUID = v.SKUID
			if v.SKUID != "" && prods.Has(v.SKUID) {
				ls.Product = prods.Get(v.SKUID).(*gdrj.Product)
			} else if v.SKUID!="" {
				ls.Product = new(gdrj.Product)
				ls.Product.Name = v.ProductName
				ls.Product.BrandCategoryID = v.PCID[4:]
				if brands.Has(ls.Product.BrandCategoryID){
					ls.Product.Brand = brands.Get(ls.Product.BrandCategoryID).(*gdrj.HBrandCategory).BrandID
				} else {
					ls.Product.BrandCategoryID = "Common"
					ls.Product.Brand = "-"
				}
			}

			ls.Value1 = v.AmountinIDR
			//ls.Value2 = v.AmountinUSD

			tLedgerAccount := new(gdrj.LedgerMaster)
			if ledgers.Has(v.Account){
				tLedgerAccount = ledgers.Get(v.Account).(*gdrj.LedgerMaster)
			}
			if tLedgerAccount.PLCode==""{
				plm := plmodels.Get("PL34").(*gdrj.PLModel)
				ls.PLCode = plm.ID
				ls.PLOrder = plm.OrderIndex
				ls.PLGroup1 = plm.PLHeader1
				ls.PLGroup2 = plm.PLHeader2
				ls.PLGroup3 = plm.PLHeader3
			} else if v.Src=="30052016SAP_EXPORT"{
				plm := plmodels.Get("PL6").(*gdrj.PLModel)
				ls.PLCode = plm.ID
				ls.PLOrder = plm.OrderIndex
				ls.PLGroup1 = plm.PLHeader1
				ls.PLGroup2 = plm.PLHeader2
				ls.PLGroup3 = plm.PLHeader3
			} else  {
				ls.PLCode = tLedgerAccount.PLCode
				ls.PLOrder = tLedgerAccount.OrderIndex
				ls.PLGroup1 = tLedgerAccount.H1
				ls.PLGroup2 = tLedgerAccount.H2
				ls.PLGroup3 = tLedgerAccount.H3
			}

			ls.Date = gdrj.NewDate(ls.Year, int(ls.Month), 1)

			sources := strings.Split(v.Src,"_")
			if len(sources)==1{
				ls.Source = sources[1]
			} else if len(sources)>1{
				ls.Source = sources[1]
			} else {
				ls.Source="OTHER"
			}

			rs := []gdrj.SalesRatio{}
			if v.Src!="30052016SAP_EXPORT"{
				srid := toolkit.Sprintf("%d_%d_%s", ls.Year, ls.Month, ls.Customer.BranchID)
				a, exists := ratios[srid]
				if exists{
					rs=a
				}
			}

			if len(rs)==0{
				r := new(gdrj.SalesRatio)
				r.Year = ls.Year
				r.Month = ls.Month
				r.Ratio = 1
				rs = append(rs, *r)
			}

			total := float64(0)
			for _, r := range rs{
				total += r.Ratio
			}

			for _, r := range rs{
				ls.ID = ls.PrepareID().(string)
				rls, lsexist := pldatas[ls.ID]
				multiplier:=float64(1)
				if v.Src!="30052016SAP_EXPORT"{
					multiplier=-1
				}

				if !lsexist{
					rls = new(gdrj.PLDataModel)
					*rls = *ls
					els := new(gdrj.PLDataModel)
					cls,_ := workerconn.NewQuery().From(ls.TableName()).
						Where(dbox.Eq("_id",rls.ID)).Cursor(nil)
					ecls:=cls.Fetch(els,1,false)
					if ecls==nil{
						rls.Value1=els.Value1
					}
					cls.Close()
				}

				rls.Value1 += ls.Value1 * r.Ratio/total * multiplier
				err = workerconn.NewQuery().From(ls.TableName()).Save().Exec(toolkit.M{}.Set("data",rls))
				if err != nil {
					toolkit.Println("Error Found : ", err.Error())
					os.Exit(1)
				}
			}
			result <- "OK"
		}
}
<commit_msg>pldata id<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
    "time"
	"strings"
)

var conn dbox.IConnection

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

var (
    pcs = toolkit.M{}
    ccs = toolkit.M{}
    ledgers = toolkit.M{}
    prods = toolkit.M{}
    custs = toolkit.M{}
	plmodels = toolkit.M{}
	brands = toolkit.M{}
	ratios = map[string][]gdrj.SalesRatio{}
)

func getCursor(obj orm.IModel)dbox.ICursor{
    c, e := gdrj.Find(obj,nil,nil)
    if e!=nil{
        return nil
    }
    return c
}

func prepMaster(){
    pc:=new(gdrj.ProfitCenter)
    cc:=new(gdrj.CostCenter)
    prod:=new(gdrj.Product)
    ledger:=new(gdrj.LedgerMaster)

    cpc := getCursor(pc)
    defer cpc.Close()
    var e error
    for e=cpc.Fetch(pc,1,false);e==nil;{
        pcs.Set(pc.ID,pc)
        pc =new(gdrj.ProfitCenter)
        e=cpc.Fetch(pc,1,false)
    }

    ccc:=getCursor(cc)
    defer ccc.Close()
    for e=ccc.Fetch(cc,1,false);e==nil;{
        ccs.Set(cc.ID,cc)
        cc = new(gdrj.CostCenter)
        e=ccc.Fetch(cc,1,false)
    }

    cprod:=getCursor(prod)
    defer cprod.Close()
    for e=cprod.Fetch(prod,1,false);e==nil;{
        prods.Set(prod.ID,prod)
        prod=new(gdrj.Product)
        e=cprod.Fetch(prod,1,false)
    }

    cledger:=getCursor(ledger)
    defer cledger.Close()
    for e=cledger.Fetch(ledger,1,false);e==nil;{
        ledgers.Set(ledger.ID,ledger)
        ledger=new(gdrj.LedgerMaster)
        e=cledger.Fetch(ledger,1,false)
    }

    cust := new(gdrj.Customer)
    ccust:=getCursor(cust)
    defer ccust.Close()
    for e=ccust.Fetch(cust,1,false);e==nil;{
        custs.Set(cust.ID,cust)
        cust=new(gdrj.Customer)
        e=ccust.Fetch(cust,1,false)
    }

	plmodel := new(gdrj.PLModel)
	cplmodel := getCursor(plmodel)
	defer cplmodel.Close()
	for e=cplmodel.Fetch(plmodel,1,false);e==nil;{
		plmodels.Set(plmodel.ID,plmodel)
		plmodel=new(gdrj.PLModel)
		e=cplmodel.Fetch(plmodel,1,false)
	}

	toolkit.Println("--> Brand")
	brand := new(gdrj.HBrandCategory)
	cbrand := getCursor(plmodel)
	defer cbrand.Close()
	for e=cbrand.Fetch(brand,1,false);e==nil;{
		brands.Set(brand.ID,brand)
		brand=new(gdrj.HBrandCategory)
		e=cbrand.Fetch(brand,1,false)
	}

	toolkit.Println("--> Sales Ratio")
	ratio := new(gdrj.SalesRatio)
	cratios := getCursor(ratio)
	defer cratios.Close()
	for {
		efetch := cratios.Fetch(ratio, 1, false)
		if efetch != nil {
			break
		}
		ratioid := toolkit.Sprintf("%d_%d_%s", ratio.Year, ratio.Month, ratio.BranchID)
		a, exist := ratios[ratioid]
		if !exist {
			a = []gdrj.SalesRatio{}
		}
		a=append(a, *ratio)
		ratio = new(gdrj.SalesRatio)
		ratios[ratioid] = a
	}
}

func main() {
	//runtime.GOMAXPROCS(runtime.NumCPU())
	setinitialconnection()
	defer gdrj.CloseDb()

    toolkit.Println("Reading Master")
    prepMaster()

	pldm := new(gdrj.PLDataModel)
	toolkit.Println("Delete existing")
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","30052016SAP_EXPORT")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_FREIGHT")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_SUSEMI")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_APINTRA")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","30052016SAP_SGAPL")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_MEGASARI")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_SALESRD")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_DISC-RDJKT")).Delete().Exec(nil)

    toolkit.Println("START...")

	//for i, src := range arrstring {
	//dbf := dbox.Contains("src", src)
	crx, err := gdrj.Find(new(gdrj.RawDataPL), nil, toolkit.M{})
	if err != nil {
		toolkit.Println("Error Found : ", err.Error())
		os.Exit(1)
	}

    count := crx.Count()

	jobs := make(chan *gdrj.RawDataPL, count)
	result := make(chan string, count)

	for wi:=1;wi<10;wi++{
		go worker(wi, jobs, result)
	}

	t0 := time.Now()
	ci := 0
	iseof := false
	for !iseof {
		arrpl := []*gdrj.RawDataPL{}
		e := crx.Fetch(&arrpl, 1000, false)
		if e!=nil{
			iseof=true
			break
		}

		for _, v := range arrpl {
			jobs <- v
			ci++
		}

		toolkit.Printfn("Processing %d of %d in %s", ci, count, time.Since(t0).String())

		if len(arrpl) < 1000 {
			iseof = true
		}
	}

	toolkit.Println("Saving")
	step := count / 100
	limit := step
	for ri := 0; ri < count; ri++ {
		<-result
		if ri >= limit {
			toolkit.Printfn("Saving %d of %d (%dpct) in %s", ri, count, ri*100/count,
				time.Since(t0).String())
			limit += step
		}
	}
	toolkit.Printfn("Done %s", time.Since(t0).String())
}

var pldatas = map[string]*gdrj.PLDataModel{}

func worker(wi int, jobs <-chan *gdrj.RawDataPL, result chan<- string){
	workerconn, err := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

		for v:= range jobs{
			if v.Src=="31052016SAP_SALESRD" || v.Src=="31052016SAP_DISC-RDJKT" || v.Src==""{
				result <- "NOK"
				continue
			}

			tdate := time.Date(v.Year, time.Month(v.Period), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 3, 0)

			ls := new(gdrj.PLDataModel)
			ls.CompanyCode = v.EntityID
			//ls.LedgerAccount = v.Account

			ls.Year = tdate.Year()
			ls.Month = int(tdate.Month())
			ls.Date = gdrj.NewDate(ls.Year, ls.Month, 1)

			ls.PCID = v.PCID
			if v.PCID != "" && pcs.Has(v.PCID) {
				ls.PC = pcs.Get(v.PCID).(*gdrj.ProfitCenter)
			}

			ls.CCID = v.CCID
			if v.CCID != "" && ccs.Has(v.CCID) {
				ls.CC = ccs.Get(v.CCID).(*gdrj.CostCenter)
			}

			ls.OutletID = v.OutletID
			if v.OutletID != "" && custs.Has(v.OutletID) {
				ls.Customer = custs.Get(v.OutletID).(*gdrj.Customer)
				//ls.Customer = gdrj.CustomerGetByID(v.OutletID)
			} else {
				c := new(gdrj.Customer)
				c.Name = v.OutletName
				c.BranchID = v.BusA
				c.ChannelID = "I3"
				c.ChannelName = "MT"
				c.CustType = "EXP"
				c.CustomerGroup = "EXP"
				c.Zone = "EXP"
				c.Region = "EXP"
				c.National = "EXP"
				c.AreaName = "EXP"
				c.CustomerGroupName = "Export"
				ls.Customer = c
			}

			ls.SKUID = v.SKUID
			if v.SKUID != "" && prods.Has(v.SKUID) {
				ls.Product = prods.Get(v.SKUID).(*gdrj.Product)
			} else if v.SKUID!="" {
				ls.Product = new(gdrj.Product)
				ls.Product.Name = v.ProductName
				ls.Product.BrandCategoryID = v.PCID[4:]
				if brands.Has(ls.Product.BrandCategoryID){
					ls.Product.Brand = brands.Get(ls.Product.BrandCategoryID).(*gdrj.HBrandCategory).BrandID
				} else {
					ls.Product.BrandCategoryID = "Common"
					ls.Product.Brand = "-"
				}
			}

			ls.Value1 = v.AmountinIDR
			//ls.Value2 = v.AmountinUSD

			tLedgerAccount := new(gdrj.LedgerMaster)
			if ledgers.Has(v.Account){
				tLedgerAccount = ledgers.Get(v.Account).(*gdrj.LedgerMaster)
			}
			if tLedgerAccount.PLCode==""{
				plm := plmodels.Get("PL34").(*gdrj.PLModel)
				ls.PLCode = plm.ID
				ls.PLOrder = plm.OrderIndex
				ls.PLGroup1 = plm.PLHeader1
				ls.PLGroup2 = plm.PLHeader2
				ls.PLGroup3 = plm.PLHeader3
			} else if v.Src=="30052016SAP_EXPORT"{
				plm := plmodels.Get("PL6").(*gdrj.PLModel)
				ls.PLCode = plm.ID
				ls.PLOrder = plm.OrderIndex
				ls.PLGroup1 = plm.PLHeader1
				ls.PLGroup2 = plm.PLHeader2
				ls.PLGroup3 = plm.PLHeader3
			} else  {
				ls.PLCode = tLedgerAccount.PLCode
				ls.PLOrder = tLedgerAccount.OrderIndex
				ls.PLGroup1 = tLedgerAccount.H1
				ls.PLGroup2 = tLedgerAccount.H2
				ls.PLGroup3 = tLedgerAccount.H3
			}

			ls.Date = gdrj.NewDate(ls.Year, int(ls.Month), 1)

			sources := strings.Split(v.Src,"_")
			if len(sources)==1{
				ls.Source = sources[1]
			} else if len(sources)>1{
				ls.Source = sources[1]
			} else {
				ls.Source="OTHER"
			}

			rs := []gdrj.SalesRatio{}
			if v.Src!="30052016SAP_EXPORT"{
				srid := toolkit.Sprintf("%d_%d_%s", ls.Year, ls.Month, ls.Customer.BranchID)
				a, exists := ratios[srid]
				if exists{
					rs=a
				}
			}

			if len(rs)==0{
				r := new(gdrj.SalesRatio)
				r.Year = ls.Year
				r.Month = ls.Month
				r.Ratio = 1
				rs = append(rs, *r)
			}

			total := float64(0)
			for _, r := range rs{
				total += r.Ratio
			}

			for _, r := range rs{
				lsexist := false
				rls := new(gdrj.PLDataModel)
				*rls = *ls
				rls.OutletID = r.OutletID
				rls.SKUID = r.SKUID
				rls.ID = rls.PrepareID().(string)
				rls, lsexist = pldatas[rls.ID]
				multiplier:=float64(1)
				if v.Src!="30052016SAP_EXPORT"{
					multiplier=-1
				}

				if !lsexist{
					//-- need to grand rls again
					*rls = *ls
					rls.OutletID = r.OutletID
					rls.SKUID = r.SKUID
					rls.ID = rls.PrepareID().(string)
					//-- end

					//-- get existing values
					els := new(gdrj.PLDataModel)
					cls,_ := workerconn.NewQuery().From(ls.TableName()).
						Where(dbox.Eq("_id",rls.ID)).Cursor(nil)
					ecls:=cls.Fetch(els,1,false)
					if ecls==nil{
						rls.Value1=els.Value1
					}
					cls.Close()
				}

				rls.Value1 += ls.Value1 * r.Ratio/total * multiplier
				err = workerconn.NewQuery().From(ls.TableName()).Save().Exec(toolkit.M{}.Set("data",rls))
				if err != nil {
					toolkit.Println("Error Found : ", err.Error())
					os.Exit(1)
				}
				pldatas[rls.ID]=rls
			}
			result <- "OK"
		}
}
<|endoftext|>


<commit_before>package slack

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
)

const (
	// ParseNone autoformats the text in messages less.
	ParseNone = "none"
	// ParseFull autoformats message text more, like creating hyperlinks
	// automatically.
	ParseFull = "full"

	// LinkNamesOn enables making usernames hyperlinks.
	LinkNamesOn = 1
	// LinkNamesOff disables making usernames hyperlinks.
	LinkNamesOff = 0
)

// API is the endpoint that handles posting a message.
var API = "https://slack.com/api/chat.postMessage"

type apiResponse struct {
	OK        bool   `json:"ok"`
	Channel   string `json:"channel"`
	Timestamp string `json:"ts"`
	Message   struct {
		Text     string `json:"text"`
		Username string `json:"username"`
		Icons    struct {
			Emoji   string `json:"emoji"`
			Image64 string `json:"image_64"`
		} `json:"icons"`
		Type      string `json:"type"`
		Subtype   string `json:"subtype"`
		Timestamp string `json:"ts"`
	} `json:"message"`
	Error string `json:"error"`
}

// Notification is a Slack notification.
type Notification struct {
	// AppURL is your Slack App's webhook URL
	AppURL string
	// Token is a user's authentication token.
	Token string
	// Channel is a notification's destination. It can be a channel, private
	// group, or username.
	Channel string
	// Text is the notification's message.
	Text string
	// Parse is the mode used to parse text.
	Parse string
	// LinkNames converts usernames into links.
	LinkNames int
	// Attachments are rich text snippets.
	Attachments map[string]string
	// UnfurlLinks attempts to expand a link to show a preview. Success depends
	// on the webpage having the right markdown.
	UnfurlLinks bool
	// UnfurlMedia attempts to expand a link to show a preview. Success depends
	// on the webpage having the right markdown.
	UnfurlMedia bool
	// Username given to bot. If AsUser is true, then message will try to be
	// sent from the given user.
	Username string
	// AsUser attempt to send a message as the user in Username.
	AsUser bool
	// IconURL is a URL to set as the user icon.
	IconURL string
	// IconEmoji is an emoji to set as the user icon.
	IconEmoji string

	Client *http.Client
}

// Send triggers a Slack notification.
func (n *Notification) Send() error {
	if n.AppURL == "" {
		if n.Token == "" {
			return errors.New("missing authentication token or App URL")
		}
		if n.Channel == "" {
			return errors.New("missing channel, group, or username destination")
		}
	}
	if n.Text == "" {
		return errors.New("missing message text")
	}

	attach, err := json.Marshal(n.Attachments)
	if err != nil {
		return err
	}

	if n.AppURL == "" {
		vals := make(url.Values)
		vals.Set("token", n.Token)
		vals.Set("channel", n.Channel)
		vals.Set("text", n.Text)
		vals.Set("parse", n.Parse)
		vals.Set("link_names", fmt.Sprint(n.LinkNames))
		vals.Set("attachments", string(attach))
		vals.Set("unfurl_links", fmt.Sprintf("%t", n.UnfurlLinks))
		vals.Set("unfurl_media", fmt.Sprintf("%t", n.UnfurlMedia))
		vals.Set("username", n.Username)
		vals.Set("as_user", fmt.Sprintf("%t", n.AsUser))
		vals.Set("icon_url", n.IconURL)
		vals.Set("icon_emoji", n.IconEmoji)

		resp, err := n.Client.PostForm(API, vals)
		if err != nil {
			return err
		}

		defer resp.Body.Close()

		var r apiResponse

		if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
			return err
		}

		if !r.OK {
			return errors.New(r.Error)
		}
	} else {
		json, _ := json.Marshal(struct {
			Text string `json:"text"`
		}{n.Text})

		resp, err := n.Client.Post(n.AppURL, "application/json", bytes.NewReader(json))
		if err != nil {
			return err
		}

		defer resp.Body.Close()

		buff := new(bytes.Buffer)
		buff.ReadFrom(resp.Body)
		s := buff.String()

		if s != "ok" {
			return errors.New("Error invoking slack API: " + s)
		}
	}

	return nil
}
<commit_msg>Refactor slack<commit_after>package slack

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
)

const (
	// ParseNone autoformats the text in messages less.
	ParseNone = "none"
	// ParseFull autoformats message text more, like creating hyperlinks
	// automatically.
	ParseFull = "full"

	// LinkNamesOn enables making usernames hyperlinks.
	LinkNamesOn = 1
	// LinkNamesOff disables making usernames hyperlinks.
	LinkNamesOff = 0
)

// API is the endpoint that handles posting a message.
var API = "https://slack.com/api/chat.postMessage"

type apiResponse struct {
	OK        bool   `json:"ok"`
	Channel   string `json:"channel"`
	Timestamp string `json:"ts"`
	Message   struct {
		Text     string `json:"text"`
		Username string `json:"username"`
		Icons    struct {
			Emoji   string `json:"emoji"`
			Image64 string `json:"image_64"`
		} `json:"icons"`
		Type      string `json:"type"`
		Subtype   string `json:"subtype"`
		Timestamp string `json:"ts"`
	} `json:"message"`
	Error string `json:"error"`
}

// Notification is a Slack notification.
type Notification struct {
	// AppURL is your Slack App's webhook URL
	AppURL string
	// Token is a user's authentication token.
	Token string
	// Channel is a notification's destination. It can be a channel, private
	// group, or username.
	Channel string
	// Text is the notification's message.
	Text string
	// Parse is the mode used to parse text.
	Parse string
	// LinkNames converts usernames into links.
	LinkNames int
	// Attachments are rich text snippets.
	Attachments map[string]string
	// UnfurlLinks attempts to expand a link to show a preview. Success depends
	// on the webpage having the right markdown.
	UnfurlLinks bool
	// UnfurlMedia attempts to expand a link to show a preview. Success depends
	// on the webpage having the right markdown.
	UnfurlMedia bool
	// Username given to bot. If AsUser is true, then message will try to be
	// sent from the given user.
	Username string
	// AsUser attempt to send a message as the user in Username.
	AsUser bool
	// IconURL is a URL to set as the user icon.
	IconURL string
	// IconEmoji is an emoji to set as the user icon.
	IconEmoji string

	Client *http.Client
}

// Send triggers a Slack notification.
func (n *Notification) Send() error {
	if n.AppURL == "" {
		if n.Token == "" {
			return errors.New("missing authentication token or App URL")
		}
		if n.Channel == "" {
			return errors.New("missing channel, group, or username destination")
		}
	}
	if n.Text == "" {
		return errors.New("missing message text")
	}

	attach, err := json.Marshal(n.Attachments)
	if err != nil {
		return err
	}

	// Legacy token-based integration.
	if n.AppURL == "" {
		vals := make(url.Values)
		vals.Set("token", n.Token)
		vals.Set("channel", n.Channel)
		vals.Set("text", n.Text)
		vals.Set("parse", n.Parse)
		vals.Set("link_names", fmt.Sprint(n.LinkNames))
		vals.Set("attachments", string(attach))
		vals.Set("unfurl_links", fmt.Sprintf("%t", n.UnfurlLinks))
		vals.Set("unfurl_media", fmt.Sprintf("%t", n.UnfurlMedia))
		vals.Set("username", n.Username)
		vals.Set("as_user", fmt.Sprintf("%t", n.AsUser))
		vals.Set("icon_url", n.IconURL)
		vals.Set("icon_emoji", n.IconEmoji)

		resp, err := n.Client.PostForm(API, vals)
		if err != nil {
			return err
		}
		defer resp.Body.Close()

		var r apiResponse
		if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
			return err
		}

		if !r.OK {
			return errors.New(r.Error)
		}

		return nil
	}

	// New Slack app URL integration.
	data, err := json.Marshal(struct {
		Text string `json:"text"`
	}{n.Text})
	if err != nil {
		return err
	}

	resp, err := n.Client.Post(n.AppURL, "application/json", bytes.NewReader(data))
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	buff := new(bytes.Buffer)
	_, err = buff.ReadFrom(resp.Body)
	if err != nil {
		return err
	}
	s := buff.String()

	if s != "ok" {
		return fmt.Errorf("slack api error: %s", s)
	}

	return nil
}
<|endoftext|>


<commit_before>package conf

import (
	"io/ioutil"
	"os"
	"regexp"
	"testing"
)

func TestReadAggregations(t *testing.T) {
	type testCase struct {
		title  string
		in     string
		expAgg Aggregations
		expErr bool
	}

	testCases := []testCase{
		{
			title:  "completely empty", // should result in just the default
			in:     "",
			expErr: false,
			expAgg: NewAggregations(),
		},
		{
			title:  "empty name ",
			in:     `[]`,
			expErr: true,
		},
		{
			title: "bad name format",
			in: `foo[]
			`,
			expErr: true,
		},
		{
			title: "missing pattern",
			in: `[foo]
			`,
			expErr: true,
		},
		{
			title: "invalid pattern",
			in: `[foo]
			pattern = "((("`,
			expErr: true,
		},
		{
			title: "defaults",
			in: `[foo]
			pattern = foo.*`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "foo",
						Pattern:           regexp.MustCompile("foo.*"),
						XFilesFactor:      0.5,
						AggregationMethod: []Method{Avg},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
		{
			title: "grafanacloud_default",
			in: `[default]
pattern = .*
xFilesFactor = 0.1
aggregationMethod = avg,sum`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "default",
						Pattern:           regexp.MustCompile(".*"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Avg, Sum},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
		{
			title: "graphite upstream storage-aggregation.conf example",
			in: `
# Aggregation methods for whisper files. Entries are scanned in order,
# and first match wins. This file is scanned for changes every 60 seconds
#
#  [name]
#  pattern = <regex>
#  xFilesFactor = <float between 0 and 1>
#  aggregationMethod = <average|sum|last|max|min>
#
#  name: Arbitrary unique name for the rule
#  pattern: Regex pattern to match against the metric name
#  xFilesFactor: Ratio of valid data points required for aggregation to the next retention to occur
#  aggregationMethod: function to apply to data points for aggregation
#
[min]
pattern = \.min$
xFilesFactor = 0.1
aggregationMethod = min

[max]
pattern = \.max$
xFilesFactor = 0.1
aggregationMethod = max

[sum]
pattern = \.count$
xFilesFactor = 0
# for monotonically increasing counters
aggregationMethod = max
# for counters that reset every interval (statsd-style)
#aggregationMethod = sum

[default_average]
pattern = .*
xFilesFactor = 0.5
aggregationMethod = average
			`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "min",
						Pattern:           regexp.MustCompile("\\.min$"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Min},
					},
					{
						Name:              "max",
						Pattern:           regexp.MustCompile("\\.max$"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Max},
					},
					{
						Name:              "sum",
						Pattern:           regexp.MustCompile("\\.count$"),
						XFilesFactor:      0,
						AggregationMethod: []Method{Max},
					},
					{
						Name:              "default_average",
						Pattern:           regexp.MustCompile(".*"),
						XFilesFactor:      0.5,
						AggregationMethod: []Method{Avg},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
		{
			title: "graphite upstream default storage-aggregation.conf",
			in: `
		# Aggregation methods for whisper files. Entries are scanned in order,
# and first match wins. This file is scanned for changes every 60 seconds
#
#  [name]
#  pattern = <regex>
#  xFilesFactor = <float between 0 and 1>
#  aggregationMethod = <average|sum|last|max|min>
#
#  name: Arbitrary unique name for the rule
#  pattern: Regex pattern to match against the metric name
#  xFilesFactor: Ratio of valid data points required for aggregation to the next retention to occur
#  aggregationMethod: function to apply to data points for aggregation
#
[min]
pattern = \.lower$
xFilesFactor = 0.1
aggregationMethod = min

[max]
pattern = \.upper(_\d+)?$
xFilesFactor = 0.1
aggregationMethod = max

[sum]
pattern = \.sum$
xFilesFactor = 0
aggregationMethod = sum

[count]
pattern = \.count$
xFilesFactor = 0
aggregationMethod = sum

[count_legacy]
pattern = ^stats_counts.*
xFilesFactor = 0
aggregationMethod = sum

[default_average]
pattern = .*
xFilesFactor = 0.3
aggregationMethod = average
`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "min",
						Pattern:           regexp.MustCompile("\\.lower$"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Min},
					},
					{
						Name:              "max",
						Pattern:           regexp.MustCompile("\\.upper(_\\d+)?$"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Max},
					},
					{
						Name:              "sum",
						Pattern:           regexp.MustCompile("\\.sum$"),
						XFilesFactor:      0,
						AggregationMethod: []Method{Sum},
					},
					{
						Name:              "count",
						Pattern:           regexp.MustCompile("\\.count$"),
						XFilesFactor:      0,
						AggregationMethod: []Method{Sum},
					},
					{
						Name:              "count_legacy",
						Pattern:           regexp.MustCompile("^stats_counts.*"),
						XFilesFactor:      0,
						AggregationMethod: []Method{Sum},
					},
					{
						Name:              "default_average",
						Pattern:           regexp.MustCompile(".*"),
						XFilesFactor:      0.3,
						AggregationMethod: []Method{Avg},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
	}

	for _, c := range testCases {
		file, err := ioutil.TempFile("", "metrictank-TestReadAggregations")
		if err != nil {
			t.Fatal(err)
		}
		defer os.Remove(file.Name())
		if _, err := file.Write([]byte(c.in)); err != nil {
			t.Fatal(err)
		}
		if err := file.Close(); err != nil {
			t.Fatal(err)
		}
		t.Logf("testing %q", c.title)
		agg, err := ReadAggregations(file.Name())
		if !c.expErr && err != nil {
			t.Fatalf("testcase %q expected no error but got error %s", c.title, err.Error())
		}
		if c.expErr && err == nil {
			t.Fatalf("testcase %q expected error but got no error", c.title)
		}
		if err == nil {
			if !agg.Equals(c.expAgg) {
				t.Fatalf("testcase %q expected\nexp agg %+v\ngot agg %+v", c.title, c.expAgg, agg)
			}
		}
	}
}
<commit_msg>add unit tests with some comments<commit_after>package conf

import (
	"io/ioutil"
	"os"
	"regexp"
	"testing"
)

func TestReadAggregations(t *testing.T) {
	type testCase struct {
		title  string
		in     string
		expAgg Aggregations
		expErr bool
	}

	testCases := []testCase{
		{
			title:  "completely empty", // should result in just the default
			in:     "",
			expErr: false,
			expAgg: NewAggregations(),
		},
		{
			title:  "empty name ",
			in:     `[]`,
			expErr: true,
		},
		{
			title: "bad name format",
			in: `foo[]
			`,
			expErr: true,
		},
		{
			title: "missing pattern",
			in: `[foo]
			`,
			expErr: true,
		},
		{
			title: "invalid pattern",
			in: `[foo]
			pattern = "((("`,
			expErr: true,
		},
		{
			title: "commented out pattern is still missing",
			in: `[foo]
			;pattern = foo.*`,
			expErr: true,
		},
		{
			title: "defaults",
			in: `[foo]
			pattern = foo.*`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "foo",
						Pattern:           regexp.MustCompile("foo.*"),
						XFilesFactor:      0.5,
						AggregationMethod: []Method{Avg},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
		{
			title: "defaults with some comments",
			in: `[foo] # comment here [does it confuse the parser if i do this?]
			pattern = foo.* # another comment here
			# pattern = this-should-be-ignored
			# and a final comment on its own line`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "foo",
						Pattern:           regexp.MustCompile("foo.*"),
						XFilesFactor:      0.5,
						AggregationMethod: []Method{Avg},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
		{
			title: "lots of comments",
			in: `;[this is not a section]
			[foo] # [commented]
			pattern = foo.* # another comment here
			; pattern = this-should-be-ignored
			xFilesFactor = 0.8 # comment
			;xFilesFactor = 0.9
			;aggregationMethod = min,avg
			#aggregationMethod = min,avg
			aggregationMethod = max
			;aggregationMethod = min,avg
			#aggregationMethod = min,avg
			; and a final comment on its own line`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "foo",
						Pattern:           regexp.MustCompile("foo.*"),
						XFilesFactor:      0.8,
						AggregationMethod: []Method{Max},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
		{
			title: "grafanacloud_default",
			in: `[default]
pattern = .*
xFilesFactor = 0.1
aggregationMethod = avg,sum`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "default",
						Pattern:           regexp.MustCompile(".*"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Avg, Sum},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
		{
			title: "graphite upstream storage-aggregation.conf example",
			in: `
# Aggregation methods for whisper files. Entries are scanned in order,
# and first match wins. This file is scanned for changes every 60 seconds
#
#  [name]
#  pattern = <regex>
#  xFilesFactor = <float between 0 and 1>
#  aggregationMethod = <average|sum|last|max|min>
#
#  name: Arbitrary unique name for the rule
#  pattern: Regex pattern to match against the metric name
#  xFilesFactor: Ratio of valid data points required for aggregation to the next retention to occur
#  aggregationMethod: function to apply to data points for aggregation
#
[min]
pattern = \.min$
xFilesFactor = 0.1
aggregationMethod = min

[max]
pattern = \.max$
xFilesFactor = 0.1
aggregationMethod = max

[sum]
pattern = \.count$
xFilesFactor = 0
# for monotonically increasing counters
aggregationMethod = max
# for counters that reset every interval (statsd-style)
#aggregationMethod = sum

[default_average]
pattern = .*
xFilesFactor = 0.5
aggregationMethod = average
			`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "min",
						Pattern:           regexp.MustCompile("\\.min$"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Min},
					},
					{
						Name:              "max",
						Pattern:           regexp.MustCompile("\\.max$"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Max},
					},
					{
						Name:              "sum",
						Pattern:           regexp.MustCompile("\\.count$"),
						XFilesFactor:      0,
						AggregationMethod: []Method{Max},
					},
					{
						Name:              "default_average",
						Pattern:           regexp.MustCompile(".*"),
						XFilesFactor:      0.5,
						AggregationMethod: []Method{Avg},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
		{
			title: "graphite upstream default storage-aggregation.conf",
			in: `
		# Aggregation methods for whisper files. Entries are scanned in order,
# and first match wins. This file is scanned for changes every 60 seconds
#
#  [name]
#  pattern = <regex>
#  xFilesFactor = <float between 0 and 1>
#  aggregationMethod = <average|sum|last|max|min>
#
#  name: Arbitrary unique name for the rule
#  pattern: Regex pattern to match against the metric name
#  xFilesFactor: Ratio of valid data points required for aggregation to the next retention to occur
#  aggregationMethod: function to apply to data points for aggregation
#
[min]
pattern = \.lower$
xFilesFactor = 0.1
aggregationMethod = min

[max]
pattern = \.upper(_\d+)?$
xFilesFactor = 0.1
aggregationMethod = max

[sum]
pattern = \.sum$
xFilesFactor = 0
aggregationMethod = sum

[count]
pattern = \.count$
xFilesFactor = 0
aggregationMethod = sum

[count_legacy]
pattern = ^stats_counts.*
xFilesFactor = 0
aggregationMethod = sum

[default_average]
pattern = .*
xFilesFactor = 0.3
aggregationMethod = average
`,
			expErr: false,
			expAgg: Aggregations{
				Data: []Aggregation{
					{
						Name:              "min",
						Pattern:           regexp.MustCompile("\\.lower$"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Min},
					},
					{
						Name:              "max",
						Pattern:           regexp.MustCompile("\\.upper(_\\d+)?$"),
						XFilesFactor:      0.1,
						AggregationMethod: []Method{Max},
					},
					{
						Name:              "sum",
						Pattern:           regexp.MustCompile("\\.sum$"),
						XFilesFactor:      0,
						AggregationMethod: []Method{Sum},
					},
					{
						Name:              "count",
						Pattern:           regexp.MustCompile("\\.count$"),
						XFilesFactor:      0,
						AggregationMethod: []Method{Sum},
					},
					{
						Name:              "count_legacy",
						Pattern:           regexp.MustCompile("^stats_counts.*"),
						XFilesFactor:      0,
						AggregationMethod: []Method{Sum},
					},
					{
						Name:              "default_average",
						Pattern:           regexp.MustCompile(".*"),
						XFilesFactor:      0.3,
						AggregationMethod: []Method{Avg},
					},
				},
				DefaultAggregation: defaultAggregation(),
			},
		},
	}

	for _, c := range testCases {
		file, err := ioutil.TempFile("", "metrictank-TestReadAggregations")
		if err != nil {
			t.Fatal(err)
		}
		defer os.Remove(file.Name())
		if _, err := file.Write([]byte(c.in)); err != nil {
			t.Fatal(err)
		}
		if err := file.Close(); err != nil {
			t.Fatal(err)
		}
		t.Logf("testing %q", c.title)
		agg, err := ReadAggregations(file.Name())
		if !c.expErr && err != nil {
			t.Fatalf("testcase %q expected no error but got error %s", c.title, err.Error())
		}
		if c.expErr && err == nil {
			t.Fatalf("testcase %q expected error but got no error", c.title)
		}
		if err == nil {
			if !agg.Equals(c.expAgg) {
				t.Fatalf("testcase %q expected\nexp agg %+v\ngot agg %+v", c.title, c.expAgg, agg)
			}
		}
	}
}
<|endoftext|>


<commit_before>// +build go1.3

package graceful

import (
	"log"
	"net"
	"net/http"

	"github.com/zenazn/goji/graceful/listener"
)

// This is a slightly hacky shim to disable keepalives when shutting a server
// down. We could have added extra functionality in listener or signal.go to
// deal with this case, but this seems simpler.
type gracefulServer struct {
	net.Listener
	s *http.Server
}

func (g gracefulServer) Close() error {
	g.s.SetKeepAlivesEnabled(false)
	return g.Listener.Close()
}

// A chaining http.ConnState wrapper
type connState func(net.Conn, http.ConnState)

func (c connState) Wrap(nc net.Conn, s http.ConnState) {
	// There are a few other states defined, most notably StateActive.
	// Unfortunately it doesn't look like it's possible to make use of
	// StateActive to implement graceful shutdown, since StateActive is set
	// after a complete request has been read off the wire with an intent to
	// process it. If we were to race a graceful shutdown against a
	// connection that was just read off the wire (but not yet in
	// StateActive), we would accidentally close the connection out from
	// underneath an active request.
	//
	// We already needed to work around this for Go 1.2 by shimming out a
	// full net.Conn object, so we can just fall back to the old behavior
	// there.
	//
	// I started a golang-nuts thread about this here:
	// https://groups.google.com/forum/#!topic/golang-nuts/Xi8yjBGWfCQ I'd
	// be very eager to find a better way to do this, so reach out to me if
	// you have any ideas.
	switch s {
	case http.StateIdle:
		if err := listener.MarkIdle(nc); err != nil {
			log.Printf("error marking conn as idle: %v", err)
		}
	case http.StateHijacked:
		if err := listener.Disown(nc); err != nil {
			log.Printf("error disowning hijacked conn: %v", err)
		}
	}
	if c != nil {
		c(nc, s)
	}
}

func (srv *Server) Serve(l net.Listener) error {
	// Spawn a shadow http.Server to do the actual servering. We do this
	// because we need to sketch on some of the parameters you passed in,
	// and it's nice to keep our sketching to ourselves.
	shadow := *(*http.Server)(srv)
	shadow.ConnState = connState(shadow.ConnState).Wrap

	l = gracefulServer{l, &shadow}
	wrap := listener.Wrap(l, listener.Automatic)
	appendListener(wrap)

	err := shadow.Serve(wrap)
	return peacefulError(err)
}
<commit_msg>Comment formatting nit<commit_after>// +build go1.3

package graceful

import (
	"log"
	"net"
	"net/http"

	"github.com/zenazn/goji/graceful/listener"
)

// This is a slightly hacky shim to disable keepalives when shutting a server
// down. We could have added extra functionality in listener or signal.go to
// deal with this case, but this seems simpler.
type gracefulServer struct {
	net.Listener
	s *http.Server
}

func (g gracefulServer) Close() error {
	g.s.SetKeepAlivesEnabled(false)
	return g.Listener.Close()
}

// A chaining http.ConnState wrapper
type connState func(net.Conn, http.ConnState)

func (c connState) Wrap(nc net.Conn, s http.ConnState) {
	// There are a few other states defined, most notably StateActive.
	// Unfortunately it doesn't look like it's possible to make use of
	// StateActive to implement graceful shutdown, since StateActive is set
	// after a complete request has been read off the wire with an intent to
	// process it. If we were to race a graceful shutdown against a
	// connection that was just read off the wire (but not yet in
	// StateActive), we would accidentally close the connection out from
	// underneath an active request.
	//
	// We already needed to work around this for Go 1.2 by shimming out a
	// full net.Conn object, so we can just fall back to the old behavior
	// there.
	//
	// I started a golang-nuts thread about this here:
	// https://groups.google.com/forum/#!topic/golang-nuts/Xi8yjBGWfCQ
	// I'd be very eager to find a better way to do this, so reach out to me
	// if you have any ideas.
	switch s {
	case http.StateIdle:
		if err := listener.MarkIdle(nc); err != nil {
			log.Printf("error marking conn as idle: %v", err)
		}
	case http.StateHijacked:
		if err := listener.Disown(nc); err != nil {
			log.Printf("error disowning hijacked conn: %v", err)
		}
	}
	if c != nil {
		c(nc, s)
	}
}

func (srv *Server) Serve(l net.Listener) error {
	// Spawn a shadow http.Server to do the actual servering. We do this
	// because we need to sketch on some of the parameters you passed in,
	// and it's nice to keep our sketching to ourselves.
	shadow := *(*http.Server)(srv)
	shadow.ConnState = connState(shadow.ConnState).Wrap

	l = gracefulServer{l, &shadow}
	wrap := listener.Wrap(l, listener.Automatic)
	appendListener(wrap)

	err := shadow.Serve(wrap)
	return peacefulError(err)
}
<|endoftext|>


<commit_before>package db

import (
	"context"
	"time"

	"github.com/mongodb/mongo-go-driver/bson"
	"github.com/mongodb/mongo-go-driver/bson/objectid"
	"github.com/mongodb/mongo-go-driver/mongo"
	"github.com/mongodb/mongo-go-driver/mongo/findopt"
	"github.com/pkg/errors"
)

const (
	EmailDocumentCollection = "email"
	DefaultLimit            = 50
)

// EmailDocument storing documents for email
type EmailDocument struct {
	ID       objectid.ObjectID       `bson:"_id,omitempty"`
	EID      string                  `bson:"eid,omitempty"`
	ReID     string                  `bson:"reid,omitempty"`
	SendDate time.Time               `bson:"sendDate,omitempty"`
	Status   string                  `bson:"status,omitempty"`
	Reason   string                  `bson:"reason,omitempty"`
	Content  EmailContentSubDocument `bson:"content,omitempty"`
}

// EmailContentSubDocument storing sub documents for email
type EmailContentSubDocument struct {
	From    string   `bson:"from,omitempty"`
	To      []string `bson:"to,omitempty"`
	Cc      []string `bson:"cc,omitempty"`
	Bcc     []string `bson:"bcc,omitempty"`
	Subject string   `bson:"subject,omitempty"`
	HTML    string   `bson:"html,omitempty"`
	Text    string   `bson:"text,omitempty"`
}

// Insert insert a email to db
func (doc *EmailDocument) Insert(ctx context.Context, db *mongo.Database) error {
	if doc == nil {
		return errors.New("document is nil")
	}
	coll := db.Collection(doc.Collection())

	total, err := coll.Count(ctx,
		bson.NewDocument(
			bson.EC.String("eid", doc.EID),
		))
	if err != nil {
		return errors.Wrapf(err, "failed to query document by %s", doc.EID)
	}
	if total > 0 {
		return &errDuplicateKey{
			error: errors.Errorf("document %s is exist", doc.EID)}
	}

	to := bson.NewArray()
	for _, addr := range doc.Content.To {
		to.Append(bson.VC.String(addr))
	}
	cc := bson.NewArray()
	for _, addr := range doc.Content.To {
		cc.Append(bson.VC.String(addr))
	}
	bcc := bson.NewArray()
	for _, addr := range doc.Content.To {
		bcc.Append(bson.VC.String(addr))
	}

	result, err := coll.InsertOne(ctx,
		bson.NewDocument(
			bson.EC.String("eid", doc.EID),
			bson.EC.String("reid", doc.ReID),
			bson.EC.Time("sendDate", doc.SendDate),
			bson.EC.String("status", doc.Status),
			bson.EC.String("reason", doc.Reason),
			bson.EC.SubDocumentFromElements("content",
				bson.EC.String("from", doc.Content.From),
				bson.EC.Array("to", to),
				bson.EC.Array("cc", cc),
				bson.EC.Array("bcc", bcc),
				bson.EC.String("subject", doc.Content.Subject),
				bson.EC.String("html", doc.Content.HTML),
				bson.EC.String("test", doc.Content.Text),
			),
		))
	if err != nil {
		return errors.Wrapf(err, "failed to insert document by %s", doc.EID)
	}

	if result != nil {
		if oid, ok := result.InsertedID.(objectid.ObjectID); ok {
			copy(doc.ID[:], oid[:])
		}
	}

	return nil
}

// EmailDocumentByEID gets a email document by eid from the db
func EmailDocumentByEID(ctx context.Context, db *mongo.Database, eid string) (*EmailDocument, error) {
	coll := db.Collection(EmailDocumentCollection)

	doc := &EmailDocument{}
	docResult := coll.FindOne(ctx,
		bson.NewDocument(
			bson.EC.String("eid", eid),
		))
	if err := docResult.Decode(doc); err != nil {
		if err == mongo.ErrNoDocuments {
			return nil, &errNoDocuments{
				error: err,
			}
		}
		return nil, err
	}

	return doc, nil
}

// EmailDocumentWhere query condition
type EmailDocumentWhere struct {
	StartTime time.Time
	EndTime   time.Time

	// pagination info
	Limit  int64
	LastID *objectid.ObjectID
}

// EmailDocumentByWhere gets pagination list of email document by condition from the db
func EmailDocumentByWhere(ctx context.Context, db *mongo.Database, where EmailDocumentWhere) ([]*EmailDocument, error) {
	coll := db.Collection(EmailDocumentCollection)
	condition := bson.NewDocument()

	whereDoc := bson.NewDocument()
	if !where.StartTime.IsZero() {
		whereDoc.Append(bson.EC.Time("$gte", where.StartTime))
	}
	if !where.EndTime.IsZero() {
		whereDoc.Append(bson.EC.Time("$lt", where.EndTime))
	}
	if !where.StartTime.IsZero() || !where.EndTime.IsZero() {
		condition.Append(
			bson.EC.SubDocument("sendDate", whereDoc),
		)
	}

	if where.LastID != nil {
		condition.Append(
			bson.EC.SubDocumentFromElements("_id",
				bson.EC.ObjectID("$gt", *where.LastID),
			))
	}

	if where.Limit == 0 {
		where.Limit = DefaultLimit
	}

	cursor, err := coll.Find(ctx, condition,
		findopt.Limit(where.Limit),
		findopt.Sort(map[string]int32{
			"sendDate": -1,
		}))
	if err != nil {
		return nil, errors.Wrap(err, "failed to query email documents")
	}
	defer cursor.Close(ctx)

	var docs []*EmailDocument
	for cursor.Next(ctx) {
		doc := &EmailDocument{}
		err := cursor.Decode(doc)
		if err != nil {
			return nil, err
		}
		docs = append(docs, doc)
	}

	return docs, nil
}

// CountEmailDocumentByWhere count email documents by condition from the db
func CountEmailDocumentByWhere(ctx context.Context, db *mongo.Database, where EmailDocumentWhere) (int64, error) {
	coll := db.Collection(EmailDocumentCollection)
	condition := bson.NewDocument()

	whereDoc := bson.NewDocument()
	if !where.StartTime.IsZero() {
		whereDoc.Append(bson.EC.Time("$gte", where.StartTime))
	}
	if !where.EndTime.IsZero() {
		whereDoc.Append(bson.EC.Time("$lt", where.EndTime))
	}
	if !where.StartTime.IsZero() || !where.EndTime.IsZero() {
		condition.Append(
			bson.EC.SubDocument("sendDate", whereDoc),
		)
	}

	total, err := coll.Count(ctx, condition)
	if err != nil {
		return 0, err
	}

	return total, nil
}

// EmailDocumentCreateIndexes create indexes to optimize the query.
func EmailDocumentCreateIndexes(ctx context.Context, db *mongo.Database) error {
	coll := db.Collection(EmailDocumentCollection)

	_, err := coll.Indexes().CreateOne(ctx,
		mongo.IndexModel{
			Keys: bson.NewDocument(
				bson.EC.Int32("sendDate", -1),
			),
		},
	)
	if err != nil {
		return errors.Wrap(err, "failed to create index")
	}

	return nil
}

// Collection return collection name
func (doc *EmailDocument) Collection() string {
	return EmailDocumentCollection
}
<commit_msg>fix<commit_after>package db

import (
	"context"
	"time"

	"github.com/mongodb/mongo-go-driver/bson"
	"github.com/mongodb/mongo-go-driver/bson/objectid"
	"github.com/mongodb/mongo-go-driver/mongo"
	"github.com/mongodb/mongo-go-driver/mongo/findopt"
	"github.com/pkg/errors"
)

const (
	// DefaultDBName default db name
	DefaultDBName = "ses"

	// EmailDocumentCollection the collection name of email
	EmailDocumentCollection = "email"

	// DefaultLimit default limit
	DefaultLimit = 50
)

// EmailDocument storing documents for email
type EmailDocument struct {
	ID       objectid.ObjectID       `bson:"_id,omitempty"`
	EID      string                  `bson:"eid,omitempty"`
	ReID     string                  `bson:"reid,omitempty"`
	SendDate time.Time               `bson:"sendDate,omitempty"`
	Status   string                  `bson:"status,omitempty"`
	Reason   string                  `bson:"reason,omitempty"`
	Content  EmailContentSubDocument `bson:"content,omitempty"`
}

// EmailContentSubDocument storing sub documents for email
type EmailContentSubDocument struct {
	From    string   `bson:"from,omitempty"`
	To      []string `bson:"to,omitempty"`
	Cc      []string `bson:"cc,omitempty"`
	Bcc     []string `bson:"bcc,omitempty"`
	Subject string   `bson:"subject,omitempty"`
	HTML    string   `bson:"html,omitempty"`
	Text    string   `bson:"text,omitempty"`
}

// Insert insert a email to db
func (doc *EmailDocument) Insert(ctx context.Context, db *mongo.Database) error {
	if doc == nil {
		return errors.New("document is nil")
	}
	coll := db.Collection(doc.Collection())

	total, err := coll.Count(ctx,
		bson.NewDocument(
			bson.EC.String("eid", doc.EID),
		))
	if err != nil {
		return errors.Wrapf(err, "failed to query document by %s", doc.EID)
	}
	if total > 0 {
		return &errDuplicateKey{
			error: errors.Errorf("document %s is exist", doc.EID)}
	}

	to := bson.NewArray()
	for _, addr := range doc.Content.To {
		to.Append(bson.VC.String(addr))
	}
	cc := bson.NewArray()
	for _, addr := range doc.Content.To {
		cc.Append(bson.VC.String(addr))
	}
	bcc := bson.NewArray()
	for _, addr := range doc.Content.To {
		bcc.Append(bson.VC.String(addr))
	}

	result, err := coll.InsertOne(ctx,
		bson.NewDocument(
			bson.EC.String("eid", doc.EID),
			bson.EC.String("reid", doc.ReID),
			bson.EC.Time("sendDate", doc.SendDate),
			bson.EC.String("status", doc.Status),
			bson.EC.String("reason", doc.Reason),
			bson.EC.SubDocumentFromElements("content",
				bson.EC.String("from", doc.Content.From),
				bson.EC.Array("to", to),
				bson.EC.Array("cc", cc),
				bson.EC.Array("bcc", bcc),
				bson.EC.String("subject", doc.Content.Subject),
				bson.EC.String("html", doc.Content.HTML),
				bson.EC.String("test", doc.Content.Text),
			),
		))
	if err != nil {
		return errors.Wrapf(err, "failed to insert document by %s", doc.EID)
	}

	if result != nil {
		if oid, ok := result.InsertedID.(objectid.ObjectID); ok {
			copy(doc.ID[:], oid[:])
		}
	}

	return nil
}

// EmailDocumentByEID gets a email document by eid from the db
func EmailDocumentByEID(ctx context.Context, db *mongo.Database, eid string) (*EmailDocument, error) {
	coll := db.Collection(EmailDocumentCollection)

	doc := &EmailDocument{}
	docResult := coll.FindOne(ctx,
		bson.NewDocument(
			bson.EC.String("eid", eid),
		))
	if err := docResult.Decode(doc); err != nil {
		if err == mongo.ErrNoDocuments {
			return nil, &errNoDocuments{
				error: err,
			}
		}
		return nil, err
	}

	return doc, nil
}

// EmailDocumentWhere query condition
type EmailDocumentWhere struct {
	StartTime time.Time
	EndTime   time.Time

	// pagination info
	Limit  int64
	LastID *objectid.ObjectID
}

// EmailDocumentByWhere gets pagination list of email document by condition from the db
func EmailDocumentByWhere(ctx context.Context, db *mongo.Database, where EmailDocumentWhere) ([]*EmailDocument, error) {
	coll := db.Collection(EmailDocumentCollection)
	condition := bson.NewDocument()

	whereDoc := bson.NewDocument()
	if !where.StartTime.IsZero() {
		whereDoc.Append(bson.EC.Time("$gte", where.StartTime))
	}
	if !where.EndTime.IsZero() {
		whereDoc.Append(bson.EC.Time("$lt", where.EndTime))
	}
	if !where.StartTime.IsZero() || !where.EndTime.IsZero() {
		condition.Append(
			bson.EC.SubDocument("sendDate", whereDoc),
		)
	}

	if where.LastID != nil {
		condition.Append(
			bson.EC.SubDocumentFromElements("_id",
				bson.EC.ObjectID("$gt", *where.LastID),
			))
	}

	if where.Limit == 0 {
		where.Limit = DefaultLimit
	}

	cursor, err := coll.Find(ctx, condition,
		findopt.Limit(where.Limit),
		findopt.Sort(map[string]int32{
			"sendDate": -1,
		}))
	if err != nil {
		return nil, errors.Wrap(err, "failed to query email documents")
	}
	defer cursor.Close(ctx)

	var docs []*EmailDocument
	for cursor.Next(ctx) {
		doc := &EmailDocument{}
		err := cursor.Decode(doc)
		if err != nil {
			return nil, err
		}
		docs = append(docs, doc)
	}

	return docs, nil
}

// CountEmailDocumentByWhere count email documents by condition from the db
func CountEmailDocumentByWhere(ctx context.Context, db *mongo.Database, where EmailDocumentWhere) (int64, error) {
	coll := db.Collection(EmailDocumentCollection)
	condition := bson.NewDocument()

	whereDoc := bson.NewDocument()
	if !where.StartTime.IsZero() {
		whereDoc.Append(bson.EC.Time("$gte", where.StartTime))
	}
	if !where.EndTime.IsZero() {
		whereDoc.Append(bson.EC.Time("$lt", where.EndTime))
	}
	if !where.StartTime.IsZero() || !where.EndTime.IsZero() {
		condition.Append(
			bson.EC.SubDocument("sendDate", whereDoc),
		)
	}

	total, err := coll.Count(ctx, condition)
	if err != nil {
		return 0, err
	}

	return total, nil
}

// EmailDocumentCreateIndexes create indexes to optimize the query.
func EmailDocumentCreateIndexes(ctx context.Context, db *mongo.Database) error {
	coll := db.Collection(EmailDocumentCollection)

	_, err := coll.Indexes().CreateOne(ctx,
		mongo.IndexModel{
			Keys: bson.NewDocument(
				bson.EC.Int32("sendDate", -1),
			),
		},
	)
	if err != nil {
		return errors.Wrap(err, "failed to create index")
	}

	return nil
}

// Collection return collection name
func (doc *EmailDocument) Collection() string {
	return EmailDocumentCollection
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
)

// TestCLI tests the "convert" and "validate" subcommand against a generated .tfplan file.
func TestCLI(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode.")
		return
	}
	// Define the reusable constraints to be use for the test cases.
	type constraint struct {
		name            string
		wantViolation   bool
		wantOutputRegex string
	}
	// Currently, we only test one rule. Moving forward, resource specific rules
	// should be added to increase the coverage.
	alwaysViolate := constraint{name: "always_violate", wantViolation: true, wantOutputRegex: "Constraint GCPAlwaysViolatesConstraintV1.always_violates_all on resource"}

	// Test cases for each type of resource is defined here.
	cases := []struct {
		name        string
		constraints []constraint
		compareConvertOutput compareConvertOutputFunc
	}{
		{name: "bucket"},
		{name: "bucket_iam"},
		{name: "disk"},
		{name: "firewall"},
		{name: "instance"},
		{name: "sql"},
		{name: "example_bigquery_dataset"},
		{name: "example_compute_disk"},
		{name: "example_compute_firewall"},
		{name: "example_compute_instance"},
		{name: "example_container_cluster"},
		{name: "example_organization_iam_binding"},
		{name: "example_organization_iam_member"},
		{name: "example_organization_iam_policy"},
		{name: "example_pubsub_topic"},
		{name: "example_project"},
		{name: "example_project_in_org"},
		{name: "example_project_in_folder"},
		{name: "example_project_iam"},
		{name: "example_project_iam_binding"},
		{name: "example_project_iam_member"},
		{name: "example_project_iam_policy"},
		{name: "example_project_service"},
		{name: "example_sql_database_instance"},
		{name: "example_storage_bucket"},
		{name: "full_compute_firewall"},
		{name: "full_compute_instance"},
		{name: "full_container_cluster"},
		{name: "full_container_node_pool"},
		{name: "full_sql_database_instance"},
		{name: "full_storage_bucket"},
	}

	// Map of cases to skip to reasons for the skip
	skipCases := map[string]string{
		"TestCLI/v=0.12/tf=example_compute_instance/offline=true/cmd=convert":                            "compute_instance doesn't work in offline mode - github.com/hashicorp/terraform-provider-google/issues/8489",
		"TestCLI/v=0.12/tf=example_compute_instance/offline=true/cmd=validate/constraint=always_violate": "compute_instance doesn't work in offline mode - github.com/hashicorp/terraform-provider-google/issues/8489",
	}
	for i := range cases {
		// Allocate a variable to make sure test can run in parallel.
		c := cases[i]
		// Add default constraints if not set.
		if len(c.constraints) == 0 {
			c.constraints = []constraint{alwaysViolate}
		}

		// Add default convert comparison func if not set
		if c.compareConvertOutput == nil {
			c.compareConvertOutput = compareUnmergedConvertOutput
		}

		// Test both offline and online mode.
		for _, offline := range []bool{true, false} {
			offline := offline
			t.Run(fmt.Sprintf("v=0.12/tf=%s/offline=%t", c.name, offline), func(t *testing.T) {
				t.Parallel()
				// Create a temporary directory for running terraform.
				dir, err := ioutil.TempDir(tmpDir, "terraform")
				if err != nil {
					log.Fatal(err)
				}
				defer os.RemoveAll(dir)

				// Generate the <name>.tf and <name>_assets.json files into the temporary directory.
				generateTestFiles(t, "../testdata/templates", dir, c.name+".tf")
				generateTestFiles(t, "../testdata/templates", dir, c.name+".json")

				terraform(t, dir, c.name)

				t.Run("cmd=convert", func(t *testing.T) {
					if reason, exists := skipCases[t.Name()]; exists {
						t.Skip(reason)
					}
					testConvertCommand(t, dir, c.name, offline, c.compareConvertOutput)
				})

				for _, ct := range c.constraints {
					t.Run(fmt.Sprintf("cmd=validate/constraint=%s", ct.name), func(t *testing.T) {
						if reason, exists := skipCases[t.Name()]; exists {
							t.Skip(reason)
						}
						testValidateCommand(t, ct.wantViolation, ct.wantOutputRegex, dir, c.name, offline, ct.name)
					})
				}
			})
		}
	}
}

type compareConvertOutputFunc func(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool)

func compareUnmergedConvertOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
	expectedJSON := normalizeAssets(t, expected, offline)
	actualJSON := normalizeAssets(t, actual, offline)
	require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

func testConvertCommand(t *testing.T, dir, name string, offline bool, compare compareConvertOutputFunc) {
	var payload []byte

	// Load expected assets
	testfile := filepath.Join(dir, name+".json")
	payload, err := ioutil.ReadFile(testfile)
	if err != nil {
		t.Fatalf("Error reading %v: %v", testfile, err)
	}
	var expected []google.Asset
	if err := json.Unmarshal(payload, &expected); err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	// Get converted assets
	payload = tfvConvert(t, dir, name+".tfplan.json", offline)
	var actual []google.Asset
	err = json.Unmarshal(payload, &actual)
	if err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	compare(t, expected, actual, offline)
}

func testValidateCommand(t *testing.T, wantViolation bool, want, dir, name string, offline bool, constraintName string) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatalf("cannot get current directory: %v", err)
	}
	policyPath := filepath.Join(cwd, samplePolicyPath, constraintName)
	var got []byte
	got = tfvValidate(t, wantViolation, dir, name+".tfplan.json", policyPath, offline)
	wantRe := regexp.MustCompile(want)
	if want != "" && !wantRe.Match(got) {
		t.Fatalf("binary did not return expect output, \ngot=%s \nwant (regex)=%s", string(got), want)
	}
}

func terraform(t *testing.T, dir, name string) {
	terraformInit(t, "terraform", dir)
	terraformPlan(t, "terraform", dir, name+".tfplan")
	payload := terraformShow(t, "terraform", dir, name+".tfplan")
	saveFile(t, dir, name+".tfplan.json", payload)
}

func terraformInit(t *testing.T, executable, dir string) {
	terraformExec(t, executable, dir, "init", "-input=false")
}

func terraformPlan(t *testing.T, executable, dir, tfplan string) {
	terraformExec(t, executable, dir, "plan", "-input=false", "--out", tfplan)
}

func terraformShow(t *testing.T, executable, dir, tfplan string) []byte {
	return terraformExec(t, executable, dir, "show", "--json", tfplan)
}

func terraformExec(t *testing.T, executable, dir string, args ...string) []byte {
	cmd := exec.Command(executable, args...)
	cmd.Env = []string{"HOME=" + filepath.Join(dir, "fakehome")}
	cmd.Dir = dir
	wantError := false
	payload, _ := run(t, cmd, wantError)
	return payload
}

func saveFile(t *testing.T, dir, filename string, payload []byte) {
	fullpath := filepath.Join(dir, filename)
	f, err := os.Create(fullpath)
	if err != nil {
		t.Fatalf("error while creating file %s, error %v", fullpath, err)
	}
	_, err = f.Write(payload)
	if err != nil {
		t.Fatalf("error while writing to file %s, error %v", fullpath, err)
	}
}

func tfvConvert(t *testing.T, dir, tfplan string, offline bool) []byte {
	executable := tfvBinary
	wantError := false
	args := []string{"convert", "--project", data.Provider["project"]}
	if offline {
		args = append(args, "--offline", "--ancestry", data.Ancestry)
	}
	args = append(args, tfplan)
	cmd := exec.Command(executable, args...)
	// Remove environment variables inherited from the test runtime.
	cmd.Env = []string{}
	// Add credentials back.
	if data.Provider["credentials"] != "" {
		cmd.Env = append(cmd.Env, "GOOGLE_APPLICATION_CREDENTIALS="+data.Provider["credentials"])
	}
	cmd.Dir = dir
	payload, _ := run(t, cmd, wantError)
	return payload
}

func tfvValidate(t *testing.T, wantError bool, dir, tfplan, policyPath string, offline bool) []byte {
	executable := tfvBinary
	args := []string{"validate", "--project", data.Provider["project"], "--policy-path", policyPath}
	if offline {
		args = append(args, "--offline", "--ancestry", data.Ancestry)
	}
	args = append(args, tfplan)
	cmd := exec.Command(executable, args...)
	cmd.Env = []string{"GOOGLE_APPLICATION_CREDENTIALS=" + data.Provider["credentials"]}
	cmd.Dir = dir
	payload, _ := run(t, cmd, wantError)
	return payload
}

// run a command and call t.Fatal on non-zero exit.
func run(t *testing.T, cmd *exec.Cmd, wantError bool) ([]byte, []byte) {
	var stderr, stdout bytes.Buffer
	cmd.Stderr, cmd.Stdout = &stderr, &stdout
	err := cmd.Run()
	if gotError := (err != nil); gotError != wantError {
		t.Fatalf("running %s: \nerror=%v \nstderr=%s \nstdout=%s", cmdToString(cmd), err, stderr.String(), stdout.String())
	}
	// Print env, stdout and stderr if verbose flag is used.
	if len(cmd.Env) != 0 {
		t.Logf("=== Environment Variable of %s ===", cmdToString(cmd))
		t.Log(strings.Join(cmd.Env, "\n"))
	}
	if stdout.String() != "" {
		t.Logf("=== STDOUT of %s ===", cmdToString(cmd))
		t.Log(stdout.String())
	}
	if stderr.String() != "" {
		t.Logf("=== STDERR of %s ===", cmdToString(cmd))
		t.Log(stderr.String())
	}
	return stdout.Bytes(), stderr.Bytes()
}

// cmdToString clones the logic of https://golang.org/pkg/os/exec/#Cmd.String.
func cmdToString(c *exec.Cmd) string {
	// report the exact executable path (plus args)
	b := new(strings.Builder)
	b.WriteString(c.Path)
	for _, a := range c.Args[1:] {
		b.WriteByte(' ')
		b.WriteString(a)
	}
	return b.String()
}
<commit_msg>Made project_iam_member tests only consider whether expected members are present<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
)

// TestCLI tests the "convert" and "validate" subcommand against a generated .tfplan file.
func TestCLI(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode.")
		return
	}
	// Define the reusable constraints to be use for the test cases.
	type constraint struct {
		name            string
		wantViolation   bool
		wantOutputRegex string
	}
	// Currently, we only test one rule. Moving forward, resource specific rules
	// should be added to increase the coverage.
	alwaysViolate := constraint{name: "always_violate", wantViolation: true, wantOutputRegex: "Constraint GCPAlwaysViolatesConstraintV1.always_violates_all on resource"}

	// Test cases for each type of resource is defined here.
	cases := []struct {
		name                 string
		constraints          []constraint
		compareConvertOutput compareConvertOutputFunc
	}{
		{name: "bucket"},
		{name: "bucket_iam"},
		{name: "disk"},
		{name: "firewall"},
		{name: "instance"},
		{name: "sql"},
		{name: "example_bigquery_dataset"},
		{name: "example_compute_disk"},
		{name: "example_compute_firewall"},
		{name: "example_compute_instance"},
		{name: "example_container_cluster"},
		{name: "example_organization_iam_binding"},
		{name: "example_organization_iam_member"},
		{name: "example_organization_iam_policy"},
		{name: "example_pubsub_topic"},
		{name: "example_project"},
		{name: "example_project_in_org"},
		{name: "example_project_in_folder"},
		{name: "example_project_iam"},
		{name: "example_project_iam_binding"},
		{name: "example_project_iam_member", compareConvertOutput: compareMergedIamMemberOutput},
		{name: "example_project_iam_policy"},
		{name: "example_project_service"},
		{name: "example_sql_database_instance"},
		{name: "example_storage_bucket"},
		{name: "full_compute_firewall"},
		{name: "full_compute_instance"},
		{name: "full_container_cluster"},
		{name: "full_container_node_pool"},
		{name: "full_sql_database_instance"},
		{name: "full_storage_bucket"},
	}

	// Map of cases to skip to reasons for the skip
	skipCases := map[string]string{
		"TestCLI/v=0.12/tf=example_compute_instance/offline=true/cmd=convert":                            "compute_instance doesn't work in offline mode - github.com/hashicorp/terraform-provider-google/issues/8489",
		"TestCLI/v=0.12/tf=example_compute_instance/offline=true/cmd=validate/constraint=always_violate": "compute_instance doesn't work in offline mode - github.com/hashicorp/terraform-provider-google/issues/8489",
	}
	for i := range cases {
		// Allocate a variable to make sure test can run in parallel.
		c := cases[i]
		// Add default constraints if not set.
		if len(c.constraints) == 0 {
			c.constraints = []constraint{alwaysViolate}
		}

		// Add default convert comparison func if not set
		if c.compareConvertOutput == nil {
			c.compareConvertOutput = compareUnmergedConvertOutput
		}

		// Test both offline and online mode.
		for _, offline := range []bool{true, false} {
			offline := offline
			t.Run(fmt.Sprintf("v=0.12/tf=%s/offline=%t", c.name, offline), func(t *testing.T) {
				t.Parallel()
				// Create a temporary directory for running terraform.
				dir, err := ioutil.TempDir(tmpDir, "terraform")
				if err != nil {
					log.Fatal(err)
				}
				defer os.RemoveAll(dir)

				// Generate the <name>.tf and <name>_assets.json files into the temporary directory.
				generateTestFiles(t, "../testdata/templates", dir, c.name+".tf")
				generateTestFiles(t, "../testdata/templates", dir, c.name+".json")

				terraform(t, dir, c.name)

				t.Run("cmd=convert", func(t *testing.T) {
					if reason, exists := skipCases[t.Name()]; exists {
						t.Skip(reason)
					}
					testConvertCommand(t, dir, c.name, offline, c.compareConvertOutput)
				})

				for _, ct := range c.constraints {
					t.Run(fmt.Sprintf("cmd=validate/constraint=%s", ct.name), func(t *testing.T) {
						if reason, exists := skipCases[t.Name()]; exists {
							t.Skip(reason)
						}
						testValidateCommand(t, ct.wantViolation, ct.wantOutputRegex, dir, c.name, offline, ct.name)
					})
				}
			})
		}
	}
}

type compareConvertOutputFunc func(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool)

func compareUnmergedConvertOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
	expectedJSON := normalizeAssets(t, expected, offline)
	actualJSON := normalizeAssets(t, actual, offline)
	require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

// For merged IAM members, only consider whether the expected members are present.
func compareMergedIamMemberOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
	var normalizedActual []google.Asset
	for i := range expected {
		expectedAsset := expected[i]
		actualAsset := expected[i]

		normalizedActualAsset := google.Asset{
			Name:     actualAsset.Name,
			Type:     actualAsset.Type,
			Ancestry: "",
		}

		expectedBindings := map[string]map[string]struct{}{}
		for _, binding := range expectedAsset.IAMPolicy.Bindings {
			expectedBindings[binding.Role] = map[string]struct{}{}
			for _, member := range binding.Members {
				expectedBindings[binding.Role][member] = struct{}{}
			}
		}

		iamPolicy := google.IAMPolicy{}
		for _, binding := range actualAsset.IAMPolicy.Bindings {
			if expectedMembers, exists := expectedBindings[binding.Role]; exists {
				iamBinding := google.IAMBinding{
					Role: binding.Role,
				}
				for _, member := range binding.Members {
					if _, exists := expectedMembers[member]; exists {
						iamBinding.Members = append(iamBinding.Members, member)
					}
				}
				iamPolicy.Bindings = append(iamPolicy.Bindings, iamBinding)
			}
		}
		normalizedActualAsset.IAMPolicy = &iamPolicy
		normalizedActual = append(normalizedActual, normalizedActualAsset)
	}

	expectedJSON := normalizeAssets(t, expected, offline)
	actualJSON := normalizeAssets(t, normalizedActual, offline)
	require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

func testConvertCommand(t *testing.T, dir, name string, offline bool, compare compareConvertOutputFunc) {
	var payload []byte

	// Load expected assets
	testfile := filepath.Join(dir, name+".json")
	payload, err := ioutil.ReadFile(testfile)
	if err != nil {
		t.Fatalf("Error reading %v: %v", testfile, err)
	}
	var expected []google.Asset
	if err := json.Unmarshal(payload, &expected); err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	// Get converted assets
	payload = tfvConvert(t, dir, name+".tfplan.json", offline)
	var actual []google.Asset
	err = json.Unmarshal(payload, &actual)
	if err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	compare(t, expected, actual, offline)
}

func testValidateCommand(t *testing.T, wantViolation bool, want, dir, name string, offline bool, constraintName string) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatalf("cannot get current directory: %v", err)
	}
	policyPath := filepath.Join(cwd, samplePolicyPath, constraintName)
	var got []byte
	got = tfvValidate(t, wantViolation, dir, name+".tfplan.json", policyPath, offline)
	wantRe := regexp.MustCompile(want)
	if want != "" && !wantRe.Match(got) {
		t.Fatalf("binary did not return expect output, \ngot=%s \nwant (regex)=%s", string(got), want)
	}
}

func terraform(t *testing.T, dir, name string) {
	terraformInit(t, "terraform", dir)
	terraformPlan(t, "terraform", dir, name+".tfplan")
	payload := terraformShow(t, "terraform", dir, name+".tfplan")
	saveFile(t, dir, name+".tfplan.json", payload)
}

func terraformInit(t *testing.T, executable, dir string) {
	terraformExec(t, executable, dir, "init", "-input=false")
}

func terraformPlan(t *testing.T, executable, dir, tfplan string) {
	terraformExec(t, executable, dir, "plan", "-input=false", "--out", tfplan)
}

func terraformShow(t *testing.T, executable, dir, tfplan string) []byte {
	return terraformExec(t, executable, dir, "show", "--json", tfplan)
}

func terraformExec(t *testing.T, executable, dir string, args ...string) []byte {
	cmd := exec.Command(executable, args...)
	cmd.Env = []string{"HOME=" + filepath.Join(dir, "fakehome")}
	cmd.Dir = dir
	wantError := false
	payload, _ := run(t, cmd, wantError)
	return payload
}

func saveFile(t *testing.T, dir, filename string, payload []byte) {
	fullpath := filepath.Join(dir, filename)
	f, err := os.Create(fullpath)
	if err != nil {
		t.Fatalf("error while creating file %s, error %v", fullpath, err)
	}
	_, err = f.Write(payload)
	if err != nil {
		t.Fatalf("error while writing to file %s, error %v", fullpath, err)
	}
}

func tfvConvert(t *testing.T, dir, tfplan string, offline bool) []byte {
	executable := tfvBinary
	wantError := false
	args := []string{"convert", "--project", data.Provider["project"]}
	if offline {
		args = append(args, "--offline", "--ancestry", data.Ancestry)
	}
	args = append(args, tfplan)
	cmd := exec.Command(executable, args...)
	// Remove environment variables inherited from the test runtime.
	cmd.Env = []string{}
	// Add credentials back.
	if data.Provider["credentials"] != "" {
		cmd.Env = append(cmd.Env, "GOOGLE_APPLICATION_CREDENTIALS="+data.Provider["credentials"])
	}
	cmd.Dir = dir
	payload, _ := run(t, cmd, wantError)
	return payload
}

func tfvValidate(t *testing.T, wantError bool, dir, tfplan, policyPath string, offline bool) []byte {
	executable := tfvBinary
	args := []string{"validate", "--project", data.Provider["project"], "--policy-path", policyPath}
	if offline {
		args = append(args, "--offline", "--ancestry", data.Ancestry)
	}
	args = append(args, tfplan)
	cmd := exec.Command(executable, args...)
	cmd.Env = []string{"GOOGLE_APPLICATION_CREDENTIALS=" + data.Provider["credentials"]}
	cmd.Dir = dir
	payload, _ := run(t, cmd, wantError)
	return payload
}

// run a command and call t.Fatal on non-zero exit.
func run(t *testing.T, cmd *exec.Cmd, wantError bool) ([]byte, []byte) {
	var stderr, stdout bytes.Buffer
	cmd.Stderr, cmd.Stdout = &stderr, &stdout
	err := cmd.Run()
	if gotError := (err != nil); gotError != wantError {
		t.Fatalf("running %s: \nerror=%v \nstderr=%s \nstdout=%s", cmdToString(cmd), err, stderr.String(), stdout.String())
	}
	// Print env, stdout and stderr if verbose flag is used.
	if len(cmd.Env) != 0 {
		t.Logf("=== Environment Variable of %s ===", cmdToString(cmd))
		t.Log(strings.Join(cmd.Env, "\n"))
	}
	if stdout.String() != "" {
		t.Logf("=== STDOUT of %s ===", cmdToString(cmd))
		t.Log(stdout.String())
	}
	if stderr.String() != "" {
		t.Logf("=== STDERR of %s ===", cmdToString(cmd))
		t.Log(stderr.String())
	}
	return stdout.Bytes(), stderr.Bytes()
}

// cmdToString clones the logic of https://golang.org/pkg/os/exec/#Cmd.String.
func cmdToString(c *exec.Cmd) string {
	// report the exact executable path (plus args)
	b := new(strings.Builder)
	b.WriteString(c.Path)
	for _, a := range c.Args[1:] {
		b.WriteByte(' ')
		b.WriteString(a)
	}
	return b.String()
}
<|endoftext|>