<commit_before>package generatorControllers

import (
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
	"strings"
	"sync"
	"time"
)

type TrendLinePlotSummary struct {
	*BaseController
}

type LatestTrendLine struct {
	ID          string ` bson:"_id" , json:"_id" `
	Projectname string
	LastUpdate  time.Time
	Type        string
}

func (m *LatestTrendLine) TableName() string {
	return "log_latesttrendline"
}

func (ev *TrendLinePlotSummary) CreateTrendLinePlotSummary(base *BaseController) {
	ev.BaseController = base

	ev.Log.AddLog("===================== Start processing Trend Line Plots Summary...", sInfo)

	var wg sync.WaitGroup
	wg.Add(2)

	go ev.processDataScada(&wg)
	go ev.processDataMet(&wg)

	wg.Wait()

	ev.Log.AddLog("===================== End processing Trend Line Plots Summary...", sInfo)
}

func (ev *TrendLinePlotSummary) getProjectList() (result []string) {
	ev.Log.AddLog("Get Project List", sInfo)

	projectData := []tk.M{}
	csrt, e := ev.Ctx.Connection.NewQuery().
		From("ref_project").Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getProjectList due to : %s", e.Error()), sError)
		return
	}
	defer csrt.Close()
	e = csrt.Fetch(&projectData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch at getProjectList due to : %s", e.Error()), sError)
		return
	}
	result = []string{}
	for _, val := range projectData {
		result = append(result, val.GetString("projectid"))
	}
	ev.Log.AddLog("Finish getting Project List", sInfo)

	return
}

func (ev *TrendLinePlotSummary) getTemperatureField() (result map[string][]string) {
	csr, e := ev.Ctx.Connection.NewQuery().
		From("ref_databrowsertag").
		Order("projectname", "label").
		Cursor(nil)
	defer csr.Close()

	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getTemperatureField due to : %s", e.Error()), sError)
		return
	}

	_data := tk.M{}
	lastProject := ""
	currProject := ""
	tempList := []string{}
	result = map[string][]string{}
	for {
		_data = tk.M{}
		e = csr.Fetch(&_data, 1, false)
		if e != nil {
			break
		}
		currProject = _data.GetString("projectname")
		if lastProject != currProject {
			if lastProject != "" {
				result[lastProject] = tempList
				tempList = []string{}
			}
			lastProject = currProject
		}
		if strings.Contains(strings.ToLower(_data.GetString("realtimefield")), "temp") {
			tempList = append(tempList, strings.ToLower(_data.GetString("realtimefield")))
		}
	}
	if lastProject != "" {
		result[lastProject] = tempList
	}

	return
}

func (ev *TrendLinePlotSummary) getLatestData(tipe string) (result map[string]time.Time) {
	ev.Log.AddLog("Get latest data for each turbine", sInfo)

	latestData := []LatestTrendLine{}
	csrt, e := ev.Ctx.Connection.NewQuery().
		From(new(LatestTrendLine).TableName()).
		Where(dbox.Eq("type", tipe)).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getLatestData due to : %s", e.Error()), sError)
		return
	}
	defer csrt.Close()
	e = csrt.Fetch(&latestData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch at getLatestData due to : %s", e.Error()), sError)
		return
	}
	result = map[string]time.Time{}
	for _, val := range latestData {
		result[val.Projectname] = val.LastUpdate
	}
	ev.Log.AddLog("Finish getting latest data for each turbine", sInfo)

	return
}

func (ev *TrendLinePlotSummary) updateLastData(projectname, tipe string, maxTimeStamp time.Time) {
	if !maxTimeStamp.IsZero() {
		data := LatestTrendLine{}
		data.Projectname = projectname
		data.ID = tk.Sprintf("%s_%s", data.Projectname, tipe)
		data.LastUpdate = maxTimeStamp
		data.Type = tipe

		e := ev.Ctx.Connection.NewQuery().SetConfig("multiexec", true).
			From(new(LatestTrendLine).TableName()).Save().Exec(tk.M{"data": data})

		if e != nil {
			ev.Log.AddLog(tk.Sprintf("Error on Save at updateLastData due to : %s", e.Error()), sError)
		}
	}
	ev.Log.AddLog(tk.Sprintf("Finish updating last data for %s on %s at %s", projectname, tipe, maxTimeStamp.String()), sInfo)
}

func (ev *TrendLinePlotSummary) processDataScada(wgScada *sync.WaitGroup) {
	defer wgScada.Done()

	t0 := time.Now()
	projectList := ev.getProjectList()
	lastUpdatePerProject := ev.getLatestData("SCADA")
	temperatureList := ev.getTemperatureField()

	var wg sync.WaitGroup
	wg.Add(len(projectList))
	for _, _project := range projectList {
		go ev.projectWorker(_project, lastUpdatePerProject[_project], temperatureList[_project], &wg)
	}
	wg.Wait()

	ev.Log.AddLog(tk.Sprintf("Duration processing scada data %f minutes", time.Since(t0).Minutes()), sInfo)
}

func (ev *TrendLinePlotSummary) projectWorker(projectname string, lastUpdate time.Time, tempList []string, wgProject *sync.WaitGroup) {
	defer wgProject.Done()

	groups := tk.M{"_id": tk.M{
		"turbine":   "$turbine",
		"timestamp": "$dateinfo.dateid",
	}}
	for _, field := range tempList {
		totalName := field + "total"
		countName := field + "count"
		fieldName := "$" + field

		countCondition := tk.M{"$cond": tk.M{}.
			Set("if", tk.M{"$ifNull": []interface{}{fieldName, false}}).
			Set("then", 1).
			Set("else", 0)}
		groups.Set(totalName, tk.M{"$sum": fieldName})
		groups.Set(countName, tk.M{"$sum": countCondition})
	}

	ev.Log.AddLog(tk.Sprintf("Update data %s from %s", projectname, lastUpdate.String()), sInfo)
	pipe := []tk.M{
		tk.M{"$match": tk.M{
			"$and": []tk.M{
				tk.M{"dateinfo.dateid": tk.M{"$gte": lastUpdate}},
				tk.M{"projectname": projectname},
				tk.M{"isnull": false},
			},
		}},
		tk.M{"$group": groups},
	}

	csr, e := ev.Ctx.Connection.NewQuery().
		From("Scada10MinHFD").
		Command("pipe", pipe).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor : %s", e.Error()), sError)
	}
	defer csr.Close()

	trendLineData := []tk.M{}
	e = csr.Fetch(&trendLineData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch : %s", e.Error()), sError)
	}

	var wg sync.WaitGroup
	totalData := len(trendLineData)
	totalWorker := 4
	dataChan := make(chan tk.M, totalData)

	wg.Add(totalWorker)
	for i := 0; i < totalWorker; i++ {
		go func() {
			defer wg.Done()
			ctxWorker, e := PrepareConnection()
			if e != nil {
				ev.Log.AddLog(e.Error(), sError)
			}
			defer ctxWorker.Close()
			csrSave := ctxWorker.NewQuery().SetConfig("multiexec", true).
				From("rpt_trendlineplot").Save()
			defer csrSave.Close()
			for data := range dataChan {
				e = csrSave.Exec(tk.M{"data": data})
				if e != nil {
					ev.Log.AddLog(tk.Sprintf("Error on Save : %s", e.Error()), sError)
				}
			}
		}()
	}

	data := tk.M{}
	maxTimeStamp := time.Time{}

	for _, _data := range trendLineData {
		data = tk.M{}
		ids := _data.Get("_id", tk.M{}).(tk.M)
		timestamp := ids.Get("timestamp", time.Time{}).(time.Time).UTC()
		data.Set("projectname", projectname)
		data.Set("turbine", ids.GetString("turbine"))
		data.Set("timestamp", timestamp)
		data.Set("_id", tk.Sprintf("%s_%s_%s", projectname, ids.GetString("turbine"), timestamp.Format("20060102")))
		for _dataKey, _dataVal := range _data {
			data.Set(_dataKey, _dataVal)
		}
		data.Set("type", "SCADA")

		if timestamp.After(maxTimeStamp) {
			maxTimeStamp = timestamp
		}

		dataChan <- data
	}

	close(dataChan)
	wg.Wait()

	ev.updateLastData(projectname, "SCADA", maxTimeStamp)
}

func (ev *TrendLinePlotSummary) projectWorkerMet(projectname string, lastupdate time.Time, wgProject *sync.WaitGroup) {
	defer wgProject.Done()

	pipe := []tk.M{
		tk.M{"$match": tk.M{
			"$and": []tk.M{
				tk.M{"dateinfo.dateid": tk.M{"$gte": lastupdate}},
				tk.M{"projectname": projectname},
			},
		}},
	}
	fieldName := "$trefhrefhumid855mavg"
	if lastupdate.Before(time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)) {
		fieldName = "$trefhreftemp855mavg"
	}
	countCondition := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{"$ifNull": []interface{}{fieldName, false}}).
		Set("then", 1).
		Set("else", 0)}
	pipe = append(pipe, tk.M{"$group": tk.M{
		"_id":              tk.M{"timestamp": "$dateinfo.dateid"},
		"tempoutdoortotal": tk.M{"$sum": fieldName},
		"tempoutdoorcount": tk.M{"$sum": countCondition},
	}})

	trendLineData := []tk.M{}
	csr, e := ev.Ctx.Connection.NewQuery().
		From("MetTower").
		Command("pipe", pipe).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor : %s", e.Error()), sError)
	}
	defer csr.Close()

	e = csr.Fetch(&trendLineData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on Fetch : %s", e.Error()), sError)
	}

	var wg sync.WaitGroup
	totalData := len(trendLineData)
	totalWorker := 4
	dataChan := make(chan tk.M, totalData)

	wg.Add(totalWorker)
	for i := 0; i < totalWorker; i++ {
		go func() {
			defer wg.Done()
			ctxWorker, e := PrepareConnection()
			if e != nil {
				ev.Log.AddLog(e.Error(), sError)
			}
			defer ctxWorker.Close()
			csrSave := ctxWorker.NewQuery().SetConfig("multiexec", true).
				From("rpt_trendlineplot").Save()
			defer csrSave.Close()
			for data := range dataChan {
				e = csrSave.Exec(tk.M{"data": data})
				if e != nil {
					ev.Log.AddLog(tk.Sprintf("Error on Save : %s", e.Error()), sError)
				}
			}
		}()
	}

	data := tk.M{}
	maxTimeStamp := time.Time{}

	for _, _data := range trendLineData {
		data = tk.M{}
		ids := _data.Get("_id", tk.M{}).(tk.M)
		timestamp := ids.Get("timestamp", time.Time{}).(time.Time).UTC()
		data.Set("projectname", projectname)
		data.Set("timestamp", timestamp)
		data.Set("_id", tk.Sprintf("%s_%s", projectname, timestamp.Format("20060102")))
		for _dataKey, _dataVal := range _data {
			data.Set(_dataKey, _dataVal)
		}
		data.Set("type", "MET")

		if timestamp.After(maxTimeStamp) {
			maxTimeStamp = timestamp
		}

		dataChan <- data
	}

	close(dataChan)
	wg.Wait()

	ev.updateLastData(projectname, "MET", maxTimeStamp)
}

func (ev *TrendLinePlotSummary) processDataMet(wgMet *sync.WaitGroup) {
	defer wgMet.Done()

	t0 := time.Now()

	projectList := ev.getProjectList()
	lastUpdatePerProject := ev.getLatestData("MET")

	var wg sync.WaitGroup
	wg.Add(len(projectList))
	for _, _project := range projectList {
		go ev.projectWorkerMet(_project, lastUpdatePerProject[_project], &wg)
	}
	wg.Wait()

	ev.Log.AddLog(tk.Sprintf("Duration process met tower data %f minutes", time.Since(t0).Minutes()), sInfo)
}
<commit_msg>fixing _id format<commit_after>package generatorControllers

import (
	. "eaciit/wfdemo-git/processapp/summaryGenerator/controllers"
	"github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/mongo"
	tk "github.com/eaciit/toolkit"
	"strings"
	"sync"
	"time"
)

type TrendLinePlotSummary struct {
	*BaseController
}

type LatestTrendLine struct {
	ID          string ` bson:"_id" , json:"_id" `
	Projectname string
	LastUpdate  time.Time
	Type        string
}

func (m *LatestTrendLine) TableName() string {
	return "log_latesttrendline"
}

func (ev *TrendLinePlotSummary) CreateTrendLinePlotSummary(base *BaseController) {
	ev.BaseController = base

	ev.Log.AddLog("===================== Start processing Trend Line Plots Summary...", sInfo)

	var wg sync.WaitGroup
	wg.Add(2)

	go ev.processDataScada(&wg)
	go ev.processDataMet(&wg)

	wg.Wait()

	ev.Log.AddLog("===================== End processing Trend Line Plots Summary...", sInfo)
}

func (ev *TrendLinePlotSummary) getProjectList() (result []string) {
	ev.Log.AddLog("Get Project List", sInfo)

	projectData := []tk.M{}
	csrt, e := ev.Ctx.Connection.NewQuery().
		From("ref_project").Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getProjectList due to : %s", e.Error()), sError)
		return
	}
	defer csrt.Close()
	e = csrt.Fetch(&projectData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch at getProjectList due to : %s", e.Error()), sError)
		return
	}
	result = []string{}
	for _, val := range projectData {
		result = append(result, val.GetString("projectid"))
	}
	ev.Log.AddLog("Finish getting Project List", sInfo)

	return
}

func (ev *TrendLinePlotSummary) getTemperatureField() (result map[string][]string) {
	csr, e := ev.Ctx.Connection.NewQuery().
		From("ref_databrowsertag").
		Order("projectname", "label").
		Cursor(nil)
	defer csr.Close()

	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getTemperatureField due to : %s", e.Error()), sError)
		return
	}

	_data := tk.M{}
	lastProject := ""
	currProject := ""
	tempList := []string{}
	result = map[string][]string{}
	for {
		_data = tk.M{}
		e = csr.Fetch(&_data, 1, false)
		if e != nil {
			break
		}
		currProject = _data.GetString("projectname")
		if lastProject != currProject {
			if lastProject != "" {
				result[lastProject] = tempList
				tempList = []string{}
			}
			lastProject = currProject
		}
		if strings.Contains(strings.ToLower(_data.GetString("realtimefield")), "temp") {
			tempList = append(tempList, strings.ToLower(_data.GetString("realtimefield")))
		}
	}
	if lastProject != "" {
		result[lastProject] = tempList
	}

	return
}

func (ev *TrendLinePlotSummary) getLatestData(tipe string) (result map[string]time.Time) {
	ev.Log.AddLog("Get latest data for each turbine", sInfo)

	latestData := []LatestTrendLine{}
	csrt, e := ev.Ctx.Connection.NewQuery().
		From(new(LatestTrendLine).TableName()).
		Where(dbox.Eq("type", tipe)).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor at getLatestData due to : %s", e.Error()), sError)
		return
	}
	defer csrt.Close()
	e = csrt.Fetch(&latestData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch at getLatestData due to : %s", e.Error()), sError)
		return
	}
	result = map[string]time.Time{}
	for _, val := range latestData {
		result[val.Projectname] = val.LastUpdate
	}
	ev.Log.AddLog("Finish getting latest data for each turbine", sInfo)

	return
}

func (ev *TrendLinePlotSummary) updateLastData(projectname, tipe string, maxTimeStamp time.Time) {
	if !maxTimeStamp.IsZero() {
		data := LatestTrendLine{}
		data.Projectname = projectname
		data.ID = tk.Sprintf("%s_%s", data.Projectname, tipe)
		data.LastUpdate = maxTimeStamp
		data.Type = tipe

		e := ev.Ctx.Connection.NewQuery().SetConfig("multiexec", true).
			From(new(LatestTrendLine).TableName()).Save().Exec(tk.M{"data": data})

		if e != nil {
			ev.Log.AddLog(tk.Sprintf("Error on Save at updateLastData due to : %s", e.Error()), sError)
		}
	}
	ev.Log.AddLog(tk.Sprintf("Finish updating last data for %s on %s at %s", projectname, tipe, maxTimeStamp.String()), sInfo)
}

func (ev *TrendLinePlotSummary) processDataScada(wgScada *sync.WaitGroup) {
	defer wgScada.Done()

	t0 := time.Now()
	projectList := ev.getProjectList()
	lastUpdatePerProject := ev.getLatestData("SCADA")
	temperatureList := ev.getTemperatureField()

	var wg sync.WaitGroup
	wg.Add(len(projectList))
	for _, _project := range projectList {
		go ev.projectWorker(_project, lastUpdatePerProject[_project], temperatureList[_project], &wg)
	}
	wg.Wait()

	ev.Log.AddLog(tk.Sprintf("Duration processing scada data %f minutes", time.Since(t0).Minutes()), sInfo)
}

func (ev *TrendLinePlotSummary) projectWorker(projectname string, lastUpdate time.Time, tempList []string, wgProject *sync.WaitGroup) {
	defer wgProject.Done()

	groups := tk.M{"_id": tk.M{
		"turbine":   "$turbine",
		"timestamp": "$dateinfo.dateid",
	}}
	for _, field := range tempList {
		totalName := field + "total"
		countName := field + "count"
		fieldName := "$" + field

		countCondition := tk.M{"$cond": tk.M{}.
			Set("if", tk.M{"$ifNull": []interface{}{fieldName, false}}).
			Set("then", 1).
			Set("else", 0)}
		groups.Set(totalName, tk.M{"$sum": fieldName})
		groups.Set(countName, tk.M{"$sum": countCondition})
	}

	ev.Log.AddLog(tk.Sprintf("Update data %s from %s", projectname, lastUpdate.String()), sInfo)
	pipe := []tk.M{
		tk.M{"$match": tk.M{
			"$and": []tk.M{
				tk.M{"dateinfo.dateid": tk.M{"$gte": lastUpdate}},
				tk.M{"projectname": projectname},
				tk.M{"isnull": false},
			},
		}},
		tk.M{"$group": groups},
	}

	csr, e := ev.Ctx.Connection.NewQuery().
		From("Scada10MinHFD").
		Command("pipe", pipe).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor : %s", e.Error()), sError)
	}
	defer csr.Close()

	trendLineData := []tk.M{}
	e = csr.Fetch(&trendLineData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on fetch : %s", e.Error()), sError)
	}

	var wg sync.WaitGroup
	totalData := len(trendLineData)
	totalWorker := 4
	dataChan := make(chan tk.M, totalData)

	wg.Add(totalWorker)
	for i := 0; i < totalWorker; i++ {
		go func() {
			defer wg.Done()
			ctxWorker, e := PrepareConnection()
			if e != nil {
				ev.Log.AddLog(e.Error(), sError)
			}
			defer ctxWorker.Close()
			csrSave := ctxWorker.NewQuery().SetConfig("multiexec", true).
				From("rpt_trendlineplot").Save()
			defer csrSave.Close()
			for data := range dataChan {
				e = csrSave.Exec(tk.M{"data": data})
				if e != nil {
					ev.Log.AddLog(tk.Sprintf("Error on Save : %s", e.Error()), sError)
				}
			}
		}()
	}

	data := tk.M{}
	maxTimeStamp := time.Time{}

	for _, _data := range trendLineData {
		data = tk.M{}
		ids := _data.Get("_id", tk.M{}).(tk.M)
		timestamp := ids.Get("timestamp", time.Time{}).(time.Time).UTC()
		data.Set("projectname", projectname)
		data.Set("turbine", ids.GetString("turbine"))
		data.Set("timestamp", timestamp)
		_data.Set("_id", tk.Sprintf("%s_%s_%s", projectname, ids.GetString("turbine"), timestamp.Format("20060102")))
		for _dataKey, _dataVal := range _data {
			data.Set(_dataKey, _dataVal)
		}
		data.Set("type", "SCADA")

		if timestamp.After(maxTimeStamp) {
			maxTimeStamp = timestamp
		}

		dataChan <- data
	}

	close(dataChan)
	wg.Wait()

	ev.updateLastData(projectname, "SCADA", maxTimeStamp)
}

func (ev *TrendLinePlotSummary) projectWorkerMet(projectname string, lastupdate time.Time, wgProject *sync.WaitGroup) {
	defer wgProject.Done()

	pipe := []tk.M{
		tk.M{"$match": tk.M{
			"$and": []tk.M{
				tk.M{"dateinfo.dateid": tk.M{"$gte": lastupdate}},
				tk.M{"projectname": projectname},
			},
		}},
	}
	fieldName := "$trefhrefhumid855mavg"
	if lastupdate.Before(time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)) {
		fieldName = "$trefhreftemp855mavg"
	}
	countCondition := tk.M{"$cond": tk.M{}.
		Set("if", tk.M{"$ifNull": []interface{}{fieldName, false}}).
		Set("then", 1).
		Set("else", 0)}
	pipe = append(pipe, tk.M{"$group": tk.M{
		"_id":              tk.M{"timestamp": "$dateinfo.dateid"},
		"tempoutdoortotal": tk.M{"$sum": fieldName},
		"tempoutdoorcount": tk.M{"$sum": countCondition},
	}})

	trendLineData := []tk.M{}
	csr, e := ev.Ctx.Connection.NewQuery().
		From("MetTower").
		Command("pipe", pipe).Cursor(nil)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on cursor : %s", e.Error()), sError)
	}
	defer csr.Close()

	e = csr.Fetch(&trendLineData, 0, false)
	if e != nil {
		ev.Log.AddLog(tk.Sprintf("Error on Fetch : %s", e.Error()), sError)
	}

	var wg sync.WaitGroup
	totalData := len(trendLineData)
	totalWorker := 4
	dataChan := make(chan tk.M, totalData)

	wg.Add(totalWorker)
	for i := 0; i < totalWorker; i++ {
		go func() {
			defer wg.Done()
			ctxWorker, e := PrepareConnection()
			if e != nil {
				ev.Log.AddLog(e.Error(), sError)
			}
			defer ctxWorker.Close()
			csrSave := ctxWorker.NewQuery().SetConfig("multiexec", true).
				From("rpt_trendlineplot").Save()
			defer csrSave.Close()
			for data := range dataChan {
				e = csrSave.Exec(tk.M{"data": data})
				if e != nil {
					ev.Log.AddLog(tk.Sprintf("Error on Save : %s", e.Error()), sError)
				}
			}
		}()
	}

	data := tk.M{}
	maxTimeStamp := time.Time{}

	for _, _data := range trendLineData {
		data = tk.M{}
		ids := _data.Get("_id", tk.M{}).(tk.M)
		timestamp := ids.Get("timestamp", time.Time{}).(time.Time).UTC()
		data.Set("projectname", projectname)
		data.Set("timestamp", timestamp)
		_data.Set("_id", tk.Sprintf("%s_%s", projectname, timestamp.Format("20060102")))
		for _dataKey, _dataVal := range _data {
			data.Set(_dataKey, _dataVal)
		}
		data.Set("type", "MET")

		if timestamp.After(maxTimeStamp) {
			maxTimeStamp = timestamp
		}

		dataChan <- data
	}

	close(dataChan)
	wg.Wait()

	ev.updateLastData(projectname, "MET", maxTimeStamp)
}

func (ev *TrendLinePlotSummary) processDataMet(wgMet *sync.WaitGroup) {
	defer wgMet.Done()

	t0 := time.Now()

	projectList := ev.getProjectList()
	lastUpdatePerProject := ev.getLatestData("MET")

	var wg sync.WaitGroup
	wg.Add(len(projectList))
	for _, _project := range projectList {
		go ev.projectWorkerMet(_project, lastUpdatePerProject[_project], &wg)
	}
	wg.Wait()

	ev.Log.AddLog(tk.Sprintf("Duration process met tower data %f minutes", time.Since(t0).Minutes()), sInfo)
}
<|endoftext|>


<commit_before>// Utility to create and manage chromium builds.
package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/buildskia"
	"go.skia.org/infra/go/util"

	"strings"
)

// Construct the name of a directory to store a chromium build. For generic clean builds, runID
// should be empty.
func ChromiumBuildDir(chromiumHash, skiaHash, runID string) string {
	if runID == "" {
		// Do not include the runID in the dir name if it is not specified.
		return fmt.Sprintf("%s-%s",
			getTruncatedHash(chromiumHash),
			getTruncatedHash(skiaHash))
	} else {
		return fmt.Sprintf("%s-%s-%s",
			getTruncatedHash(chromiumHash),
			getTruncatedHash(skiaHash),
			runID)
	}
}

// CreateChromiumBuild creates a chromium build using the specified arguments.

// runID is the unique id of the current run (typically requester + timestamp).
// targetPlatform is the platform the benchmark will run on (Android / Linux ).
// chromiumHash is the hash the checkout should be synced to. If not specified then
// Chromium's Tot hash is used.
// skiaHash is the hash the checkout should be synced to. If not specified then
// Skia's LKGR hash is used (the hash in Chromium's DEPS file).
// applyPatches if true looks for Chromium/Skia patches in the temp dir and
// runs once with the patch applied and once without the patch applied.
func CreateChromiumBuild(runID, targetPlatform, chromiumHash, skiaHash string, applyPatches bool) (string, string, error) {
	// Determine which build dir and fetch target to use.
	var chromiumBuildDir, fetchTarget string
	if targetPlatform == "Android" {
		chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "android_base")
		fetchTarget = "android"
	} else if targetPlatform == "Linux" {
		chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "linux_base")
		fetchTarget = "chromium"
	} else {
		return "", "", fmt.Errorf("Unrecognized target_platform %s", targetPlatform)
	}
	util.MkdirAll(chromiumBuildDir, 0700)

	// Find which Chromium commit hash should be used.
	var err error
	if chromiumHash == "" {
		chromiumHash, err = getChromiumHash()
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Chromium's Hash: %s", err)
		}
	}

	// Find which Skia commit hash should be used.
	if skiaHash == "" {
		skiaHash, err = buildskia.GetSkiaHash(nil)
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Skia's Hash: %s", err)
		}
	}

	// Run chromium sync command using the above commit hashes.
	// Construct path to the sync_skia_in_chrome python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(currentFile)))),
		"py")
	syncArgs := []string{
		filepath.Join(pathToPyFiles, "sync_skia_in_chrome.py"),
		"--destination=" + chromiumBuildDir,
		"--fetch_target=" + fetchTarget,
		"--chrome_revision=" + chromiumHash,
		"--skia_revision=" + skiaHash,
	}
	err = ExecuteCmd("python", syncArgs, []string{}, SYNC_SKIA_IN_CHROME_TIMEOUT, nil, nil)
	if err != nil {
		glog.Warning("There was an error. Deleting base directory and trying again.")
		util.RemoveAll(chromiumBuildDir)
		util.MkdirAll(chromiumBuildDir, 0700)
		err := ExecuteCmd("python", syncArgs, []string{}, SYNC_SKIA_IN_CHROME_TIMEOUT, nil,
			nil)
		if err != nil {
			return "", "", fmt.Errorf("There was an error checking out chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
	}

	// Make sure we are starting from a clean slate.
	if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
		return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
	}
	googleStorageDirName := ChromiumBuildDir(chromiumHash, skiaHash, runID)
	if applyPatches {
		if err := applyRepoPatches(filepath.Join(chromiumBuildDir, "src"), runID); err != nil {
			return "", "", fmt.Errorf("Could not apply patches in the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Add "try" prefix and "withpatch" suffix.
		googleStorageDirName = fmt.Sprintf("try-%s-withpatch", googleStorageDirName)
	}
	// Build chromium.
	if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
		return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
	}

	// Upload to Google Storage.
	gs, err := NewGsUtil(nil)
	if err != nil {
		return "", "", fmt.Errorf("Could not create GS object: %s", err)
	}
	if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
		return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
	}

	// Check for the applypatch flag and reset and then build again and copy to
	// google storage.
	if applyPatches {
		// Now build chromium without the patches and upload it to Google Storage.

		// Make sure we are starting from a clean slate.
		if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
			return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Build chromium.
		if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
			return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
		// Upload to Google Storage.
		googleStorageDirName = fmt.Sprintf("try-%s-%s-%s-nopatch", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID)
		if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
			return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
		}
	}
	return getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), nil
}

func getChromiumHash() (string, error) {
	// Find Chromium's Tot commit hash.
	stdoutFilePath := filepath.Join(os.TempDir(), "chromium-tot")
	stdoutFile, err := os.Create(stdoutFilePath)
	defer util.Close(stdoutFile)
	defer util.Remove(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Could not create %s: %s", stdoutFilePath, err)
	}
	totArgs := []string{"ls-remote", "https://chromium.googlesource.com/chromium/src.git", "--verify", "refs/heads/master"}
	err = ExecuteCmd(BINARY_GIT, totArgs, []string{}, GIT_LS_REMOTE_TIMEOUT, stdoutFile, nil)
	if err != nil {
		return "", fmt.Errorf("Error while finding Chromium's ToT: %s", err)
	}
	output, err := ioutil.ReadFile(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Cannot read %s: %s", stdoutFilePath, err)
	}
	tokens := strings.Split(string(output), "\t")
	return tokens[0], nil
}

func uploadChromiumBuild(localOutDir, gsDir, targetPlatform string, gs *GsUtil) error {
	localUploadDir := localOutDir
	if targetPlatform == "Android" {
		localUploadDir = filepath.Join(localUploadDir, "apks")
	} else {
		// Temporarily move the not needed large "gen" and "obj" directories so
		// that they do not get uploaded to Google Storage. Move them back after
		// the method completes.

		genDir := filepath.Join(localOutDir, "gen")
		genTmpDir := filepath.Join(ChromiumBuildsDir, "gen")
		// Make sure the tmp dir is empty.
		util.RemoveAll(genTmpDir)
		if err := os.Rename(genDir, genTmpDir); err != nil {
			return fmt.Errorf("Could not rename gen dir: %s", err)
		}
		defer util.Rename(genTmpDir, genDir)

		objDir := filepath.Join(localOutDir, "obj")
		objTmpDir := filepath.Join(ChromiumBuildsDir, "obj")
		// Make sure the tmp dir is empty.
		util.RemoveAll(objTmpDir)
		if err := os.Rename(objDir, objTmpDir); err != nil {
			return fmt.Errorf("Could not rename obj dir: %s", err)
		}
		defer util.Rename(objTmpDir, objDir)
	}
	return gs.UploadDir(localUploadDir, gsDir, true)
}

func buildChromium(chromiumDir, targetPlatform string) error {
	if err := os.Chdir(filepath.Join(chromiumDir, "src")); err != nil {
		return fmt.Errorf("Could not chdir to %s/src: %s", chromiumDir, err)
	}

	// Find the build target to use while building chromium.
	buildTarget := "chrome"
	if targetPlatform == "Android" {
		buildTarget = "chrome_public_apk"
	}

	// Start Goma's compiler proxy right before building the checkout.
	err := ExecuteCmd("python", []string{filepath.Join(GomaDir, "goma_ctl.py"), "start"},
		os.Environ(), GOMA_CTL_RESTART_TIMEOUT, nil, nil)
	if err != nil {
		return fmt.Errorf("Error while starting goma compiler proxy: %s", err)
	}

	// Run "gn gen out/Release --args=...".
	gn_args := []string{"is_debug=false", "use_goma=true", fmt.Sprintf("goma_dir=\"%s\"", GomaDir)}
	if targetPlatform == "Android" {
		gn_args = append(gn_args, "target_os=\"android\"")
	}
	err = ExecuteCmd("gn", []string{"gen", "out/Release", fmt.Sprintf("--args=%s", strings.Join(gn_args, " "))}, os.Environ(), GN_CHROMIUM_TIMEOUT, nil, nil)
	if err != nil {
		return fmt.Errorf("Error while running gn: %s", err)
	}
	// Run "ninja -C out/Release -j100 ${build_target}".
	// Use the full system env while building chromium.
	args := []string{"-C", "out/Release", "-j100", buildTarget}
	return ExecuteCmd("ninja", args, os.Environ(), NINJA_TIMEOUT, nil, nil)
}

func getTruncatedHash(commitHash string) string {
	return commitHash[0:7]
}

func resetChromiumCheckout(chromiumSrcDir string) error {
	// Reset Skia.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	if err := ResetCheckout(skiaDir); err != nil {
		return fmt.Errorf("Could not reset Skia's checkout in %s: %s", skiaDir, err)
	}
	// Reset Chromium.
	if err := ResetCheckout(chromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset Chromium's checkout in %s: %s", chromiumSrcDir, err)
	}
	return nil
}

func applyRepoPatches(chromiumSrcDir, runID string) error {
	// Apply Skia patch.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	skiaPatch := filepath.Join(os.TempDir(), runID+".skia.patch")
	skiaPatchFile, _ := os.Open(skiaPatch)
	skiaPatchFileInfo, _ := skiaPatchFile.Stat()
	if skiaPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(skiaPatch, skiaDir); err != nil {
			return fmt.Errorf("Could not apply Skia's patch in %s: %s", skiaDir, err)
		}
	}
	// Apply Chromium patch.
	chromiumPatch := filepath.Join(os.TempDir(), runID+".chromium.patch")
	chromiumPatchFile, _ := os.Open(chromiumPatch)
	chromiumPatchFileInfo, _ := chromiumPatchFile.Stat()
	if chromiumPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(chromiumPatch, chromiumSrcDir); err != nil {
			return fmt.Errorf("Could not apply Chromium's patch in %s: %s", chromiumSrcDir, err)
		}
	}
	return nil
}

func InstallChromeAPK(chromiumBuildName string) error {
	// Install the APK on the Android device.
	chromiumApk := filepath.Join(ChromiumBuildsDir, chromiumBuildName, ApkName)
	glog.Infof("Installing the APK at %s", chromiumApk)
	err := ExecuteCmd(BINARY_ADB, []string{"install", "-r", chromiumApk}, []string{},
		ADB_INSTALL_TIMEOUT, nil, nil)
	if err != nil {
		return fmt.Errorf("Could not install the chromium APK at %s: %s", chromiumBuildName, err)
	}
	return nil
}
<commit_msg>Add treat_warnings_as_errors = false to gn args<commit_after>// Utility to create and manage chromium builds.
package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/buildskia"
	"go.skia.org/infra/go/util"

	"strings"
)

// Construct the name of a directory to store a chromium build. For generic clean builds, runID
// should be empty.
func ChromiumBuildDir(chromiumHash, skiaHash, runID string) string {
	if runID == "" {
		// Do not include the runID in the dir name if it is not specified.
		return fmt.Sprintf("%s-%s",
			getTruncatedHash(chromiumHash),
			getTruncatedHash(skiaHash))
	} else {
		return fmt.Sprintf("%s-%s-%s",
			getTruncatedHash(chromiumHash),
			getTruncatedHash(skiaHash),
			runID)
	}
}

// CreateChromiumBuild creates a chromium build using the specified arguments.

// runID is the unique id of the current run (typically requester + timestamp).
// targetPlatform is the platform the benchmark will run on (Android / Linux ).
// chromiumHash is the hash the checkout should be synced to. If not specified then
// Chromium's Tot hash is used.
// skiaHash is the hash the checkout should be synced to. If not specified then
// Skia's LKGR hash is used (the hash in Chromium's DEPS file).
// applyPatches if true looks for Chromium/Skia patches in the temp dir and
// runs once with the patch applied and once without the patch applied.
func CreateChromiumBuild(runID, targetPlatform, chromiumHash, skiaHash string, applyPatches bool) (string, string, error) {
	// Determine which build dir and fetch target to use.
	var chromiumBuildDir, fetchTarget string
	if targetPlatform == "Android" {
		chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "android_base")
		fetchTarget = "android"
	} else if targetPlatform == "Linux" {
		chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "linux_base")
		fetchTarget = "chromium"
	} else {
		return "", "", fmt.Errorf("Unrecognized target_platform %s", targetPlatform)
	}
	util.MkdirAll(chromiumBuildDir, 0700)

	// Find which Chromium commit hash should be used.
	var err error
	if chromiumHash == "" {
		chromiumHash, err = getChromiumHash()
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Chromium's Hash: %s", err)
		}
	}

	// Find which Skia commit hash should be used.
	if skiaHash == "" {
		skiaHash, err = buildskia.GetSkiaHash(nil)
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Skia's Hash: %s", err)
		}
	}

	// Run chromium sync command using the above commit hashes.
	// Construct path to the sync_skia_in_chrome python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(currentFile)))),
		"py")
	syncArgs := []string{
		filepath.Join(pathToPyFiles, "sync_skia_in_chrome.py"),
		"--destination=" + chromiumBuildDir,
		"--fetch_target=" + fetchTarget,
		"--chrome_revision=" + chromiumHash,
		"--skia_revision=" + skiaHash,
	}
	err = ExecuteCmd("python", syncArgs, []string{}, SYNC_SKIA_IN_CHROME_TIMEOUT, nil, nil)
	if err != nil {
		glog.Warning("There was an error. Deleting base directory and trying again.")
		util.RemoveAll(chromiumBuildDir)
		util.MkdirAll(chromiumBuildDir, 0700)
		err := ExecuteCmd("python", syncArgs, []string{}, SYNC_SKIA_IN_CHROME_TIMEOUT, nil,
			nil)
		if err != nil {
			return "", "", fmt.Errorf("There was an error checking out chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
	}

	// Make sure we are starting from a clean slate.
	if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
		return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
	}
	googleStorageDirName := ChromiumBuildDir(chromiumHash, skiaHash, runID)
	if applyPatches {
		if err := applyRepoPatches(filepath.Join(chromiumBuildDir, "src"), runID); err != nil {
			return "", "", fmt.Errorf("Could not apply patches in the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Add "try" prefix and "withpatch" suffix.
		googleStorageDirName = fmt.Sprintf("try-%s-withpatch", googleStorageDirName)
	}
	// Build chromium.
	if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
		return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
	}

	// Upload to Google Storage.
	gs, err := NewGsUtil(nil)
	if err != nil {
		return "", "", fmt.Errorf("Could not create GS object: %s", err)
	}
	if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
		return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
	}

	// Check for the applypatch flag and reset and then build again and copy to
	// google storage.
	if applyPatches {
		// Now build chromium without the patches and upload it to Google Storage.

		// Make sure we are starting from a clean slate.
		if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
			return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Build chromium.
		if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
			return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
		// Upload to Google Storage.
		googleStorageDirName = fmt.Sprintf("try-%s-%s-%s-nopatch", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID)
		if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
			return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
		}
	}
	return getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), nil
}

func getChromiumHash() (string, error) {
	// Find Chromium's Tot commit hash.
	stdoutFilePath := filepath.Join(os.TempDir(), "chromium-tot")
	stdoutFile, err := os.Create(stdoutFilePath)
	defer util.Close(stdoutFile)
	defer util.Remove(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Could not create %s: %s", stdoutFilePath, err)
	}
	totArgs := []string{"ls-remote", "https://chromium.googlesource.com/chromium/src.git", "--verify", "refs/heads/master"}
	err = ExecuteCmd(BINARY_GIT, totArgs, []string{}, GIT_LS_REMOTE_TIMEOUT, stdoutFile, nil)
	if err != nil {
		return "", fmt.Errorf("Error while finding Chromium's ToT: %s", err)
	}
	output, err := ioutil.ReadFile(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Cannot read %s: %s", stdoutFilePath, err)
	}
	tokens := strings.Split(string(output), "\t")
	return tokens[0], nil
}

func uploadChromiumBuild(localOutDir, gsDir, targetPlatform string, gs *GsUtil) error {
	localUploadDir := localOutDir
	if targetPlatform == "Android" {
		localUploadDir = filepath.Join(localUploadDir, "apks")
	} else {
		// Temporarily move the not needed large "gen" and "obj" directories so
		// that they do not get uploaded to Google Storage. Move them back after
		// the method completes.

		genDir := filepath.Join(localOutDir, "gen")
		genTmpDir := filepath.Join(ChromiumBuildsDir, "gen")
		// Make sure the tmp dir is empty.
		util.RemoveAll(genTmpDir)
		if err := os.Rename(genDir, genTmpDir); err != nil {
			return fmt.Errorf("Could not rename gen dir: %s", err)
		}
		defer util.Rename(genTmpDir, genDir)

		objDir := filepath.Join(localOutDir, "obj")
		objTmpDir := filepath.Join(ChromiumBuildsDir, "obj")
		// Make sure the tmp dir is empty.
		util.RemoveAll(objTmpDir)
		if err := os.Rename(objDir, objTmpDir); err != nil {
			return fmt.Errorf("Could not rename obj dir: %s", err)
		}
		defer util.Rename(objTmpDir, objDir)
	}
	return gs.UploadDir(localUploadDir, gsDir, true)
}

func buildChromium(chromiumDir, targetPlatform string) error {
	if err := os.Chdir(filepath.Join(chromiumDir, "src")); err != nil {
		return fmt.Errorf("Could not chdir to %s/src: %s", chromiumDir, err)
	}

	// Find the build target to use while building chromium.
	buildTarget := "chrome"
	if targetPlatform == "Android" {
		buildTarget = "chrome_public_apk"
	}

	// Start Goma's compiler proxy right before building the checkout.
	err := ExecuteCmd("python", []string{filepath.Join(GomaDir, "goma_ctl.py"), "start"},
		os.Environ(), GOMA_CTL_RESTART_TIMEOUT, nil, nil)
	if err != nil {
		return fmt.Errorf("Error while starting goma compiler proxy: %s", err)
	}

	// Run "gn gen out/Release --args=...".
	gn_args := []string{"is_debug=false", "use_goma=true", fmt.Sprintf("goma_dir=\"%s\"", GomaDir), "treat_warnings_as_errors=false"}
	if targetPlatform == "Android" {
		gn_args = append(gn_args, "target_os=\"android\"")
	}
	err = ExecuteCmd("gn", []string{"gen", "out/Release", fmt.Sprintf("--args=%s", strings.Join(gn_args, " "))}, os.Environ(), GN_CHROMIUM_TIMEOUT, nil, nil)
	if err != nil {
		return fmt.Errorf("Error while running gn: %s", err)
	}
	// Run "ninja -C out/Release -j100 ${build_target}".
	// Use the full system env while building chromium.
	args := []string{"-C", "out/Release", "-j100", buildTarget}
	return ExecuteCmd("ninja", args, os.Environ(), NINJA_TIMEOUT, nil, nil)
}

func getTruncatedHash(commitHash string) string {
	return commitHash[0:7]
}

func resetChromiumCheckout(chromiumSrcDir string) error {
	// Reset Skia.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	if err := ResetCheckout(skiaDir); err != nil {
		return fmt.Errorf("Could not reset Skia's checkout in %s: %s", skiaDir, err)
	}
	// Reset Chromium.
	if err := ResetCheckout(chromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset Chromium's checkout in %s: %s", chromiumSrcDir, err)
	}
	return nil
}

func applyRepoPatches(chromiumSrcDir, runID string) error {
	// Apply Skia patch.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	skiaPatch := filepath.Join(os.TempDir(), runID+".skia.patch")
	skiaPatchFile, _ := os.Open(skiaPatch)
	skiaPatchFileInfo, _ := skiaPatchFile.Stat()
	if skiaPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(skiaPatch, skiaDir); err != nil {
			return fmt.Errorf("Could not apply Skia's patch in %s: %s", skiaDir, err)
		}
	}
	// Apply Chromium patch.
	chromiumPatch := filepath.Join(os.TempDir(), runID+".chromium.patch")
	chromiumPatchFile, _ := os.Open(chromiumPatch)
	chromiumPatchFileInfo, _ := chromiumPatchFile.Stat()
	if chromiumPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(chromiumPatch, chromiumSrcDir); err != nil {
			return fmt.Errorf("Could not apply Chromium's patch in %s: %s", chromiumSrcDir, err)
		}
	}
	return nil
}

func InstallChromeAPK(chromiumBuildName string) error {
	// Install the APK on the Android device.
	chromiumApk := filepath.Join(ChromiumBuildsDir, chromiumBuildName, ApkName)
	glog.Infof("Installing the APK at %s", chromiumApk)
	err := ExecuteCmd(BINARY_ADB, []string{"install", "-r", chromiumApk}, []string{},
		ADB_INSTALL_TIMEOUT, nil, nil)
	if err != nil {
		return fmt.Errorf("Could not install the chromium APK at %s: %s", chromiumBuildName, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package sqlego

import (
	s "strings"
	"testing"
)

func TestSelectStatement(t *testing.T) {
	node := Select("Users", []string{"id", "name", "email"})
	sql := node.Compile()
	if sql != "SELECT id,name,email FROM Users;" {
		t.Fatal(sql)
	}
}

func TestInsertStatement(t *testing.T) {
	node := Insert("Users", map[string]string{"name": "Bruce"})
	sql := node.Compile()
	if sql != "INSERT INTO Users (name) VALUES (Bruce);" {
		t.Fatal(sql)
	}
}

func TestUpdateStatement(t *testing.T) {
	node := Update("Users", map[string]string{"id": "2", "name": "Bruce", "email": "bruce@example.com"})
	sql := node.Compile()
	if !s.Contains(sql, "UPDATE Users SET") && !s.Contains(sql, "id=2") && !s.Contains(sql, "name=Bruce") && !s.Contains(sql, "email=bruce@example.com") {
		t.Fatal(sql)
	}
}
<commit_msg>Add basic DELETE statement testing<commit_after>package sqlego

import (
	s "strings"
	"testing"
)

func TestSelectStatement(t *testing.T) {
	node := Select("Users", []string{"id", "name", "email"})
	sql := node.Compile()
	if sql != "SELECT id,name,email FROM Users;" {
		t.Fatal(sql)
	}
}

func TestInsertStatement(t *testing.T) {
	node := Insert("Users", map[string]string{"name": "Bruce"})
	sql := node.Compile()
	if sql != "INSERT INTO Users (name) VALUES (Bruce);" {
		t.Fatal(sql)
	}
}

func TestUpdateStatement(t *testing.T) {
	node := Update("Users", map[string]string{"id": "2", "name": "Bruce", "email": "bruce@example.com"})
	sql := node.Compile()
	if !s.Contains(sql, "UPDATE Users SET") && !s.Contains(sql, "id=2") && !s.Contains(sql, "name=Bruce") && !s.Contains(sql, "email=bruce@example.com") {
		t.Fatal(sql)
	}
}

func TestDeleteStatement(t *testing.T) {
	node := Delete("Users")
	sql := node.Compile()
	if sql != "DELETE FROM Users;" {
		t.Fatal(sql)
	}
}
<|endoftext|>


<commit_before>package do

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"html/template"
	"strconv"

	"koding/kites/kloud/stack"
	"koding/kites/kloud/stack/provider"
	"koding/kites/kloud/userdata"

	"github.com/digitalocean/godo"
	"golang.org/x/oauth2"
)

//go:generate $GOPATH/bin/go-bindata -mode 420 -modtime 1470666525 -pkg do -o bootstrap.json.tmpl.go bootstrap.json.tmpl
//go:generate go fmt bootstrap.json.tmpl.go

var bootstrapTmpl = template.Must(
	template.New("").Parse(string(MustAsset("bootstrap.json.tmpl"))),
)

const (
	defaultUbuntuImage = "ubuntu-14-04-x64"
)

var _ provider.Stack = (*Stack)(nil)

// Stack is responsible of handling the terraform templates
type Stack struct {
	*provider.BaseStack
}

func newStack(stack *provider.BaseStack) (provider.Stack, error) {
	return &Stack{BaseStack: stack}, nil
}

// VerifyCredential verifies whether the users DO credentials (access token) is
// valid or not
func (s *Stack) VerifyCredential(c *stack.Credential) error {
	cred := c.Credential.(*Credential)

	if err := cred.Valid(); err != nil {
		return err
	}

	oauthClient := oauth2.NewClient(
		oauth2.NoContext,
		oauth2.StaticTokenSource(&oauth2.Token{AccessToken: cred.AccessToken}),
	)

	client := godo.NewClient(oauthClient)

	// let's retrieve our Account information. If it's successful, we're good
	// to go
	_, _, err := client.Account.Get()
	if err != nil {
		return &stack.Error{
			Err: err,
		}
	}

	return nil
}

// BootstrapTemplates returns terraform templates that needs to be executed
// before a droplet is created. In our case we'll create a template that
// creates a ssh key on behalf of Koding, that will be later used during
// ApplyTemplate()
func (s *Stack) BootstrapTemplates(c *stack.Credential) ([]*stack.Template, error) {
	type tmplData struct {
		KeyName   string
		PublicKey string
	}

	// fill the template
	var buf bytes.Buffer
	if err := bootstrapTmpl.Execute(&buf, &tmplData{
		KeyName:   s.keyName(),
		PublicKey: s.Keys.PublicKey,
	}); err != nil {
		return nil, err
	}

	return []*stack.Template{
		{Content: buf.String()},
	}, nil
}

// keyName returns the keyName used for the bootstrap data.
//
// The key pair creation is idempotent - if the key already
// exists with the same name and content the create
// operation is a nop. If key already exists, but under
// a different name, key pair creation is going to fail
// due to a name conflict.
func (s *Stack) keyName() string {
	sum := sha1.Sum([]byte(s.Keys.PublicKey))
	return "koding-deployment-" + hex.EncodeToString(sum[:])
}

// ApplyTemplate enhances and updates the DigitalOcean terraform template. It
// updates the various sections of the template, such as Provider, Resources,
// Variables, etc... so it can be executed without any problems
func (s *Stack) ApplyTemplate(c *stack.Credential) (*stack.Template, error) {
	cred, ok := c.Credential.(*Credential)
	if !ok {
		return nil, fmt.Errorf("credential is not of type do.Credential: %T", c.Credential)
	}

	bootstrap, ok := c.Bootstrap.(*Bootstrap)
	if !ok {
		return nil, fmt.Errorf("bootstrap is not of type do.Bootstrap: %T", c.Bootstrap)
	}

	template := s.Builder.Template
	template.Provider["digitalocean"] = map[string]interface{}{
		"token": cred.AccessToken,
	}

	keyID, err := strconv.Atoi(bootstrap.KeyID)
	if err != nil {
		return nil, err
	}

	droplet, err := s.modifyDroplets(keyID)
	if err != nil {
		return nil, err
	}

	template.Resource["digitalocean_droplet"] = droplet

	if err := template.ShadowVariables("FORBIDDEN", "digitalocean_access_token"); err != nil {
		return nil, err
	}

	if err := template.Flush(); err != nil {
		return nil, err
	}

	content, err := template.JsonOutput()
	if err != nil {
		return nil, err
	}

	return &stack.Template{
		Content: content,
	}, nil
}

// modifyDroplets returns a modified 'digitalocean_droplet' terraform resource
// from the stack that changes things like image, injects kite and ssh_key,
// etc...
func (s *Stack) modifyDroplets(keyID int) (map[string]map[string]interface{}, error) {
	var resource struct {
		Droplet map[string]map[string]interface{} `hcl:"digitalocean_droplet"`
	}

	if err := s.Builder.Template.DecodeResource(&resource); err != nil {
		return nil, err
	}

	if len(resource.Droplet) == 0 {
		return nil, errors.New("there are no droplets available")
	}

	// we might have multipel droplets, iterate over all of them
	for dropletName, droplet := range resource.Droplet {
		// Do not overwrite SSH key pair with the bootstrap one
		// when user sets it explicitly in a template.
		if s, ok := droplet["ssh_keys"]; !ok {
			droplet["ssh_keys"] = []int{keyID}
		} else if keyIds, ok := s.([]int); ok {
			keys := []int{keyID}
			if len(keyIds) != 0 {
				keys = append(keys, keyIds...)
			}

			droplet["ssh_keys"] = keys
		}

		// if nothing is provided or the image is empty use default Ubuntu image
		if i, ok := droplet["image"]; !ok {
			droplet["image"] = defaultUbuntuImage
		} else if image, ok := i.(string); ok && image == "" {
			droplet["image"] = defaultUbuntuImage
		}

		// means there will be several instances, we need to create a userdata
		// with count interpolation, because each machine must have an unique
		// kite id.
		count := 1
		if n, ok := droplet["count"].(int); ok && n > 1 {
			count = n
		}

		labels := []string{dropletName}
		if count > 1 {
			for i := 0; i < count; i++ {
				labels = append(labels, fmt.Sprintf("%s.%d", dropletName, i))
			}
		}

		kiteKeyName := fmt.Sprintf("kitekeys_%s", dropletName)

		s.Builder.InterpolateField(droplet, dropletName, "user_data")

		// this part will be the same for all machines
		userCfg := &userdata.CloudInitConfig{
			Username: s.Req.Username,
			Groups:   []string{"sudo"},
			Hostname: s.Req.Username, // no typo here. hostname = username
			KiteKey:  fmt.Sprintf("${lookup(var.%s, count.index)}", kiteKeyName),
		}

		if s, ok := droplet["user_data"].(string); ok {
			userCfg.UserData = s
		}

		userdata, err := s.Session.Userdata.Create(userCfg)
		if err != nil {
			return nil, err
		}

		droplet["user_data"] = string(userdata)

		// create independent kiteKey for each machine and create a Terraform
		// lookup map, which is used in conjuctuon with the `count.index`
		countKeys := make(map[string]string, count)
		for i, label := range labels {
			kiteKey, err := s.BuildKiteKey(label, s.Req.Username)
			if err != nil {
				return nil, err
			}

			countKeys[strconv.Itoa(i)] = kiteKey
		}

		s.Builder.Template.Variable[kiteKeyName] = map[string]interface{}{
			"default": countKeys,
		}

		resource.Droplet[dropletName] = droplet
	}

	return resource.Droplet, nil
}

// BootstrapArg returns the bootstrap argument made to the bootrap kite
func (s *Stack) BootstrapArg() *stack.BootstrapRequest {
	return s.BaseStack.Arg.(*stack.BootstrapRequest)
}
<commit_msg>provider/do: use unique SSH key-pair<commit_after>package do

import (
	"bytes"
	"errors"
	"fmt"
	"html/template"
	"strconv"

	"koding/kites/kloud/stack"
	"koding/kites/kloud/stack/provider"
	"koding/kites/kloud/userdata"

	"github.com/digitalocean/godo"
	"golang.org/x/oauth2"
)

//go:generate $GOPATH/bin/go-bindata -mode 420 -modtime 1470666525 -pkg do -o bootstrap.json.tmpl.go bootstrap.json.tmpl
//go:generate go fmt bootstrap.json.tmpl.go

var bootstrapTmpl = template.Must(
	template.New("").Parse(string(MustAsset("bootstrap.json.tmpl"))),
)

const (
	defaultUbuntuImage = "ubuntu-14-04-x64"
)

var _ provider.Stack = (*Stack)(nil)

// Stack is responsible of handling the terraform templates
type Stack struct {
	*provider.BaseStack

	sshKeyPair *stack.SSHKeyPair
}

func newStack(bs *provider.BaseStack) (provider.Stack, error) {
	s := &Stack{
		BaseStack: bs,
	}

	bs.SSHKeyPairFunc = s.setSSHKeyPair

	return s, nil
}

// VerifyCredential verifies whether the users DO credentials (access token) is
// valid or not
func (s *Stack) VerifyCredential(c *stack.Credential) error {
	cred := c.Credential.(*Credential)

	if err := cred.Valid(); err != nil {
		return err
	}

	oauthClient := oauth2.NewClient(
		oauth2.NoContext,
		oauth2.StaticTokenSource(&oauth2.Token{AccessToken: cred.AccessToken}),
	)

	client := godo.NewClient(oauthClient)

	// let's retrieve our Account information. If it's successful, we're good
	// to go
	_, _, err := client.Account.Get()
	if err != nil {
		return &stack.Error{
			Err: err,
		}
	}

	return nil
}

func (s *Stack) setSSHKeyPair(keypair *stack.SSHKeyPair) error {
	s.sshKeyPair = keypair
	return nil
}

// BootstrapTemplates returns terraform templates that needs to be executed
// before a droplet is created. In our case we'll create a template that
// creates a ssh key on behalf of Koding, that will be later used during
// ApplyTemplate()
func (s *Stack) BootstrapTemplates(c *stack.Credential) ([]*stack.Template, error) {
	type tmplData struct {
		KeyName   string
		PublicKey string
	}

	// fill the template
	var buf bytes.Buffer
	if err := bootstrapTmpl.Execute(&buf, &tmplData{
		KeyName:   s.sshKeyPair.Name,
		PublicKey: string(s.sshKeyPair.Public),
	}); err != nil {
		return nil, err
	}

	return []*stack.Template{
		{Content: buf.String()},
	}, nil
}

// ApplyTemplate enhances and updates the DigitalOcean terraform template. It
// updates the various sections of the template, such as Provider, Resources,
// Variables, etc... so it can be executed without any problems
func (s *Stack) ApplyTemplate(c *stack.Credential) (*stack.Template, error) {
	cred, ok := c.Credential.(*Credential)
	if !ok {
		return nil, fmt.Errorf("credential is not of type do.Credential: %T", c.Credential)
	}

	bootstrap, ok := c.Bootstrap.(*Bootstrap)
	if !ok {
		return nil, fmt.Errorf("bootstrap is not of type do.Bootstrap: %T", c.Bootstrap)
	}

	template := s.Builder.Template
	template.Provider["digitalocean"] = map[string]interface{}{
		"token": cred.AccessToken,
	}

	keyID, err := strconv.Atoi(bootstrap.KeyID)
	if err != nil {
		return nil, err
	}

	droplet, err := s.modifyDroplets(keyID)
	if err != nil {
		return nil, err
	}

	template.Resource["digitalocean_droplet"] = droplet

	if err := template.ShadowVariables("FORBIDDEN", "digitalocean_access_token"); err != nil {
		return nil, err
	}

	if err := template.Flush(); err != nil {
		return nil, err
	}

	content, err := template.JsonOutput()
	if err != nil {
		return nil, err
	}

	return &stack.Template{
		Content: content,
	}, nil
}

// modifyDroplets returns a modified 'digitalocean_droplet' terraform resource
// from the stack that changes things like image, injects kite and ssh_key,
// etc...
func (s *Stack) modifyDroplets(keyID int) (map[string]map[string]interface{}, error) {
	var resource struct {
		Droplet map[string]map[string]interface{} `hcl:"digitalocean_droplet"`
	}

	if err := s.Builder.Template.DecodeResource(&resource); err != nil {
		return nil, err
	}

	if len(resource.Droplet) == 0 {
		return nil, errors.New("there are no droplets available")
	}

	// we might have multipel droplets, iterate over all of them
	for dropletName, droplet := range resource.Droplet {
		// Do not overwrite SSH key pair with the bootstrap one
		// when user sets it explicitly in a template.
		if s, ok := droplet["ssh_keys"]; !ok {
			droplet["ssh_keys"] = []int{keyID}
		} else if keyIds, ok := s.([]int); ok {
			keys := []int{keyID}
			if len(keyIds) != 0 {
				keys = append(keys, keyIds...)
			}

			droplet["ssh_keys"] = keys
		}

		// if nothing is provided or the image is empty use default Ubuntu image
		if i, ok := droplet["image"]; !ok {
			droplet["image"] = defaultUbuntuImage
		} else if image, ok := i.(string); ok && image == "" {
			droplet["image"] = defaultUbuntuImage
		}

		// means there will be several instances, we need to create a userdata
		// with count interpolation, because each machine must have an unique
		// kite id.
		count := 1
		if n, ok := droplet["count"].(int); ok && n > 1 {
			count = n
		}

		labels := []string{dropletName}
		if count > 1 {
			for i := 0; i < count; i++ {
				labels = append(labels, fmt.Sprintf("%s.%d", dropletName, i))
			}
		}

		kiteKeyName := fmt.Sprintf("kitekeys_%s", dropletName)

		s.Builder.InterpolateField(droplet, dropletName, "user_data")

		// this part will be the same for all machines
		userCfg := &userdata.CloudInitConfig{
			Username: s.Req.Username,
			Groups:   []string{"sudo"},
			Hostname: s.Req.Username, // no typo here. hostname = username
			KiteKey:  fmt.Sprintf("${lookup(var.%s, count.index)}", kiteKeyName),
		}

		if s, ok := droplet["user_data"].(string); ok {
			userCfg.UserData = s
		}

		userdata, err := s.Session.Userdata.Create(userCfg)
		if err != nil {
			return nil, err
		}

		droplet["user_data"] = string(userdata)

		// create independent kiteKey for each machine and create a Terraform
		// lookup map, which is used in conjuctuon with the `count.index`
		countKeys := make(map[string]string, count)
		for i, label := range labels {
			kiteKey, err := s.BuildKiteKey(label, s.Req.Username)
			if err != nil {
				return nil, err
			}

			countKeys[strconv.Itoa(i)] = kiteKey
		}

		s.Builder.Template.Variable[kiteKeyName] = map[string]interface{}{
			"default": countKeys,
		}

		resource.Droplet[dropletName] = droplet
	}

	return resource.Droplet, nil
}

// BootstrapArg returns the bootstrap argument made to the bootrap kite
func (s *Stack) BootstrapArg() *stack.BootstrapRequest {
	return s.BaseStack.Arg.(*stack.BootstrapRequest)
}
<|endoftext|>


<commit_before>package main

import (
	"math/rand"
	"socialapi/models"
	"socialapi/rest"
	"strconv"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestPrivateMesssage(t *testing.T) {
	Convey("while testing private messages", t, func() {
		account := models.NewAccount()
		account.OldId = AccountOldId.Hex()
		account, err := rest.CreateAccount(account)
		So(err, ShouldBeNil)
		So(account, ShouldNotBeNil)

		recipient := models.NewAccount()
		recipient.OldId = AccountOldId2.Hex()
		recipient, err = rest.CreateAccount(recipient)
		So(err, ShouldBeNil)
		So(recipient, ShouldNotBeNil)

		recipient2 := models.NewAccount()
		recipient2.OldId = AccountOldId3.Hex()
		recipient2, err = rest.CreateAccount(recipient2)
		So(err, ShouldBeNil)
		So(recipient2, ShouldNotBeNil)

		groupName := "testgroup" + strconv.FormatInt(rand.Int63(), 10)

		Convey("one can send private message to one person", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body message for private message @chris @devrim @sinan",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

		})

		Convey("0 recipient should fail", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message",
				groupName,
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)

		})
		Convey("if body is nil, should fail to create PM", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"",
				groupName,
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)
		})
		Convey("if group name is nil, should not fail to create PM", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message @chris @devrim @sinan",
				"",
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
		})

		Convey("if sender is not defined should fail to create PM", func() {
			cmc, err := rest.SendPrivateMessage(
				0,
				"this is a body for private message",
				"",
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)
		})

		Convey("one can send private message to multiple person", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message @sinan",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

		})
		Convey("private message response should have created channel", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message @devrim @sinan",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.Channel.TypeConstant, ShouldEqual, models.Channel_TYPE_PRIVATE_MESSAGE)
			So(cmc.Channel.Id, ShouldBeGreaterThan, 0)
			So(cmc.Channel.GroupName, ShouldEqual, groupName)
			So(cmc.Channel.PrivacyConstant, ShouldEqual, models.Channel_PRIVACY_PRIVATE)

		})

		Convey("private message response should have participant status data", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message @chris @devrim @sinan",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.IsParticipant, ShouldBeTrue)
		})

		Convey("private message response should have participant count", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for @sinan private message @devrim",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.ParticipantCount, ShouldEqual, 3)
		})

		Convey("private message response should have participant preview", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is @chris a body for @devrim private message",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(len(cmc.ParticipantsPreview), ShouldEqual, 3)
		})

		Convey("private message response should have last Message", func() {
			body := "hi @devrim this is a body for private message also for @chris"
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				body,
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.LastMessage.Message.Body, ShouldEqual, body)
		})

		Convey("private message should be listed by all recipients", func() {
			// use a different group name
			// in order not to interfere with another request
			groupName := "testgroup" + strconv.FormatInt(rand.Int63(), 10)

			body := "hi @devrim this is a body for private message also for @chris"
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				body,
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

			pm, err := rest.GetPrivateMessages(account.Id, groupName)
			So(err, ShouldBeNil)
			So(pm, ShouldNotBeNil)
			So(pm[0], ShouldNotBeNil)
			So(pm[0].Channel.TypeConstant, ShouldEqual, models.Channel_TYPE_PRIVATE_MESSAGE)
			So(pm[0].Channel.Id, ShouldEqual, cmc.Channel.Id)
			So(pm[0].Channel.GroupName, ShouldEqual, cmc.Channel.GroupName)
			So(pm[0].LastMessage.Message.Body, ShouldEqual, cmc.LastMessage.Message.Body)
			So(pm[0].Channel.PrivacyConstant, ShouldEqual, models.Channel_PRIVACY_PRIVATE)
			So(len(pm[0].ParticipantsPreview), ShouldEqual, 3)
			So(pm[0].IsParticipant, ShouldBeTrue)

		})

		Convey("targetted account should be able to list private message channel of himself", nil)

	})
}
<commit_msg>Social: update test functions with the new structure<commit_after>package main

import (
	"math/rand"
	"socialapi/models"
	"socialapi/rest"
	"strconv"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestPrivateMesssage(t *testing.T) {
	Convey("while testing private messages", t, func() {
		account := models.NewAccount()
		account.OldId = AccountOldId.Hex()
		account, err := rest.CreateAccount(account)
		So(err, ShouldBeNil)
		So(account, ShouldNotBeNil)

		recipient := models.NewAccount()
		recipient.OldId = AccountOldId2.Hex()
		recipient, err = rest.CreateAccount(recipient)
		So(err, ShouldBeNil)
		So(recipient, ShouldNotBeNil)

		recipient2 := models.NewAccount()
		recipient2.OldId = AccountOldId3.Hex()
		recipient2, err = rest.CreateAccount(recipient2)
		So(err, ShouldBeNil)
		So(recipient2, ShouldNotBeNil)

		groupName := "testgroup" + strconv.FormatInt(rand.Int63(), 10)

		Convey("one can send private message to one person", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body message for private message @chris @devrim @sinan",
				groupName,
				[]string{"chris", "devrim", "sinan"},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

		})

		Convey("0 recipient should not fail", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message",
				groupName,
				[]string{},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

		})
		Convey("if body is nil, should fail to create PM", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"",
				groupName,
				[]string{},
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)
		})
		Convey("if group name is nil, should not fail to create PM", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message @chris @devrim @sinan",
				"",
				[]string{"chris", "devrim", "sinan"},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
		})

		Convey("if sender is not defined should fail to create PM", func() {
			cmc, err := rest.SendPrivateMessage(
				0,
				"this is a body for private message",
				"",
				[]string{},
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)
		})

		Convey("one can send private message to multiple person", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message @sinan",
				groupName,
				[]string{"sinan"},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

		})
		Convey("private message response should have created channel", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message @devrim @sinan",
				groupName,
				[]string{"devrim", "sinan"},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.Channel.TypeConstant, ShouldEqual, models.Channel_TYPE_PRIVATE_MESSAGE)
			So(cmc.Channel.Id, ShouldBeGreaterThan, 0)
			So(cmc.Channel.GroupName, ShouldEqual, groupName)
			So(cmc.Channel.PrivacyConstant, ShouldEqual, models.Channel_PRIVACY_PRIVATE)

		})

		Convey("private message response should have participant status data", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for private message @chris @devrim @sinan",
				groupName,
				[]string{"chris", "devrim", "sinan"},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.IsParticipant, ShouldBeTrue)
		})

		Convey("private message response should have participant count", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is a body for @sinan private message @devrim",
				groupName,
				[]string{"devrim", "sinan"},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.ParticipantCount, ShouldEqual, 3)
		})

		Convey("private message response should have participant preview", func() {
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				"this is @chris a body for @devrim private message",
				groupName,
				[]string{"chris", "devrim"},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(len(cmc.ParticipantsPreview), ShouldEqual, 3)
		})

		Convey("private message response should have last Message", func() {
			body := "hi @devrim this is a body for private message also for @chris"
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				body,
				groupName,
				[]string{"chris", "devrim"},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.LastMessage.Message.Body, ShouldEqual, body)
		})

		Convey("private message should be listed by all recipients", func() {
			// use a different group name
			// in order not to interfere with another request
			groupName := "testgroup" + strconv.FormatInt(rand.Int63(), 10)

			body := "hi @devrim this is a body for private message also for @chris"
			cmc, err := rest.SendPrivateMessage(
				account.Id,
				body,
				groupName,
				[]string{"chris", "devrim"},
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

			pm, err := rest.GetPrivateMessages(account.Id, groupName)
			So(err, ShouldBeNil)
			So(pm, ShouldNotBeNil)
			So(pm[0], ShouldNotBeNil)
			So(pm[0].Channel.TypeConstant, ShouldEqual, models.Channel_TYPE_PRIVATE_MESSAGE)
			So(pm[0].Channel.Id, ShouldEqual, cmc.Channel.Id)
			So(pm[0].Channel.GroupName, ShouldEqual, cmc.Channel.GroupName)
			So(pm[0].LastMessage.Message.Body, ShouldEqual, cmc.LastMessage.Message.Body)
			So(pm[0].Channel.PrivacyConstant, ShouldEqual, models.Channel_PRIVACY_PRIVATE)
			So(len(pm[0].ParticipantsPreview), ShouldEqual, 3)
			So(pm[0].IsParticipant, ShouldBeTrue)

		})

		Convey("targetted account should be able to list private message channel of himself", nil)

	})
}
<|endoftext|>


<commit_before>// package fifonodupes manages a queue as described: an ordered
// queue with no duplicates.
// The configured processor function is called on an entry that's
// received and this happens in parallel until we reach maxCapacity
// go-routines are running in parallel.  Any new requests get queued
// and processed as a go routine becomes free.  The queue is processed
// in FIFO order.  If a request is received for an element that is
// already in the queue or being processed then it will be silently
// ignored.
package discovery

import (
	"errors"
	"sync"

	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/inst"
)

type Queue struct {
	concurrency    uint                      // current concurrency
	done           chan inst.InstanceKey     // for synchronising completed discoveries
	inputChan      <-chan inst.InstanceKey   // input channel we are reading from
	knownKeys      map[inst.InstanceKey]bool // pending instances so we don't queue anything up more than one
	lock           sync.Mutex                // lock while making changes
	maxConcurrency uint                      // maximum concurrency of the queue
	processor      func(i inst.InstanceKey)  // process to run on each received key
	queue          []inst.InstanceKey        // instances in fifo order so we process in the order received.
}

var emptyKey = inst.InstanceKey{}

// provide a channel to read from and a function to run on the instance to be processed
func NewQueue(maxConcurrency uint, inputChan chan inst.InstanceKey, processor func(i inst.InstanceKey)) *Queue {
	log.Infof("Queue.NewQueue()")
	q := new(Queue)

	q.concurrency = 0                    // explicitly
	q.done = make(chan inst.InstanceKey) // Do I need this to be larger?
	q.inputChan = inputChan
	q.knownKeys = make(map[inst.InstanceKey]bool)
	q.maxConcurrency = maxConcurrency
	q.processor = processor
	q.queue = make([]inst.InstanceKey, 0)

	return q
}

// add the key to the slice if it does not exist in known keys
// - goroutine safe as only called inside the mutex
func (q *Queue) push(key inst.InstanceKey) {
	if key == emptyKey {
		log.Fatal("Queue.push(%v) is empty", key)
	}
	// log.Debugf("Queue.push(%+v)", key)

	if _, found := q.knownKeys[key]; !found {
		// log.Debugf("Queue.push() adding %+v to knownKeys", key)
		// add to the items that are being processed
		q.knownKeys[key] = true
		q.queue = append(q.queue, key)
	} else {
		// If key already there we just ignore it as the request is in the queue.
		// the known key also records stuff in the queue, so pending + active jobs.
		// log.Debugf("Queue.push() ignoring knownKey %+v", key)
	}
}

// remove the entry and remove it from known keys
func (q *Queue) pop() (inst.InstanceKey, error) {
	if len(q.queue) == 0 {
		return inst.InstanceKey{}, errors.New("q.pop() on empty queue")
	}
	key := q.queue[0]
	q.queue = q.queue[1:]
	delete(q.knownKeys, key)
	// log.Debugf("Queue.pop() returns %+v", key)
	return key, nil
}

// dispatch a job from the queue (assumes we are in a locked state)
func (q *Queue) dispatch() {
	key, err := q.pop() // should never give an error but let's check anyway
	if err != nil {
		log.Fatal("Queue.dispatch() q.pop() returns: %+v", err)
		return
	}
	if key == emptyKey {
		log.Fatal("Queue.dispatch() key is empty")
	}

	q.concurrency++
	q.knownKeys[key] = true

	// log.Debugf("Queue.dispatch() key: %q, concurrency: %d", key, q.concurrency)

	// dispatch a discoverInstance() but tell us when we're done (to limit concurrency)
	go func() { // discover asynchronously
		q.processor(key)
		q.done <- key
	}()
}

// acknowledge a job has finished
// - we deal with the locking inside
func (q *Queue) acknowledgeJob(key inst.InstanceKey) {
	q.lock.Lock()
	delete(q.knownKeys, key)
	q.concurrency--
	// log.Debugf("Queue.acknowledgeJob(%+v) q.concurrency: %d", key, q.concurrency)
	q.lock.Unlock()
}

// drain queue by dispatching any jobs we have still
func (q *Queue) maybeDispatch() {
	q.lock.Lock()
	// log.Debugf("Queue.maybeDispatch() q.concurrency: %d, q.maxConcurrency: %d, len(q.queue): %d", q.concurrency, q.maxConcurrency, len(q.queue))
	if q.concurrency < q.maxConcurrency && len(q.queue) > 0 {
		q.dispatch()
	}
	q.lock.Unlock()
}

// add an entry to the queue and dispatch something if concurrency is low enough
// - we deal with locking inside
func (q *Queue) queueAndMaybeDispatch(key inst.InstanceKey) {
	if key == emptyKey {
		log.Fatal("Queue.queueAndMaybeDispatch(%v) is empty", key)
	}
	q.lock.Lock()
	// log.Debugf("Queue.queueAndMaybeDispatch(%+v) concurency: %d", key, q.concurrency)
	q.push(key)
	if q.concurrency < q.maxConcurrency && len(q.queue) > 0 {
		q.dispatch()
	}
	q.lock.Unlock()
}

// cleanup is called when the input channel closes.
// we can not sit in the loop so we have to wait for running go-routines to finish
// but also to dispatch anything left in the queue until finally everything is done.
func (q *Queue) cleanup() {
	log.Infof("Queue.cleanup()")
	for q.concurrency > 0 && len(q.queue) > 0 {
		q.maybeDispatch()
		if key, closed := <-q.done; closed {
			return
		} else {
			q.acknowledgeJob(key)
		}
	}
}

// Ends when all elements in the queue have been handled.
// we read from inputChan and call processor up to maxConcurrency times in parallel
func (q *Queue) HandleRequests() {
	if q == nil {
		log.Infof("Queue.HandleRequests() q == nil ??. Should not happen")

		// no queue, nothing to do
		return
	}
	log.Infof("Queue.NewQueue() processing requests")
	for {
		select {
		case key, ok := <-q.inputChan:
			if ok {
				if key != emptyKey {
					q.queueAndMaybeDispatch(key)
				} else {
					log.Warningf("Queue.HandleRequests() q.inputChan received empty key %+v, ignoring (fix the upstream code to prevent this)", key)
				}
			} else {
				q.cleanup()
				log.Infof("Queue.HandleRequests() q.inputChan is closed. returning")
				return
			}
		case key, ok := <-q.done:
			if ok {
				q.acknowledgeJob(key)
			} else {
				log.Infof("Queue.HandleRequests() q.done is closed. returning (shouldn't get here)")
				return // we shouldn't get here as the return above should get triggered first
			}
		}
	}
}
<commit_msg>Add missing license<commit_after>/*
   Copyright 2016 Simon J Mudd <sjmudd@pobox.com>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

// package discovery manages a queue as described: an ordered
// queue with no duplicates.
// The configured processor function is called on an entry that's
// received and this happens in parallel until we reach maxCapacity
// go-routines are running in parallel.  Any new requests get queued
// and processed as a go routine becomes free.  The queue is processed
// in FIFO order.  If a request is received for an element that is
// already in the queue or being processed then it will be silently
// ignored.
package discovery

import (
	"errors"
	"sync"

	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/inst"
)

type Queue struct {
	concurrency    uint                      // current concurrency
	done           chan inst.InstanceKey     // for synchronising completed discoveries
	inputChan      <-chan inst.InstanceKey   // input channel we are reading from
	knownKeys      map[inst.InstanceKey]bool // pending instances so we don't queue anything up more than one
	lock           sync.Mutex                // lock while making changes
	maxConcurrency uint                      // maximum concurrency of the queue
	processor      func(i inst.InstanceKey)  // process to run on each received key
	queue          []inst.InstanceKey        // instances in fifo order so we process in the order received.
}

var emptyKey = inst.InstanceKey{}

// provide a channel to read from and a function to run on the instance to be processed
func NewQueue(maxConcurrency uint, inputChan chan inst.InstanceKey, processor func(i inst.InstanceKey)) *Queue {
	log.Infof("Queue.NewQueue()")
	q := new(Queue)

	q.concurrency = 0                    // explicitly
	q.done = make(chan inst.InstanceKey) // Do I need this to be larger?
	q.inputChan = inputChan
	q.knownKeys = make(map[inst.InstanceKey]bool)
	q.maxConcurrency = maxConcurrency
	q.processor = processor
	q.queue = make([]inst.InstanceKey, 0)

	return q
}

// add the key to the slice if it does not exist in known keys
// - goroutine safe as only called inside the mutex
func (q *Queue) push(key inst.InstanceKey) {
	if key == emptyKey {
		log.Fatal("Queue.push(%v) is empty", key)
	}
	// log.Debugf("Queue.push(%+v)", key)

	if _, found := q.knownKeys[key]; !found {
		// log.Debugf("Queue.push() adding %+v to knownKeys", key)
		// add to the items that are being processed
		q.knownKeys[key] = true
		q.queue = append(q.queue, key)
	} else {
		// If key already there we just ignore it as the request is in the queue.
		// the known key also records stuff in the queue, so pending + active jobs.
		// log.Debugf("Queue.push() ignoring knownKey %+v", key)
	}
}

// remove the entry and remove it from known keys
func (q *Queue) pop() (inst.InstanceKey, error) {
	if len(q.queue) == 0 {
		return inst.InstanceKey{}, errors.New("q.pop() on empty queue")
	}
	key := q.queue[0]
	q.queue = q.queue[1:]
	delete(q.knownKeys, key)
	// log.Debugf("Queue.pop() returns %+v", key)
	return key, nil
}

// dispatch a job from the queue (assumes we are in a locked state)
func (q *Queue) dispatch() {
	key, err := q.pop() // should never give an error but let's check anyway
	if err != nil {
		log.Fatal("Queue.dispatch() q.pop() returns: %+v", err)
		return
	}
	if key == emptyKey {
		log.Fatal("Queue.dispatch() key is empty")
	}

	q.concurrency++
	q.knownKeys[key] = true

	// log.Debugf("Queue.dispatch() key: %q, concurrency: %d", key, q.concurrency)

	// dispatch a discoverInstance() but tell us when we're done (to limit concurrency)
	go func() { // discover asynchronously
		q.processor(key)
		q.done <- key
	}()
}

// acknowledge a job has finished
// - we deal with the locking inside
func (q *Queue) acknowledgeJob(key inst.InstanceKey) {
	q.lock.Lock()
	delete(q.knownKeys, key)
	q.concurrency--
	// log.Debugf("Queue.acknowledgeJob(%+v) q.concurrency: %d", key, q.concurrency)
	q.lock.Unlock()
}

// drain queue by dispatching any jobs we have still
func (q *Queue) maybeDispatch() {
	q.lock.Lock()
	// log.Debugf("Queue.maybeDispatch() q.concurrency: %d, q.maxConcurrency: %d, len(q.queue): %d", q.concurrency, q.maxConcurrency, len(q.queue))
	if q.concurrency < q.maxConcurrency && len(q.queue) > 0 {
		q.dispatch()
	}
	q.lock.Unlock()
}

// add an entry to the queue and dispatch something if concurrency is low enough
// - we deal with locking inside
func (q *Queue) queueAndMaybeDispatch(key inst.InstanceKey) {
	if key == emptyKey {
		log.Fatal("Queue.queueAndMaybeDispatch(%v) is empty", key)
	}
	q.lock.Lock()
	// log.Debugf("Queue.queueAndMaybeDispatch(%+v) concurency: %d", key, q.concurrency)
	q.push(key)
	if q.concurrency < q.maxConcurrency && len(q.queue) > 0 {
		q.dispatch()
	}
	q.lock.Unlock()
}

// cleanup is called when the input channel closes.
// we can not sit in the loop so we have to wait for running go-routines to finish
// but also to dispatch anything left in the queue until finally everything is done.
func (q *Queue) cleanup() {
	log.Infof("Queue.cleanup()")
	for q.concurrency > 0 && len(q.queue) > 0 {
		q.maybeDispatch()
		if key, closed := <-q.done; closed {
			return
		} else {
			q.acknowledgeJob(key)
		}
	}
}

// Ends when all elements in the queue have been handled.
// we read from inputChan and call processor up to maxConcurrency times in parallel
func (q *Queue) HandleRequests() {
	if q == nil {
		log.Infof("Queue.HandleRequests() q == nil ??. Should not happen")

		// no queue, nothing to do
		return
	}
	log.Infof("Queue.NewQueue() processing requests")
	for {
		select {
		case key, ok := <-q.inputChan:
			if ok {
				if key != emptyKey {
					q.queueAndMaybeDispatch(key)
				} else {
					log.Warningf("Queue.HandleRequests() q.inputChan received empty key %+v, ignoring (fix the upstream code to prevent this)", key)
				}
			} else {
				q.cleanup()
				log.Infof("Queue.HandleRequests() q.inputChan is closed. returning")
				return
			}
		case key, ok := <-q.done:
			if ok {
				q.acknowledgeJob(key)
			} else {
				log.Infof("Queue.HandleRequests() q.done is closed. returning (shouldn't get here)")
				return // we shouldn't get here as the return above should get triggered first
			}
		}
	}
}
<|endoftext|>


<commit_before>package engine

import (
	"testing"

	"github.com/keybase/client/go/libkb"
)

func TestCurrentUID(t *testing.T) {
	tc := SetupEngineTest(t, "current")
	defer tc.Cleanup()
	CreateAndSignupFakeUser(tc, "login")

	currentUID, err := CurrentUID(tc.G)
	if err != nil {
		t.Fatal(err)
	}

	loadedUser, err := libkb.LoadMe(libkb.NewLoadUserArg(tc.G))
	if err != nil {
		t.Fatal(err)
	}

	if currentUID != loadedUser.GetUID() {
		t.Errorf("current uid: %s, loaded uid: %s\n", currentUID, loadedUser.GetUID())
	}

	Logout(tc)

	currentUID, err = CurrentUID(tc.G)
	if err == nil {
		t.Fatal("expected error in CurrentUID when logged out")
	}
	if _, ok := err.(libkb.LoginRequiredError); !ok {
		t.Fatalf("expected LoginRequiredError, got %T", err)
	}
}
<commit_msg>Test logout, login current uid still ok<commit_after>package engine

import (
	"testing"

	"github.com/keybase/client/go/libkb"
)

func TestCurrentUID(t *testing.T) {
	tc := SetupEngineTest(t, "current")
	defer tc.Cleanup()
	u := CreateAndSignupFakeUser(tc, "login")

	currentUID, err := CurrentUID(tc.G)
	if err != nil {
		t.Fatal(err)
	}

	loadedUser, err := libkb.LoadMe(libkb.NewLoadUserArg(tc.G))
	if err != nil {
		t.Fatal(err)
	}

	if currentUID != loadedUser.GetUID() {
		t.Errorf("current uid: %s, loaded uid: %s\n", currentUID, loadedUser.GetUID())
	}

	Logout(tc)

	currentUID, err = CurrentUID(tc.G)
	if err == nil {
		t.Fatal("expected error in CurrentUID when logged out")
	}
	if _, ok := err.(libkb.LoginRequiredError); !ok {
		t.Fatalf("expected LoginRequiredError, got %T", err)
	}

	u.LoginOrBust(tc)
	currentUID, err = CurrentUID(tc.G)
	if err != nil {
		t.Fatal(err)
	}
	if currentUID != loadedUser.GetUID() {
		t.Errorf("after logout/login: current uid: %s, loaded uid: %s\n", currentUID, loadedUser.GetUID())
	}
}
<|endoftext|>


<commit_before>// ----------------------------------------------------------------------------
//
//     ***     AUTO GENERATED CODE    ***    AUTO GENERATED CODE     ***
//
// ----------------------------------------------------------------------------
//
//     This file is automatically generated by Magic Modules and manual
//     changes will be clobbered when the file is regenerated.
//
//     Please read more about how to change this file in
//     .github/CONTRIBUTING.md.
//
// ----------------------------------------------------------------------------

package google

import "reflect"

func GetComputeNetworkCaiObject(d TerraformResourceData, config *Config) (Asset, error) {
	name, err := assetName(d, config, "//compute.googleapis.com/projects/{{project}}/global/networks/{{name}}")
	if err != nil {
		return Asset{}, err
	}
	if obj, err := GetComputeNetworkApiObject(d, config); err == nil {
		return Asset{
			Name: name,
			Type: "compute.googleapis.com/Network",
			Resource: &AssetResource{
				Version:              "v1",
				DiscoveryDocumentURI: "https://www.googleapis.com/discovery/v1/apis/compute/v1/rest",
				DiscoveryName:        "Network",
				Data:                 obj,
			},
		}, nil
	} else {
		return Asset{}, err
	}
}

func GetComputeNetworkApiObject(d TerraformResourceData, config *Config) (map[string]interface{}, error) {
	obj := make(map[string]interface{})
	descriptionProp, err := expandComputeNetworkDescription(d.Get("description"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("description"); !isEmptyValue(reflect.ValueOf(descriptionProp)) && (ok || !reflect.DeepEqual(v, descriptionProp)) {
		obj["description"] = descriptionProp
	}
	nameProp, err := expandComputeNetworkName(d.Get("name"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("name"); !isEmptyValue(reflect.ValueOf(nameProp)) && (ok || !reflect.DeepEqual(v, nameProp)) {
		obj["name"] = nameProp
	}
	autoCreateSubnetworksProp, err := expandComputeNetworkAutoCreateSubnetworks(d.Get("auto_create_subnetworks"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("auto_create_subnetworks"); ok || !reflect.DeepEqual(v, autoCreateSubnetworksProp) {
		obj["autoCreateSubnetworks"] = autoCreateSubnetworksProp
	}
	routingConfigProp, err := expandComputeNetworkRoutingConfig(nil, d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("routing_config"); !isEmptyValue(reflect.ValueOf(routingConfigProp)) && (ok || !reflect.DeepEqual(v, routingConfigProp)) {
		obj["routingConfig"] = routingConfigProp
	}

	return obj, nil
}

func expandComputeNetworkDescription(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandComputeNetworkName(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandComputeNetworkAutoCreateSubnetworks(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandComputeNetworkRoutingConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	transformed := make(map[string]interface{})
	transformedRoutingMode, err := expandComputeNetworkRoutingConfigRoutingMode(d.Get("routing_mode"), d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedRoutingMode); val.IsValid() && !isEmptyValue(val) {
		transformed["routingMode"] = transformedRoutingMode
	}

	return transformed, nil
}

func expandComputeNetworkRoutingConfigRoutingMode(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}
<commit_msg>vpc network custom mtu support (#4126) (#553)<commit_after>// ----------------------------------------------------------------------------
//
//     ***     AUTO GENERATED CODE    ***    AUTO GENERATED CODE     ***
//
// ----------------------------------------------------------------------------
//
//     This file is automatically generated by Magic Modules and manual
//     changes will be clobbered when the file is regenerated.
//
//     Please read more about how to change this file in
//     .github/CONTRIBUTING.md.
//
// ----------------------------------------------------------------------------

package google

import "reflect"

func GetComputeNetworkCaiObject(d TerraformResourceData, config *Config) (Asset, error) {
	name, err := assetName(d, config, "//compute.googleapis.com/projects/{{project}}/global/networks/{{name}}")
	if err != nil {
		return Asset{}, err
	}
	if obj, err := GetComputeNetworkApiObject(d, config); err == nil {
		return Asset{
			Name: name,
			Type: "compute.googleapis.com/Network",
			Resource: &AssetResource{
				Version:              "v1",
				DiscoveryDocumentURI: "https://www.googleapis.com/discovery/v1/apis/compute/v1/rest",
				DiscoveryName:        "Network",
				Data:                 obj,
			},
		}, nil
	} else {
		return Asset{}, err
	}
}

func GetComputeNetworkApiObject(d TerraformResourceData, config *Config) (map[string]interface{}, error) {
	obj := make(map[string]interface{})
	descriptionProp, err := expandComputeNetworkDescription(d.Get("description"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("description"); !isEmptyValue(reflect.ValueOf(descriptionProp)) && (ok || !reflect.DeepEqual(v, descriptionProp)) {
		obj["description"] = descriptionProp
	}
	nameProp, err := expandComputeNetworkName(d.Get("name"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("name"); !isEmptyValue(reflect.ValueOf(nameProp)) && (ok || !reflect.DeepEqual(v, nameProp)) {
		obj["name"] = nameProp
	}
	autoCreateSubnetworksProp, err := expandComputeNetworkAutoCreateSubnetworks(d.Get("auto_create_subnetworks"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("auto_create_subnetworks"); ok || !reflect.DeepEqual(v, autoCreateSubnetworksProp) {
		obj["autoCreateSubnetworks"] = autoCreateSubnetworksProp
	}
	routingConfigProp, err := expandComputeNetworkRoutingConfig(nil, d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("routing_config"); !isEmptyValue(reflect.ValueOf(routingConfigProp)) && (ok || !reflect.DeepEqual(v, routingConfigProp)) {
		obj["routingConfig"] = routingConfigProp
	}
	mtuProp, err := expandComputeNetworkMtu(d.Get("mtu"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("mtu"); !isEmptyValue(reflect.ValueOf(mtuProp)) && (ok || !reflect.DeepEqual(v, mtuProp)) {
		obj["mtu"] = mtuProp
	}

	return obj, nil
}

func expandComputeNetworkDescription(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandComputeNetworkName(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandComputeNetworkAutoCreateSubnetworks(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandComputeNetworkRoutingConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	transformed := make(map[string]interface{})
	transformedRoutingMode, err := expandComputeNetworkRoutingConfigRoutingMode(d.Get("routing_mode"), d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedRoutingMode); val.IsValid() && !isEmptyValue(val) {
		transformed["routingMode"] = transformedRoutingMode
	}

	return transformed, nil
}

func expandComputeNetworkRoutingConfigRoutingMode(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandComputeNetworkMtu(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}
<|endoftext|>