<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/xenolf/lego/acme"
)

func checkFolder(path string) error {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return os.MkdirAll(path, 0700)
	}
	return nil
}

func setup(c *cli.Context) (*Configuration, *Account, *acme.Client) {
	err := checkFolder(c.GlobalString("path"))
	if err != nil {
		logger().Fatalf("Cound not check/create path: %v", err)
	}

	conf := NewConfiguration(c)
	if !c.GlobalIsSet("email") {
		logger().Fatal("You have to pass an account (email address) to the program using --email or -m")
	}

	//TODO: move to account struct? Currently MUST pass email.
	acc := NewAccount(c.GlobalString("email"), conf)

	client, err := acme.NewClient(c.GlobalString("server"), acc, conf.RsaBits(), conf.OptPort())
	if err != nil {
		logger().Fatal("Could not create client:", err)
	}

	return conf, acc, client
}

func saveCertRes(certRes acme.CertificateResource, conf *Configuration) {
	// We store the certificate, private key and metadata in different files
	// as web servers would not be able to work with a combined file.
	certOut := path.Join(conf.CertPath(), certRes.Domain+".crt")
	privOut := path.Join(conf.CertPath(), certRes.Domain+".key")
	metaOut := path.Join(conf.CertPath(), certRes.Domain+".json")

	err := ioutil.WriteFile(certOut, certRes.Certificate, 0600)
	if err != nil {
		logger().Printf("Unable to save Certificate for domain %s\n\t%v", certRes.Domain, err)
	}

	err = ioutil.WriteFile(privOut, certRes.PrivateKey, 0600)
	if err != nil {
		logger().Printf("Unable to save PrivateKey for domain %s\n\t%v", certRes.Domain, err)
	}

	jsonBytes, err := json.MarshalIndent(certRes, "", "\t")
	if err != nil {
		logger().Printf("Unable to marshal CertResource for domain %s\n\t%v", certRes.Domain, err)
	}

	err = ioutil.WriteFile(metaOut, jsonBytes, 0600)
	if err != nil {
		logger().Printf("Unable to save CertResource for domain %s\n\t%v", certRes.Domain, err)
	}
}

func run(c *cli.Context) {
	conf, acc, client := setup(c)
	if acc.Registration == nil {
		reg, err := client.Register()
		if err != nil {
			logger().Fatalf("Could not complete registration\n\t%v", err)
		}

		acc.Registration = reg
		acc.Save()

		logger().Print("!!!! HEADS UP !!!!")
		logger().Printf(`
		Your account credentials have been saved in your Let's Encrypt
		configuration directory at "%s".
		You should make a secure backup	of this folder now. This
		configuration directory will also contain certificates and
		private keys obtained from Let's Encrypt so making regular
		backups of this folder is ideal.`, conf.AccountPath(c.GlobalString("email")))

	}

	if acc.Registration.Body.Agreement == "" {
		reader := bufio.NewReader(os.Stdin)
		logger().Printf("Please review the TOS at %s", acc.Registration.TosURL)

		for {
			logger().Println("Do you accept the TOS? Y/n")
			text, err := reader.ReadString('\n')
			if err != nil {
				logger().Fatalf("Could not read from console -> %v", err)
			}

			text = strings.Trim(text, "\r\n")

			if text == "n" {
				logger().Fatal("You did not accept the TOS. Unable to proceed.")
			}

			if text == "Y" || text == "y" || text == "" {
				err = client.AgreeToTOS()
				if err != nil {
					logger().Fatalf("Could not agree to tos -> %v", err)
				}
				acc.Save()
				break
			}

			logger().Println("Your input was invalid. Please answer with one of Y/y, n or by pressing enter.")
		}
	}

	if !c.GlobalIsSet("domains") {
		logger().Fatal("Please specify --domains")
	}

	certs, failures := client.ObtainCertificates(c.GlobalStringSlice("domains"), true)
	if len(failures) > 0 {
		for k, v := range failures {
			logger().Printf("[%s] Could not obtain certificates\n\t%v", k, v)
		}
	}

	err := checkFolder(conf.CertPath())
	if err != nil {
		logger().Fatalf("Cound not check/create path: %v", err)
	}

	for _, certRes := range certs {
		saveCertRes(certRes, conf)
	}
}

func revoke(c *cli.Context) {

	conf, _, client := setup(c)

	err := checkFolder(conf.CertPath())
	if err != nil {
		logger().Fatalf("Cound not check/create path: %v", err)
	}

	for _, domain := range c.GlobalStringSlice("domains") {
		logger().Printf("Trying to revoke certificate for domain %s", domain)

		certPath := path.Join(conf.CertPath(), domain+".crt")
		certBytes, err := ioutil.ReadFile(certPath)

		err = client.RevokeCertificate(certBytes)
		if err != nil {
			logger().Printf("Error while revoking the certificate for domain %s\n\t%v", domain, err)
		} else {
			logger().Print("Certificate was revoked.")
		}
	}
}

func renew(c *cli.Context) {
	conf, _, client := setup(c)

	for _, domain := range c.GlobalStringSlice("domains") {
		// load the cert resource from files.
		// We store the certificate, private key and metadata in different files
		// as web servers would not be able to work with a combined file.
		certPath := path.Join(conf.CertPath(), domain+".crt")
		privPath := path.Join(conf.CertPath(), domain+".key")
		metaPath := path.Join(conf.CertPath(), domain+".json")

		certBytes, err := ioutil.ReadFile(certPath)
		if err != nil {
			logger().Printf("Error while loading the certificate for domain %s\n\t%v", domain, err)
			return
		}

		keyBytes, err := ioutil.ReadFile(privPath)
		if err != nil {
			logger().Printf("Error while loading the private key for domain %s\n\t%v", domain, err)
			return
		}

		metaBytes, err := ioutil.ReadFile(metaPath)
		if err != nil {
			logger().Printf("Error while loading the meta data for domain %s\n\t%v", domain, err)
			return
		}

		var certRes acme.CertificateResource
		err = json.Unmarshal(metaBytes, &certRes)
		if err != nil {
			logger().Printf("Error while marshalling the meta data for domain %s\n\t%v", domain, err)
			return
		}

		certRes.PrivateKey = keyBytes
		certRes.Certificate = certBytes

		newCert, err := client.RenewCertificate(certRes, true, true)
		if err != nil {
			logger().Printf("%v", err)
			return
		}

		saveCertRes(newCert, conf)
	}
}
<commit_msg>Make the CLI use SAN certs<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/xenolf/lego/acme"
)

func checkFolder(path string) error {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return os.MkdirAll(path, 0700)
	}
	return nil
}

func setup(c *cli.Context) (*Configuration, *Account, *acme.Client) {
	err := checkFolder(c.GlobalString("path"))
	if err != nil {
		logger().Fatalf("Cound not check/create path: %v", err)
	}

	conf := NewConfiguration(c)
	if !c.GlobalIsSet("email") {
		logger().Fatal("You have to pass an account (email address) to the program using --email or -m")
	}

	//TODO: move to account struct? Currently MUST pass email.
	acc := NewAccount(c.GlobalString("email"), conf)

	client, err := acme.NewClient(c.GlobalString("server"), acc, conf.RsaBits(), conf.OptPort())
	if err != nil {
		logger().Fatal("Could not create client:", err)
	}

	return conf, acc, client
}

func saveCertRes(certRes acme.CertificateResource, conf *Configuration) {
	// We store the certificate, private key and metadata in different files
	// as web servers would not be able to work with a combined file.
	certOut := path.Join(conf.CertPath(), certRes.Domain+".crt")
	privOut := path.Join(conf.CertPath(), certRes.Domain+".key")
	metaOut := path.Join(conf.CertPath(), certRes.Domain+".json")

	err := ioutil.WriteFile(certOut, certRes.Certificate, 0600)
	if err != nil {
		logger().Printf("Unable to save Certificate for domain %s\n\t%v", certRes.Domain, err)
	}

	err = ioutil.WriteFile(privOut, certRes.PrivateKey, 0600)
	if err != nil {
		logger().Printf("Unable to save PrivateKey for domain %s\n\t%v", certRes.Domain, err)
	}

	jsonBytes, err := json.MarshalIndent(certRes, "", "\t")
	if err != nil {
		logger().Printf("Unable to marshal CertResource for domain %s\n\t%v", certRes.Domain, err)
	}

	err = ioutil.WriteFile(metaOut, jsonBytes, 0600)
	if err != nil {
		logger().Printf("Unable to save CertResource for domain %s\n\t%v", certRes.Domain, err)
	}
}

func run(c *cli.Context) {
	conf, acc, client := setup(c)
	if acc.Registration == nil {
		reg, err := client.Register()
		if err != nil {
			logger().Fatalf("Could not complete registration\n\t%v", err)
		}

		acc.Registration = reg
		acc.Save()

		logger().Print("!!!! HEADS UP !!!!")
		logger().Printf(`
		Your account credentials have been saved in your Let's Encrypt
		configuration directory at "%s".
		You should make a secure backup	of this folder now. This
		configuration directory will also contain certificates and
		private keys obtained from Let's Encrypt so making regular
		backups of this folder is ideal.`, conf.AccountPath(c.GlobalString("email")))

	}

	if acc.Registration.Body.Agreement == "" {
		reader := bufio.NewReader(os.Stdin)
		logger().Printf("Please review the TOS at %s", acc.Registration.TosURL)

		for {
			logger().Println("Do you accept the TOS? Y/n")
			text, err := reader.ReadString('\n')
			if err != nil {
				logger().Fatalf("Could not read from console -> %v", err)
			}

			text = strings.Trim(text, "\r\n")

			if text == "n" {
				logger().Fatal("You did not accept the TOS. Unable to proceed.")
			}

			if text == "Y" || text == "y" || text == "" {
				err = client.AgreeToTOS()
				if err != nil {
					logger().Fatalf("Could not agree to tos -> %v", err)
				}
				acc.Save()
				break
			}

			logger().Println("Your input was invalid. Please answer with one of Y/y, n or by pressing enter.")
		}
	}

	if !c.GlobalIsSet("domains") {
		logger().Fatal("Please specify --domains")
	}

	cert, failures := client.ObtainSANCertificate(c.GlobalStringSlice("domains"), true)
	if len(failures) > 0 {
		for k, v := range failures {
			logger().Printf("[%s] Could not obtain certificates\n\t%v", k, v)
		}
	}

	err := checkFolder(conf.CertPath())
	if err != nil {
		logger().Fatalf("Cound not check/create path: %v", err)
	}

	//for _, certRes := range certs {
	//		saveCertRes(certRes, conf)
	//}
	saveCertRes(cert, conf)
}

func revoke(c *cli.Context) {

	conf, _, client := setup(c)

	err := checkFolder(conf.CertPath())
	if err != nil {
		logger().Fatalf("Cound not check/create path: %v", err)
	}

	for _, domain := range c.GlobalStringSlice("domains") {
		logger().Printf("Trying to revoke certificate for domain %s", domain)

		certPath := path.Join(conf.CertPath(), domain+".crt")
		certBytes, err := ioutil.ReadFile(certPath)

		err = client.RevokeCertificate(certBytes)
		if err != nil {
			logger().Printf("Error while revoking the certificate for domain %s\n\t%v", domain, err)
		} else {
			logger().Print("Certificate was revoked.")
		}
	}
}

func renew(c *cli.Context) {
	conf, _, client := setup(c)

	for _, domain := range c.GlobalStringSlice("domains") {
		// load the cert resource from files.
		// We store the certificate, private key and metadata in different files
		// as web servers would not be able to work with a combined file.
		certPath := path.Join(conf.CertPath(), domain+".crt")
		privPath := path.Join(conf.CertPath(), domain+".key")
		metaPath := path.Join(conf.CertPath(), domain+".json")

		certBytes, err := ioutil.ReadFile(certPath)
		if err != nil {
			logger().Printf("Error while loading the certificate for domain %s\n\t%v", domain, err)
			return
		}

		keyBytes, err := ioutil.ReadFile(privPath)
		if err != nil {
			logger().Printf("Error while loading the private key for domain %s\n\t%v", domain, err)
			return
		}

		metaBytes, err := ioutil.ReadFile(metaPath)
		if err != nil {
			logger().Printf("Error while loading the meta data for domain %s\n\t%v", domain, err)
			return
		}

		var certRes acme.CertificateResource
		err = json.Unmarshal(metaBytes, &certRes)
		if err != nil {
			logger().Printf("Error while marshalling the meta data for domain %s\n\t%v", domain, err)
			return
		}

		certRes.PrivateKey = keyBytes
		certRes.Certificate = certBytes

		newCert, err := client.RenewCertificate(certRes, true, true)
		if err != nil {
			logger().Printf("%v", err)
			return
		}

		saveCertRes(newCert, conf)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"path"
	"strconv"
	"sync"
	"time"

	"github.com/giagiannis/data-profiler/core"
)

// TaskEngine is deployed once for the server's lifetime and keeps the tasks
// which are executed.
type TaskEngine struct {
	Tasks []*Task
	lock  sync.Mutex
}

// NewTaskEngine initializes a new TasEngine object. Called once per server deployment.
func NewTaskEngine() *TaskEngine {
	te := new(TaskEngine)
	te.lock = *new(sync.Mutex)
	return te
}

// Submit appends a new task to the task engine and initializes its execution.
func (e *TaskEngine) Submit(t *Task) {
	if t == nil {
		return
	}
	e.lock.Lock()
	go t.Run()
	e.Tasks = append(e.Tasks, t)
	e.lock.Unlock()
}

// Task is the primitive struct that represents a task of the server.
type Task struct {
	Status      string
	Started     time.Time
	Duration    float64
	Description string
	Dataset     *ModelDataset
	fnc         func() error
}

// Run is responsible to execute to task's method and update the task status
// accordingly.
func (t *Task) Run() {
	t.Started = time.Now()
	t.Status = "RUNNING"
	err := t.fnc()
	if err != nil {
		t.Status = "ERROR - " + err.Error()
	} else {
		t.Status = "DONE"
		t.Duration = time.Since(t.Started).Seconds()
	}
}

// NewSMComputationTask initializes a new Similarity Matrix computation task.
func NewSMComputationTask(datasetID string, conf map[string]string) *Task {
	task := new(Task)
	dts := modelDatasetGetInfo(datasetID)
	task.Dataset = dts
	task.Description = fmt.Sprintf("SM Computation for %s, type %s\n",
		dts.Name, conf["estimatorType"])
	task.fnc = func() error {
		datasets := core.DiscoverDatasets(dts.Path)
		estType := *core.NewDatasetSimilarityEstimatorType(conf["estimatorType"])
		est := core.NewDatasetSimilarityEstimator(estType, datasets)
		est.Configure(conf)
		if conf["popPolicy"] == "aprx" {
			pop := new(core.DatasetSimilarityPopulationPolicy)
			pop.PolicyType = core.PopulationPolicyAprx
			val, err := strconv.ParseFloat(conf["popParameterValue"], 64)
			if err != nil {
				log.Println(err)
			}
			if conf["popParameter"] == "popCount" {
				pop.Parameters = map[string]float64{"count": val}
			}
			if conf["popParameter"] == "popThreshold" {
				pop.Parameters = map[string]float64{"threshold": val}
			}
			est.SetPopulationPolicy(*pop)
		}
		err := est.Compute()
		if err != nil {
			return err
		}
		sm := est.SimilarityMatrix()
		//		var smID string
		if sm != nil {
			//smID =
			modelSimilarityMatrixInsert(datasetID, sm.Serialize(), est.Serialize(), conf)
		}
		//modelEstimatorInsert(datasetID, smID, est.Serialize(), conf)
		return nil
	}
	return task
}

// NewMDSComputationTask initializes a new Multidimensional Scaling execution task.
func NewMDSComputationTask(smID, datasetID string, conf map[string]string) *Task {
	smModel := modelSimilarityMatrixGet(smID)
	if smModel == nil {
		log.Println("SM not found")
		return nil
	}

	cnt, err := ioutil.ReadFile(smModel.Path)
	if err != nil {
		log.Println(err)
	}
	sm := new(core.DatasetSimilarityMatrix)
	sm.Deserialize(cnt)
	k, err := strconv.ParseInt(conf["k"], 10, 64)
	if err != nil {
		log.Println(err)
	}

	dat := modelDatasetGetInfo(datasetID)
	task := new(Task)
	task.Dataset = dat
	task.Description = fmt.Sprintf("MDS Execution for %s with k=%d\n",
		dat.Name, k)
	task.fnc = func() error {
		mds := core.NewMDScaling(sm, int(k), Conf.Scripts.MDS)
		err = mds.Compute()
		if err != nil {
			return err
		}
		gof := fmt.Sprintf("%.5f", mds.Gof())
		stress := fmt.Sprintf("%.5f", mds.Stress())
		modelCoordinatesInsert(mds.Coordinates(), dat.ID, conf["k"], gof, stress, smID)
		return nil
	}
	return task
}

// NewOperatorRunTask initializes a new operator execution task.
func NewOperatorRunTask(operatorID string) *Task {
	m := modelOperatorGet(operatorID)
	if m == nil {
		log.Println("Operator was not found")
		return nil
	}
	dat := modelDatasetGetInfo(m.DatasetID)
	for _, f := range modelDatasetGetFiles(dat.ID) {
		dat.Files = append(dat.Files, dat.Path+"/"+f)
	}
	task := new(Task)
	task.Description = fmt.Sprintf("%s evaluation", m.Name)
	task.Dataset = dat
	task.fnc = func() error {
		eval, err := core.NewDatasetEvaluator(core.OnlineEval,
			map[string]string{
				"script":  m.Path,
				"testset": "",
			})
		if err != nil {
			log.Println(err)
		}
		scores := core.NewDatasetScores()
		for _, f := range dat.Files {
			s, err := eval.Evaluate(f)
			if err != nil {
				log.Println(err)
			} else {
				scores.Scores[path.Base(f)] = s
			}
		}
		cnt, _ := scores.Serialize()
		modelOperatorScoresInsert(operatorID, cnt)
		return nil
	}
	return task
}

// NewModelTrainTask generates a new ML for a given (dataset,operator) combination,
// according to the user-specified parameters.
func NewModelTrainTask(datasetID, operatorID string, sr float64,
	modelType string,
	coordinatesID, mlScript string,
	matrixID, k string) *Task {
	m := modelDatasetGetInfo(datasetID)
	task := new(Task)
	task.Description = fmt.Sprintf("Model training (%s for %s)", path.Base(mlScript), m.Name)
	task.Dataset = m
	task.fnc = func() error {
		datasets := core.DiscoverDatasets(m.Path)
		o := modelOperatorGet(operatorID)
		var evaluator core.DatasetEvaluator
		var err error
		if o.ScoresFile != "" {
			evaluator, err = core.NewDatasetEvaluator(core.FileBasedEval, map[string]string{"scores": o.ScoresFile})
		} else {
			evaluator, err = core.NewDatasetEvaluator(core.OnlineEval, map[string]string{"script": o.Path, "testset": ""})
		}
		if err != nil {
			log.Println(err)
			return err
		}
		t := core.NewModelerType(modelType)
		modeler := core.NewModeler(t, datasets, sr, evaluator)
		var conf map[string]string
		if t == core.ScriptBasedModelerType {
			c := modelCoordinatesGet(coordinatesID)
			conf = map[string]string{"script": mlScript, "coordinates": c.Path}
		} else if t == core.KNNModelerType {
			m := modelSimilarityMatrixGet(matrixID)
			conf = map[string]string{"k": k, "smatrix": m.Path}
		}
		modeler.Configure(conf)
		err = modeler.Run()
		if err != nil {
			log.Println(err)
			return err
		}
		// serialze appxValues
		var cnt [][]float64
		cnt = append(cnt, modeler.AppxValues())
		appxBuffer := serializeCSVFile(cnt)
		samplesBuffer, _ := json.Marshal(modeler.Samples())
		errors := make(map[string]string)
		for k, v := range modeler.ErrorMetrics() {
			errors[k] = fmt.Sprintf("%.5f", v)
		}
		modelDatasetModelInsert(coordinatesID, operatorID, datasetID, samplesBuffer, appxBuffer, conf, errors, sr)
		return nil
	}
	return task
}
<commit_msg>now storing base path of coordinates/sim matrix during modeling to save some space<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"path"
	"strconv"
	"sync"
	"time"

	"github.com/giagiannis/data-profiler/core"
)

// TaskEngine is deployed once for the server's lifetime and keeps the tasks
// which are executed.
type TaskEngine struct {
	Tasks []*Task
	lock  sync.Mutex
}

// NewTaskEngine initializes a new TasEngine object. Called once per server deployment.
func NewTaskEngine() *TaskEngine {
	te := new(TaskEngine)
	te.lock = *new(sync.Mutex)
	return te
}

// Submit appends a new task to the task engine and initializes its execution.
func (e *TaskEngine) Submit(t *Task) {
	if t == nil {
		return
	}
	e.lock.Lock()
	go t.Run()
	e.Tasks = append(e.Tasks, t)
	e.lock.Unlock()
}

// Task is the primitive struct that represents a task of the server.
type Task struct {
	Status      string
	Started     time.Time
	Duration    float64
	Description string
	Dataset     *ModelDataset
	fnc         func() error
}

// Run is responsible to execute to task's method and update the task status
// accordingly.
func (t *Task) Run() {
	t.Started = time.Now()
	t.Status = "RUNNING"
	err := t.fnc()
	if err != nil {
		t.Status = "ERROR - " + err.Error()
	} else {
		t.Status = "DONE"
		t.Duration = time.Since(t.Started).Seconds()
	}
}

// NewSMComputationTask initializes a new Similarity Matrix computation task.
func NewSMComputationTask(datasetID string, conf map[string]string) *Task {
	task := new(Task)
	dts := modelDatasetGetInfo(datasetID)
	task.Dataset = dts
	task.Description = fmt.Sprintf("SM Computation for %s, type %s\n",
		dts.Name, conf["estimatorType"])
	task.fnc = func() error {
		datasets := core.DiscoverDatasets(dts.Path)
		estType := *core.NewDatasetSimilarityEstimatorType(conf["estimatorType"])
		est := core.NewDatasetSimilarityEstimator(estType, datasets)
		est.Configure(conf)
		if conf["popPolicy"] == "aprx" {
			pop := new(core.DatasetSimilarityPopulationPolicy)
			pop.PolicyType = core.PopulationPolicyAprx
			val, err := strconv.ParseFloat(conf["popParameterValue"], 64)
			if err != nil {
				log.Println(err)
			}
			if conf["popParameter"] == "popCount" {
				pop.Parameters = map[string]float64{"count": val}
			}
			if conf["popParameter"] == "popThreshold" {
				pop.Parameters = map[string]float64{"threshold": val}
			}
			est.SetPopulationPolicy(*pop)
		}
		err := est.Compute()
		if err != nil {
			return err
		}
		sm := est.SimilarityMatrix()
		//		var smID string
		if sm != nil {
			//smID =
			modelSimilarityMatrixInsert(datasetID, sm.Serialize(), est.Serialize(), conf)
		}
		//modelEstimatorInsert(datasetID, smID, est.Serialize(), conf)
		return nil
	}
	return task
}

// NewMDSComputationTask initializes a new Multidimensional Scaling execution task.
func NewMDSComputationTask(smID, datasetID string, conf map[string]string) *Task {
	smModel := modelSimilarityMatrixGet(smID)
	if smModel == nil {
		log.Println("SM not found")
		return nil
	}

	cnt, err := ioutil.ReadFile(smModel.Path)
	if err != nil {
		log.Println(err)
	}
	sm := new(core.DatasetSimilarityMatrix)
	sm.Deserialize(cnt)
	k, err := strconv.ParseInt(conf["k"], 10, 64)
	if err != nil {
		log.Println(err)
	}

	dat := modelDatasetGetInfo(datasetID)
	task := new(Task)
	task.Dataset = dat
	task.Description = fmt.Sprintf("MDS Execution for %s with k=%d\n",
		dat.Name, k)
	task.fnc = func() error {
		mds := core.NewMDScaling(sm, int(k), Conf.Scripts.MDS)
		err = mds.Compute()
		if err != nil {
			return err
		}
		gof := fmt.Sprintf("%.5f", mds.Gof())
		stress := fmt.Sprintf("%.5f", mds.Stress())
		modelCoordinatesInsert(mds.Coordinates(), dat.ID, conf["k"], gof, stress, smID)
		return nil
	}
	return task
}

// NewOperatorRunTask initializes a new operator execution task.
func NewOperatorRunTask(operatorID string) *Task {
	m := modelOperatorGet(operatorID)
	if m == nil {
		log.Println("Operator was not found")
		return nil
	}
	dat := modelDatasetGetInfo(m.DatasetID)
	for _, f := range modelDatasetGetFiles(dat.ID) {
		dat.Files = append(dat.Files, dat.Path+"/"+f)
	}
	task := new(Task)
	task.Description = fmt.Sprintf("%s evaluation", m.Name)
	task.Dataset = dat
	task.fnc = func() error {
		eval, err := core.NewDatasetEvaluator(core.OnlineEval,
			map[string]string{
				"script":  m.Path,
				"testset": "",
			})
		if err != nil {
			log.Println(err)
		}
		scores := core.NewDatasetScores()
		for _, f := range dat.Files {
			s, err := eval.Evaluate(f)
			if err != nil {
				log.Println(err)
			} else {
				scores.Scores[path.Base(f)] = s
			}
		}
		cnt, _ := scores.Serialize()
		modelOperatorScoresInsert(operatorID, cnt)
		return nil
	}
	return task
}

// NewModelTrainTask generates a new ML for a given (dataset,operator) combination,
// according to the user-specified parameters.
func NewModelTrainTask(datasetID, operatorID string, sr float64,
	modelType string,
	coordinatesID, mlScript string,
	matrixID, k string) *Task {
	m := modelDatasetGetInfo(datasetID)
	task := new(Task)
	task.Description = fmt.Sprintf("Model training (%s for %s)", path.Base(mlScript), m.Name)
	task.Dataset = m
	task.fnc = func() error {
		datasets := core.DiscoverDatasets(m.Path)
		o := modelOperatorGet(operatorID)
		var evaluator core.DatasetEvaluator
		var err error
		if o.ScoresFile != "" {
			evaluator, err = core.NewDatasetEvaluator(core.FileBasedEval, map[string]string{"scores": o.ScoresFile})
		} else {
			evaluator, err = core.NewDatasetEvaluator(core.OnlineEval, map[string]string{"script": o.Path, "testset": ""})
		}
		if err != nil {
			log.Println(err)
			return err
		}
		t := core.NewModelerType(modelType)
		modeler := core.NewModeler(t, datasets, sr, evaluator)
		var conf map[string]string
		if t == core.ScriptBasedModelerType {
			c := modelCoordinatesGet(coordinatesID)
			conf = map[string]string{"script": mlScript, "coordinates": path.Base(c.Path)}
		} else if t == core.KNNModelerType {
			m := modelSimilarityMatrixGet(matrixID)
			conf = map[string]string{"k": k, "smatrix": path.Base(m.Path)}
		}
		modeler.Configure(conf)
		err = modeler.Run()
		if err != nil {
			log.Println(err)
			return err
		}
		// serialze appxValues
		var cnt [][]float64
		cnt = append(cnt, modeler.AppxValues())
		appxBuffer := serializeCSVFile(cnt)
		samplesBuffer, _ := json.Marshal(modeler.Samples())
		errors := make(map[string]string)
		for k, v := range modeler.ErrorMetrics() {
			errors[k] = fmt.Sprintf("%.5f", v)
		}
		modelDatasetModelInsert(coordinatesID, operatorID, datasetID, samplesBuffer, appxBuffer, conf, errors, sr)
		return nil
	}
	return task
}
<|endoftext|>


<commit_before>package host

import (
	"errors"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/consensus"
	"github.com/NebulousLabs/Sia/hash"
)

func (h *Host) consensusListen(updateChan chan consensus.ConsensusChange) {
	for consensusChange := range updateChan {
		fmt.Println(consensusChange)
		// For every contract that we recognize that gets destroyed, mark that contract as inactive

		// For every contract that we recognize that gets created, mark that contract as active.
	}
}

// Create a proof of storage for a contract, using the state height to
// determine the random seed. Create proof must be under a host and state lock.
func (h *Host) createStorageProof(entry ContractEntry, heightForProof consensus.BlockHeight) (sp consensus.StorageProof, err error) {
	// Get the file associated with the contract.
	contractObligation, exists := h.contracts[entry.ID]
	if !exists {
		err = errors.New("no record of that file")
		return
	}
	fullname := h.hostDir + contractObligation.filename

	// Open the file.
	file, err := os.Open(fullname)
	if err != nil {
		return
	}
	defer file.Close()

	// Build the proof using the hash library.
	numSegments := hash.CalculateSegments(entry.Contract.FileSize)
	windowIndex, err := entry.Contract.WindowIndex(heightForProof)
	if err != nil {
		return
	}
	segmentIndex, err := h.state.StorageProofSegmentIndex(entry.ID, windowIndex)
	if err != nil {
		return
	}
	base, hashSet, err := hash.BuildReaderProof(file, numSegments, segmentIndex)
	if err != nil {
		return
	}
	sp = consensus.StorageProof{entry.ID, windowIndex, base, hashSet}
	return
}

/*
// storageProofMaintenance tracks when storage proofs need to be submitted as
// transactions, then creates the proof and submits the transaction.
// storageProofMaintenance must be under a state and host lock.
//
// TODO: Make sure that when a contract terminates, the space is returned to
// the unsold space pool, file is deleted, etc.
//
// TODO: Have some method for pruning the backwards contracts map.
//
// TODO: Make sure that hosts don't need to submit a storage proof for the last
// window.
func (h *Host) storageProofMaintenance(initialStateHeight consensus.BlockHeight, rewoundBlocks []consensus.Block, appliedBlocks []consensus.Block) {
	// Resubmit any proofs that changed as a result of the rewinding.
	height := initialStateHeight
	var proofs []consensus.StorageProof
	for _ = range rewoundBlocks {
		needActionContracts := h.backwardContracts[height]
		for _, contractEntry := range needActionContracts {
			proof, err := h.createStorageProof(contractEntry, height)
			if err != nil {
				fmt.Println("High Priority Error: storage proof failed:", err)
				continue
			}
			proofs = append(proofs, proof)
		}
		height--
	}

	// Submit any proofs that are triggered as the result of new blocks being added.
	for _ = range appliedBlocks {
		needActionContracts := h.forwardContracts[height]
		for _, contractEntry := range needActionContracts {
			proof, err := h.createStorageProof(contractEntry, height)
			if err != nil {
				fmt.Println("High Priority Error: storage proof failed:", err)
				// TODO: Do something that will have the program try again, or
				// revitalize or whatever.
				continue
			}
			proofs = append(proofs, proof)

			// Add this contract proof to the backwards contracts list.
			h.backwardContracts[height-StorageProofReorgDepth+1] = append(h.backwardContracts[height-StorageProofReorgDepth+1], contractEntry)

			// Add this contract entry to ForwardContracts windowsize blocks
			// into the future if the contract has another window.
			nextProof := height + contractEntry.Contract.ChallengeWindow
			if nextProof < contractEntry.Contract.End {
				h.forwardContracts[nextProof] = append(h.forwardContracts[nextProof], contractEntry)
			} else {
				// Delete the file, etc. ==> Can't do this until we resolve the
				// collision problem.
			}
		}
		delete(h.forwardContracts, height)
		height++
	}

	// Create and submit a transaction for every storage proof.
	for _, proof := range proofs {
		// Create the transaction.
		minerFee := consensus.Currency(10) // TODO: ask wallet.
		id, err := h.wallet.RegisterTransaction(consensus.Transaction{})
		if err != nil {
			fmt.Println("High Priority Error: RegisterTransaction failed:", err)
			continue
		}
		err = h.wallet.FundTransaction(id, minerFee)
		if err != nil {
			fmt.Println("High Priority Error: FundTransaction failed:", err)
			continue
		}
		err = h.wallet.AddMinerFee(id, minerFee)
		if err != nil {
			fmt.Println("High Priority Error: AddMinerFee failed:", err)
			continue
		}
		err = h.wallet.AddStorageProof(id, proof)
		if err != nil {
			fmt.Println("High Priority Error: AddStorageProof failed:", err)
			continue
		}
		transaction, err := h.wallet.SignTransaction(id, true)
		if err != nil {
			fmt.Println("High Priority Error: SignTransaction failed:", err)
			continue
		}

		// Submit the transaction.
		h.transactionChan <- transaction
	}
}
*/
<commit_msg>more correctly add proofs and delete files<commit_after>package host

import (
	"errors"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/consensus"
	"github.com/NebulousLabs/Sia/hash"
)

// TODO: Integrate with the wallet in a way that allows funds to be reclaimed
// if not spent for a long time. Perhaps the wallet should have a feature or
// something about reclaiming lost coins, perhaps involving the reset counter
// going up once per block and transactions/funding gets to choose how long to
// wait for the reset. (It would make sense to be on a per-transaction basis,
// but that might be more complex to implement given the other design decisions
// we've made)

// TODO: Hold off on both storage proofs and deleting files for a few blocks
// after the first possible opportunity to reduce risk of loss due to
// blockchain reorganization.
func (h *Host) consensusListen(updateChan chan consensus.ConsensusChange) {
	for consensusChange := range updateChan {
		h.lock()

		var importantChanges []consensus.ContractDiff
		for _, blockDiff := range consensusChange.AppliedBlocks {
			for _, transactionChanges := range blockDiff.TransactionDiffs {
				importantChanges = append(importantChanges, transactionChanges.ContractDiffs...)
			}
			importantChanges = append(importantChanges, blockDiff.BlockChanges.ContractDiffs...)
		}

		var deletions []consensus.ContractID
		var proofs []consensus.StorageProof
		for _, contractDiff := range importantChanges {
			// Check that the contract belongs to us.
			_, exists := h.contracts[contractDiff.ContractID]
			if !exists {
				continue
			}

			// See if one of our contracts has terminated, and prepare to
			// delete the file if it has.
			if contractDiff.Terminated {
				deletions = append(deletions, contractDiff.ContractID)
			}
			if contractDiff.NewOpenContract.WindowSatisfied {
				continue
			}

			entry := ContractEntry{
				ID:       contractDiff.ContractID,
				Contract: contractDiff.Contract,
			}
			proof, err := h.createStorageProof(entry, h.state.Height())
			if err != nil {
				fmt.Println(err)
				continue
			}
			proofs = append(proofs, proof)
		}

		// Create and submit a transaction for every storage proof.
		for _, proof := range proofs {
			// Create the transaction.
			minerFee := consensus.Currency(10) // TODO: ask wallet.
			id, err := h.wallet.RegisterTransaction(consensus.Transaction{})
			if err != nil {
				fmt.Println("High Priority Error: RegisterTransaction failed:", err)
				continue
			}
			err = h.wallet.FundTransaction(id, minerFee)
			if err != nil {
				fmt.Println("High Priority Error: FundTransaction failed:", err)
				continue
			}
			err = h.wallet.AddMinerFee(id, minerFee)
			if err != nil {
				fmt.Println("High Priority Error: AddMinerFee failed:", err)
				continue
			}
			err = h.wallet.AddStorageProof(id, proof)
			if err != nil {
				fmt.Println("High Priority Error: AddStorageProof failed:", err)
				continue
			}
			transaction, err := h.wallet.SignTransaction(id, true)
			if err != nil {
				fmt.Println("High Priority Error: SignTransaction failed:", err)
				continue
			}

			// Submit the transaction.
			h.transactionChan <- transaction
		}

		// Delete all contracts which have expired.
		for _, contractID := range deletions {
			expiredContract := h.contracts[contractID]

			stat, err := os.Stat(h.hostDir + expiredContract.filename)
			if err != nil {
				fmt.Println(err)
			}
			err = os.Remove(h.hostDir + expiredContract.filename)
			h.spaceRemaining += stat.Size()
			if err != nil {
				fmt.Println(err)
			}
			delete(h.contracts, contractID)
		}

		h.unlock()
	}
}

// Create a proof of storage for a contract, using the state height to
// determine the random seed. Create proof must be under a host and state lock.
func (h *Host) createStorageProof(entry ContractEntry, heightForProof consensus.BlockHeight) (sp consensus.StorageProof, err error) {
	// Get the file associated with the contract.
	contractObligation, exists := h.contracts[entry.ID]
	if !exists {
		err = errors.New("no record of that file")
		return
	}
	fullname := h.hostDir + contractObligation.filename

	// Open the file.
	file, err := os.Open(fullname)
	if err != nil {
		return
	}
	defer file.Close()

	// Build the proof using the hash library.
	numSegments := hash.CalculateSegments(entry.Contract.FileSize)
	windowIndex, err := entry.Contract.WindowIndex(heightForProof)
	if err != nil {
		return
	}
	segmentIndex, err := h.state.StorageProofSegmentIndex(entry.ID, windowIndex)
	if err != nil {
		return
	}
	base, hashSet, err := hash.BuildReaderProof(file, numSegments, segmentIndex)
	if err != nil {
		return
	}
	sp = consensus.StorageProof{entry.ID, windowIndex, base, hashSet}
	return
}
<|endoftext|>


<commit_before>// Copyright 2016 IBM Corporation
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package sidecar

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/urfave/cli"

	"encoding/json"
	"net/http"
	"time"

	controllerclient "github.com/amalgam8/amalgam8/controller/client"
	"github.com/amalgam8/amalgam8/controller/rules"
	registryclient "github.com/amalgam8/amalgam8/registry/client"
	"github.com/amalgam8/amalgam8/sidecar/api"
	"github.com/amalgam8/amalgam8/sidecar/config"
	"github.com/amalgam8/amalgam8/sidecar/proxy"
	"github.com/amalgam8/amalgam8/sidecar/proxy/monitor"
	"github.com/amalgam8/amalgam8/sidecar/proxy/nginx"
	"github.com/amalgam8/amalgam8/sidecar/register"
	"github.com/amalgam8/amalgam8/sidecar/supervisor"
	"github.com/amalgam8/registry/client"
	"github.com/ant0ine/go-json-rest/rest"
)

// Main is the entrypoint for the sidecar when running as an executable
func Main() {
	logrus.ErrorKey = "error"
	logrus.SetLevel(logrus.DebugLevel) // Initial logging until we parse the user provided log level argument
	logrus.SetOutput(os.Stderr)

	app := cli.NewApp()

	app.Name = "sidecar"
	app.Usage = "Amalgam8 Sidecar"
	app.Version = "0.3.1"
	app.Flags = config.Flags
	app.Action = sidecarCommand

	err := app.Run(os.Args)
	if err != nil {
		logrus.WithError(err).Error("Failure launching sidecar")
	}
}

func sidecarCommand(context *cli.Context) error {
	conf, err := config.New(context)
	if err != nil {
		return err
	}
	return Run(*conf)
}

// Run the sidecar with the given configuration
func Run(conf config.Config) error {
	var err error

	if conf.Debug != "" {
		cliCommand(conf.Debug)
		return nil
	}

	if err = conf.Validate(); err != nil {
		logrus.WithError(err).Error("Validation of config failed")
		return err
	}

	logrusLevel, err := logrus.ParseLevel(conf.LogLevel)
	if err != nil {
		logrus.WithError(err).Errorf("Failure parsing requested log level (%v)", conf.LogLevel)
		logrusLevel = logrus.DebugLevel
	}
	logrus.SetLevel(logrusLevel)

	if conf.Log {
		//Replace the LOGSTASH_REPLACEME string in filebeat.yml with
		//the value provided by the user

		//TODO: Make this configurable
		filebeatConf := "/etc/filebeat/filebeat.yml"
		filebeat, err := ioutil.ReadFile(filebeatConf)
		if err != nil {
			logrus.WithError(err).Error("Could not read filebeat conf")
			return err
		}

		fileContents := strings.Replace(string(filebeat), "LOGSTASH_REPLACEME", conf.LogstashServer, -1)

		err = ioutil.WriteFile("/tmp/filebeat.yml", []byte(fileContents), 0)
		if err != nil {
			logrus.WithError(err).Error("Could not write filebeat conf")
			return err
		}

		// TODO: Log failure?
		go supervisor.DoLogManagement("/tmp/filebeat.yml")
	}

	if conf.Proxy {
		if err = startProxy(&conf); err != nil {
			logrus.WithError(err).Error("Could not start proxy")
		}
	}

	if conf.Register {
		logrus.Info("Registering")

		registryClient, err := registryclient.New(registryclient.Config{
			URL:       conf.Registry.URL,
			AuthToken: conf.Registry.Token,
		})
		if err != nil {
			logrus.WithError(err).Error("Could not create registry client")
			return err
		}

		address := fmt.Sprintf("%v:%v", conf.Endpoint.Host, conf.Endpoint.Port)
		serviceInstance := &registryclient.ServiceInstance{
			ServiceName: conf.Service.Name,
			Tags:        conf.Service.Tags,
			Endpoint: registryclient.ServiceEndpoint{
				Type:  conf.Endpoint.Type,
				Value: address,
			},
			TTL: 60,
		}

		agent, err := register.NewRegistrationAgent(register.RegistrationConfig{
			Client:          registryClient,
			ServiceInstance: serviceInstance,
		})
		if err != nil {
			logrus.WithError(err).Error("Could not create registry agent")
			return err
		}

		healthChecks, err := register.BuildHealthChecks(conf.HealthChecks)
		if err != nil {
			logrus.WithError(err).Error("Could not build health checks")
			return err
		}

		// Control the registration agent via the health checker if any health checks were provided. If no
		// health checks are provided, just start the registration agent.
		if len(healthChecks) > 0 {
			checker := register.NewHealthChecker(agent, healthChecks)
			checker.Start()
		} else {
			agent.Start()
		}
	}

	if conf.Supervise {
		supervisor.DoAppSupervision(conf.App)
	} else {
		select {}
	}

	return nil
}

func startProxy(conf *config.Config) error {
	var err error

	nginxClient := nginx.NewClient("http://localhost:5813")
	nginxManager := nginx.NewManager(
		nginx.Config{
			Service: nginx.NewService(fmt.Sprintf("%v:%v", conf.Service.Name, strings.Join(conf.Service.Tags, ","))),
			Client:  nginxClient,
		},
	)
	nginxProxy := proxy.NewNGINXProxy(nginxManager)

	controllerClient, err := controllerclient.New(controllerclient.Config{
		URL:       conf.Controller.URL,
		AuthToken: conf.Controller.Token,
	})
	if err != nil {
		logrus.WithError(err).Error("Could not create controller client")
		return err
	}

	registryClient, err := registryclient.New(registryclient.Config{
		URL:       conf.Registry.URL,
		AuthToken: conf.Registry.Token,
	})
	if err != nil {
		logrus.WithError(err).Error("Could not create registry client")
		return err
	}

	controllerMonitor := monitor.NewController(monitor.ControllerConfig{
		Client: controllerClient,
		Listeners: []monitor.ControllerListener{
			nginxProxy,
		},
		PollInterval: conf.Controller.Poll,
	})
	go func() {
		if err = controllerMonitor.Start(); err != nil {
			logrus.WithError(err).Error("Controller monitor failed")
		}
	}()

	registryMonitor := monitor.NewRegistry(monitor.RegistryConfig{
		PollInterval: conf.Registry.Poll,
		Listeners: []monitor.RegistryListener{
			nginxProxy,
		},
		RegistryClient: registryClient,
	})
	go func() {
		if err = registryMonitor.Start(); err != nil {
			logrus.WithError(err).Error("Registry monitor failed")
		}
	}()

	debugger := api.NewDebugAPI(nginxProxy)

	a := rest.NewApi()
	a.Use(
		&rest.TimerMiddleware{},
		&rest.RecorderMiddleware{},
		&rest.RecoverMiddleware{
			EnableResponseStackTrace: false,
		},
		&rest.ContentTypeCheckerMiddleware{},
		//&middleware.RequestIDMiddleware{},
		//&middleware.LoggingMiddleware{},
		//middleware.NewRequireHTTPS(middleware.CheckRequest{
		//	IsSecure: middleware.IsUsingSecureConnection,
		//	Disabled: !conf.RequireHTTPS,
		//}),
	)

	routes := debugger.Routes()
	router, err := rest.MakeRouter(
		routes...,
	)
	if err != nil {
		logrus.WithError(err).Error("Could not start API server")
		return err
	}
	a.SetApp(router)

	go func() {
		http.ListenAndServe(fmt.Sprintf(":%v", 6116), a.MakeHandler())
	}()

	return nil
}

// Instance TODO
type Instance struct {
	Tags string `json:"tags"`
	Type string `json:"type"`
	Host string `json:"host"`
	Port int    `json:"port"`
}

// NGINXState nginx cached lua state
type NGINXState struct {
	Routes    map[string][]rules.Rule `json:"routes"`
	Instances map[string][]Instance   `json:"instances"`
	Actions   map[string][]rules.Rule `json:"actions"`
}

func cliCommand(command string) {

	switch command {
	case "show-state":
		httpClient := http.Client{
			Timeout: time.Second * 10,
		}
		req, err := http.NewRequest("GET", "http://localhost:6116/state", nil)
		if err != nil {
			fmt.Println("Error occurred building the request:", err.Error())
			return
		}

		resp, err := httpClient.Do(req)
		if err != nil {
			fmt.Println("Error occurred sending the request:", err.Error())
			return
		}

		respBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("Error occurred reading response body:", err.Error())
			return
		}

		sidecarstate := struct {
			Instances []client.ServiceInstance `json:"instances"`
			Rules     []rules.Rule             `json:"rules"`
		}{}

		err = json.Unmarshal(respBytes, &sidecarstate)
		if err != nil {
			fmt.Println("Error occurred loading JSON response:", err.Error())
			return
		}

		req, err = http.NewRequest("GET", "http://localhost:5813/a8-admin", nil)
		if err != nil {
			fmt.Println("Error occurred building the request:", err.Error())
			return
		}

		resp, err = httpClient.Do(req)
		if err != nil {
			fmt.Println("Error occurred sending the request:", err.Error())
			return
		}

		respBytes, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("Error occurred reading response body:", err.Error())
			return
		}

		nginxstate := NGINXState{}

		err = json.Unmarshal(respBytes, &nginxstate)
		if err != nil {
			fmt.Println("Error occurred loading JSON response:", err.Error())
			return
		}

		sidecarBytes, _ := json.MarshalIndent(&sidecarstate, "", "   ")
		nginxBytes, _ := json.MarshalIndent(&nginxstate, "", "   ")
		fmt.Println("\n**************\nSidecar cached info:\n**************")
		fmt.Println(string(sidecarBytes))
		fmt.Println("\n**************\nNginx cached state:\n**************")
		fmt.Println(string(nginxBytes))

	default:
		fmt.Println("Unrecognized command: ", command)
	}
}
<commit_msg>bug fix and typo<commit_after>// Copyright 2016 IBM Corporation
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package sidecar

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/urfave/cli"

	"encoding/json"
	"net/http"
	"time"

	controllerclient "github.com/amalgam8/amalgam8/controller/client"
	"github.com/amalgam8/amalgam8/controller/rules"
	registryclient "github.com/amalgam8/amalgam8/registry/client"
	"github.com/amalgam8/amalgam8/sidecar/api"
	"github.com/amalgam8/amalgam8/sidecar/config"
	"github.com/amalgam8/amalgam8/sidecar/proxy"
	"github.com/amalgam8/amalgam8/sidecar/proxy/monitor"
	"github.com/amalgam8/amalgam8/sidecar/proxy/nginx"
	"github.com/amalgam8/amalgam8/sidecar/register"
	"github.com/amalgam8/amalgam8/sidecar/supervisor"
	"github.com/ant0ine/go-json-rest/rest"
)

// Main is the entrypoint for the sidecar when running as an executable
func Main() {
	logrus.ErrorKey = "error"
	logrus.SetLevel(logrus.DebugLevel) // Initial logging until we parse the user provided log level argument
	logrus.SetOutput(os.Stderr)

	app := cli.NewApp()

	app.Name = "sidecar"
	app.Usage = "Amalgam8 Sidecar"
	app.Version = "0.3.1"
	app.Flags = config.Flags
	app.Action = sidecarCommand

	err := app.Run(os.Args)
	if err != nil {
		logrus.WithError(err).Error("Failure launching sidecar")
	}
}

func sidecarCommand(context *cli.Context) error {
	conf, err := config.New(context)
	if err != nil {
		return err
	}
	return Run(*conf)
}

// Run the sidecar with the given configuration
func Run(conf config.Config) error {
	var err error

	if conf.Debug != "" {
		cliCommand(conf.Debug)
		return nil
	}

	if err = conf.Validate(); err != nil {
		logrus.WithError(err).Error("Validation of config failed")
		return err
	}

	logrusLevel, err := logrus.ParseLevel(conf.LogLevel)
	if err != nil {
		logrus.WithError(err).Errorf("Failure parsing requested log level (%v)", conf.LogLevel)
		logrusLevel = logrus.DebugLevel
	}
	logrus.SetLevel(logrusLevel)

	if conf.Log {
		//Replace the LOGSTASH_REPLACEME string in filebeat.yml with
		//the value provided by the user

		//TODO: Make this configurable
		filebeatConf := "/etc/filebeat/filebeat.yml"
		filebeat, err := ioutil.ReadFile(filebeatConf)
		if err != nil {
			logrus.WithError(err).Error("Could not read filebeat conf")
			return err
		}

		fileContents := strings.Replace(string(filebeat), "LOGSTASH_REPLACEME", conf.LogstashServer, -1)

		err = ioutil.WriteFile("/tmp/filebeat.yml", []byte(fileContents), 0)
		if err != nil {
			logrus.WithError(err).Error("Could not write filebeat conf")
			return err
		}

		// TODO: Log failure?
		go supervisor.DoLogManagement("/tmp/filebeat.yml")
	}

	if conf.Proxy {
		if err = startProxy(&conf); err != nil {
			logrus.WithError(err).Error("Could not start proxy")
		}
	}

	if conf.Register {
		logrus.Info("Registering")

		registryClient, err := registryclient.New(registryclient.Config{
			URL:       conf.Registry.URL,
			AuthToken: conf.Registry.Token,
		})
		if err != nil {
			logrus.WithError(err).Error("Could not create registry client")
			return err
		}

		address := fmt.Sprintf("%v:%v", conf.Endpoint.Host, conf.Endpoint.Port)
		serviceInstance := &registryclient.ServiceInstance{
			ServiceName: conf.Service.Name,
			Tags:        conf.Service.Tags,
			Endpoint: registryclient.ServiceEndpoint{
				Type:  conf.Endpoint.Type,
				Value: address,
			},
			TTL: 60,
		}

		agent, err := register.NewRegistrationAgent(register.RegistrationConfig{
			Client:          registryClient,
			ServiceInstance: serviceInstance,
		})
		if err != nil {
			logrus.WithError(err).Error("Could not create registry agent")
			return err
		}

		healthChecks, err := register.BuildHealthChecks(conf.HealthChecks)
		if err != nil {
			logrus.WithError(err).Error("Could not build health checks")
			return err
		}

		// Control the registration agent via the health checker if any health checks were provided. If no
		// health checks are provided, just start the registration agent.
		if len(healthChecks) > 0 {
			checker := register.NewHealthChecker(agent, healthChecks)
			checker.Start()
		} else {
			agent.Start()
		}
	}

	if conf.Supervise {
		supervisor.DoAppSupervision(conf.App)
	} else {
		select {}
	}

	return nil
}

func startProxy(conf *config.Config) error {
	var err error

	nginxClient := nginx.NewClient("http://localhost:5813")
	nginxManager := nginx.NewManager(
		nginx.Config{
			Service: nginx.NewService(fmt.Sprintf("%v:%v", conf.Service.Name, strings.Join(conf.Service.Tags, ","))),
			Client:  nginxClient,
		},
	)
	nginxProxy := proxy.NewNGINXProxy(nginxManager)

	controllerClient, err := controllerclient.New(controllerclient.Config{
		URL:       conf.Controller.URL,
		AuthToken: conf.Controller.Token,
	})
	if err != nil {
		logrus.WithError(err).Error("Could not create controller client")
		return err
	}

	registryClient, err := registryclient.New(registryclient.Config{
		URL:       conf.Registry.URL,
		AuthToken: conf.Registry.Token,
	})
	if err != nil {
		logrus.WithError(err).Error("Could not create registry client")
		return err
	}

	controllerMonitor := monitor.NewController(monitor.ControllerConfig{
		Client: controllerClient,
		Listeners: []monitor.ControllerListener{
			nginxProxy,
		},
		PollInterval: conf.Controller.Poll,
	})
	go func() {
		if err = controllerMonitor.Start(); err != nil {
			logrus.WithError(err).Error("Controller monitor failed")
		}
	}()

	registryMonitor := monitor.NewRegistry(monitor.RegistryConfig{
		PollInterval: conf.Registry.Poll,
		Listeners: []monitor.RegistryListener{
			nginxProxy,
		},
		RegistryClient: registryClient,
	})
	go func() {
		if err = registryMonitor.Start(); err != nil {
			logrus.WithError(err).Error("Registry monitor failed")
		}
	}()

	debugger := api.NewDebugAPI(nginxProxy)

	a := rest.NewApi()
	a.Use(
		&rest.TimerMiddleware{},
		&rest.RecorderMiddleware{},
		&rest.RecoverMiddleware{
			EnableResponseStackTrace: false,
		},
		&rest.ContentTypeCheckerMiddleware{},
		//&middleware.RequestIDMiddleware{},
		//&middleware.LoggingMiddleware{},
		//middleware.NewRequireHTTPS(middleware.CheckRequest{
		//	IsSecure: middleware.IsUsingSecureConnection,
		//	Disabled: !conf.RequireHTTPS,
		//}),
	)

	routes := debugger.Routes()
	router, err := rest.MakeRouter(
		routes...,
	)
	if err != nil {
		logrus.WithError(err).Error("Could not start API server")
		return err
	}
	a.SetApp(router)

	go func() {
		http.ListenAndServe(fmt.Sprintf(":%v", 6116), a.MakeHandler())
	}()

	return nil
}

// Instance TODO
type Instance struct {
	Tags string `json:"tags"`
	Type string `json:"type"`
	Host string `json:"host"`
	Port int    `json:"port"`
}

// NGINXState nginx cached lua state
type NGINXState struct {
	Routes    map[string][]rules.Rule `json:"routes"`
	Instances map[string][]Instance   `json:"instances"`
	Actions   map[string][]rules.Rule `json:"actions"`
}

func cliCommand(command string) {

	switch command {
	case "show-state":
		httpClient := http.Client{
			Timeout: time.Second * 10,
		}
		req, err := http.NewRequest("GET", "http://localhost:6116/state", nil)
		if err != nil {
			fmt.Println("Error occurred building the request:", err.Error())
			return
		}

		resp, err := httpClient.Do(req)
		if err != nil {
			fmt.Println("Error occurred sending the request:", err.Error())
			return
		}

		respBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("Error occurred reading response body:", err.Error())
			return
		}

		sidecarstate := struct {
			Instances []registryclient.ServiceInstance `json:"instances"`
			Rules     []rules.Rule                     `json:"rules"`
		}{}

		err = json.Unmarshal(respBytes, &sidecarstate)
		if err != nil {
			fmt.Println("Error occurred loading JSON response:", err.Error())
			return
		}

		req, err = http.NewRequest("GET", "http://localhost:5813/a8-admin", nil)
		if err != nil {
			fmt.Println("Error occurred building the request:", err.Error())
			return
		}

		resp, err = httpClient.Do(req)
		if err != nil {
			fmt.Println("Error occurred sending the request:", err.Error())
			return
		}

		respBytes, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("Error occurred reading response body:", err.Error())
			return
		}

		nginxstate := NGINXState{}

		err = json.Unmarshal(respBytes, &nginxstate)
		if err != nil {
			fmt.Println("Error occurred loading JSON response:", err.Error())
			return
		}

		sidecarBytes, _ := json.MarshalIndent(&sidecarstate, "", "   ")
		nginxBytes, _ := json.MarshalIndent(&nginxstate, "", "   ")
		fmt.Println("\n**************\nSidecar cached state:\n**************")
		fmt.Println(string(sidecarBytes))
		fmt.Println("\n**************\nNginx cached state:\n**************")
		fmt.Println(string(nginxBytes))

	default:
		fmt.Println("Unrecognized command: ", command)
	}
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"strings"
	"testing"

	"k8s.io/minikube/test/integration/util"
)

func TestDocker(t *testing.T) {
	minikubeRunner := util.MinikubeRunner{
		Args:       *args,
		BinaryPath: *binaryPath,
		T:          t}

	minikubeRunner.RunCommand("delete", false)

	startCmd := fmt.Sprintf("start %s %s", minikubeRunner.Args, "--docker-env=FOO=BAR --docker-env=BAZ=BAT --docker-opt=debug --docker-opt=icc=true")
	minikubeRunner.RunCommand(startCmd, true)
	minikubeRunner.EnsureRunning()

	filename := "/etc/systemd/system/docker.service"

	profileContents := minikubeRunner.RunCommand(fmt.Sprintf("ssh sudo cat %s", filename), true)
	fmt.Println(profileContents)
	for _, envVar := range []string{"FOO=BAR", "BAZ=BAT"} {
		if !strings.Contains(profileContents, envVar) {
			t.Fatalf("Env var %s missing from file: %s.", envVar, profileContents)
		}
	}
	for _, opt := range []string{"--debug", "--icc=true"} {
		if !strings.Contains(profileContents, opt) {
			t.Fatalf("Option %s missing from file: %s.", opt, profileContents)
		}
	}
}
<commit_msg>Dump docker systemd config using systemctl<commit_after>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"strings"
	"testing"

	"k8s.io/minikube/test/integration/util"
)

func TestDocker(t *testing.T) {
	minikubeRunner := util.MinikubeRunner{
		Args:       *args,
		BinaryPath: *binaryPath,
		T:          t}

	minikubeRunner.RunCommand("delete", false)

	startCmd := fmt.Sprintf("start %s %s", minikubeRunner.Args, "--docker-env=FOO=BAR --docker-env=BAZ=BAT --docker-opt=debug --docker-opt=icc=true")
	minikubeRunner.RunCommand(startCmd, true)
	minikubeRunner.EnsureRunning()

	dockerdEnvironment := minikubeRunner.RunCommand("ssh -- systemctl show docker --property=Environment", true)
	fmt.Println(dockerdEnvironment)
	for _, envVar := range []string{"FOO=BAR", "BAZ=BAT"} {
		if !strings.Contains(dockerdEnvironment, envVar) {
			t.Fatalf("Env var %s missing from Environment: %s.", envVar, dockerdEnvironment)
		}
	}

	dockerdExecStart := minikubeRunner.RunCommand("ssh -- systemctl show docker --property=ExecStart", true)
	fmt.Println(dockerdExecStart)
	for _, opt := range []string{"--debug", "--icc=true"} {
		if !strings.Contains(dockerdExecStart, opt) {
			t.Fatalf("Option %s missing from ExecStart: %s.", opt, dockerdExecStart)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/minikube/pkg/minikube/tunnel"
	commonutil "k8s.io/minikube/pkg/util"
	"k8s.io/minikube/test/integration/util"
)

func testTunnel(t *testing.T) {
	if runtime.GOOS != "windows" {
		// Otherwise minikube fails waiting for a password.
		if err := exec.Command("sudo", "-n", "route").Run(); err != nil {
			t.Skipf("password required to execute 'route', skipping testTunnel: %v", err)
		}
	}

	t.Log("starting tunnel test...")
	runner := NewMinikubeRunner(t)
	go func() {
		output := runner.RunCommand("tunnel --alsologtostderr -v 8 --logtostderr", true)
		if t.Failed() {
			fmt.Println(output)
		}
	}()

	err := tunnel.NewManager().CleanupNotRunningTunnels()

	if err != nil {
		t.Fatal(errors.Wrap(err, "cleaning up tunnels"))
	}

	kubectlRunner := util.NewKubectlRunner(t)

	t.Log("deploying nginx...")
	podPath, _ := filepath.Abs("testdata/testsvc.yaml")
	if _, err := kubectlRunner.RunCommand([]string{"apply", "-f", podPath}); err != nil {
		t.Fatalf("creating nginx ingress resource: %s", err)
	}

	client, err := commonutil.GetClient()

	if err != nil {
		t.Fatal(errors.Wrap(err, "getting kubernetes client"))
	}

	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx-svc"}))
	if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
		t.Fatal(errors.Wrap(err, "waiting for nginx pods"))
	}

	if err := commonutil.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil {
		t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up"))
	}

	t.Log("getting nginx ingress...")

	nginxIP := ""

	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
		cmd := []string{"get", "svc", "nginx-svc", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}"}
		stdout, err := kubectlRunner.RunCommand(cmd)
		switch {
		case err == nil:
			nginxIP = string(stdout)
			return len(stdout) != 0, nil
		case !commonutil.IsRetryableAPIError(err):
			t.Errorf("`%s` failed with non retriable error: %v", cmd, err)
			return false, err
		default:
			t.Errorf("`%s` failed: %v", cmd, err)
			return false, nil
		}
	})

	if err != nil {
		t.Errorf("error getting ingress IP for nginx: %s", err)
	}

	if len(nginxIP) == 0 {
		stdout, err := kubectlRunner.RunCommand([]string{"get", "svc", "nginx-svc", "-o", "jsonpath={.status}"})

		if err != nil {
			t.Errorf("error debugging nginx service: %s", err)
		}

		t.Fatalf("svc should have ingress after tunnel is created, but it was empty! Result of `kubectl describe svc nginx-svc`:\n %s", string(stdout))
	}

	responseBody, err := getResponseBody(nginxIP)
	if err != nil {
		t.Fatalf("error reading from nginx at address(%s): %s", nginxIP, err)
	}
	if !strings.Contains(responseBody, "Welcome to nginx!") {
		t.Fatalf("response body doesn't seem like an nginx response:\n%s", responseBody)
	}
}

// getResponseBody returns the contents of a URL
func getResponseBody(address string) (string, error) {
	httpClient := http.DefaultClient
	httpClient.Timeout = 5 * time.Second

	var resp *http.Response
	var err error

	request := func() error {
		resp, err = httpClient.Get(fmt.Sprintf("http://%s", address))
		if err != nil {
			retriable := &commonutil.RetriableError{Err: err}
			return retriable
		}
		return nil
	}

	if err = commonutil.RetryAfter(5, request, 1*time.Second); err != nil {
		return "", err
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil || len(body) == 0 {
		return "", errors.Wrapf(err, "error reading body, len bytes read: %d", len(body))
	}

	return string(body), nil
}
<commit_msg>fixing file path for windows<commit_after>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"

	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/minikube/pkg/minikube/tunnel"
	commonutil "k8s.io/minikube/pkg/util"
	"k8s.io/minikube/test/integration/util"
)

func testTunnel(t *testing.T) {
	if runtime.GOOS != "windows" {
		// Otherwise minikube fails waiting for a password.
		if err := exec.Command("sudo", "-n", "route").Run(); err != nil {
			t.Skipf("password required to execute 'route', skipping testTunnel: %v", err)
		}
	}

	t.Log("starting tunnel test...")
	runner := NewMinikubeRunner(t)
	go func() {
		output := runner.RunCommand("tunnel --alsologtostderr -v 8 --logtostderr", true)
		if t.Failed() {
			fmt.Println(output)
		}
	}()

	err := tunnel.NewManager().CleanupNotRunningTunnels()

	if err != nil {
		t.Fatal(errors.Wrap(err, "cleaning up tunnels"))
	}

	kubectlRunner := util.NewKubectlRunner(t)

	t.Log("deploying nginx...")
	curdir, err := filepath.Abs("")
	if err != nil {
		t.Errorf("Error getting the file path for current directory: %s", curdir)
	}
	podPath := path.Join(curdir, "testdata/testsvc.yaml")
	if _, err := kubectlRunner.RunCommand([]string{"apply", "-f", podPath}); err != nil {
		t.Fatalf("creating nginx ingress resource: %s", err)
	}

	client, err := commonutil.GetClient()

	if err != nil {
		t.Fatal(errors.Wrap(err, "getting kubernetes client"))
	}

	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx-svc"}))
	if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
		t.Fatal(errors.Wrap(err, "waiting for nginx pods"))
	}

	if err := commonutil.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil {
		t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up"))
	}

	t.Log("getting nginx ingress...")

	nginxIP := ""

	err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
		cmd := []string{"get", "svc", "nginx-svc", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}"}
		stdout, err := kubectlRunner.RunCommand(cmd)
		switch {
		case err == nil:
			nginxIP = string(stdout)
			return len(stdout) != 0, nil
		case !commonutil.IsRetryableAPIError(err):
			t.Errorf("`%s` failed with non retriable error: %v", cmd, err)
			return false, err
		default:
			t.Errorf("`%s` failed: %v", cmd, err)
			return false, nil
		}
	})

	if err != nil {
		t.Errorf("error getting ingress IP for nginx: %s", err)
	}

	if len(nginxIP) == 0 {
		stdout, err := kubectlRunner.RunCommand([]string{"get", "svc", "nginx-svc", "-o", "jsonpath={.status}"})

		if err != nil {
			t.Errorf("error debugging nginx service: %s", err)
		}

		t.Fatalf("svc should have ingress after tunnel is created, but it was empty! Result of `kubectl describe svc nginx-svc`:\n %s", string(stdout))
	}

	responseBody, err := getResponseBody(nginxIP)
	if err != nil {
		t.Fatalf("error reading from nginx at address(%s): %s", nginxIP, err)
	}
	if !strings.Contains(responseBody, "Welcome to nginx!") {
		t.Fatalf("response body doesn't seem like an nginx response:\n%s", responseBody)
	}
}

// getResponseBody returns the contents of a URL
func getResponseBody(address string) (string, error) {
	httpClient := http.DefaultClient
	httpClient.Timeout = 5 * time.Second

	var resp *http.Response
	var err error

	request := func() error {
		resp, err = httpClient.Get(fmt.Sprintf("http://%s", address))
		if err != nil {
			retriable := &commonutil.RetriableError{Err: err}
			return retriable
		}
		return nil
	}

	if err = commonutil.RetryAfter(5, request, 1*time.Second); err != nil {
		return "", err
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil || len(body) == 0 {
		return "", errors.Wrapf(err, "error reading body, len bytes read: %d", len(body))
	}

	return string(body), nil
}
<|endoftext|>


<commit_before>package xpi

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"log"
	"math/big"
	"time"

	"github.com/pkg/errors"
)

// every minute, add an rsa key to the cache. This will block if
// the cache channel is already full, which is what we want anyway
func (s *PKCS7Signer) populateRsaCache(size int) {
	for {
		key, err := rsa.GenerateKey(rand.Reader, size)
		if err != nil {
			log.Fatalf("xpi.populateRsaCache: %v", err)
		}
		s.rsaCache <- key
		time.Sleep(time.Minute)
	}
}

// retrieve a key from the cache or generate one if it takes too long
// or if the size is wrong
func (s *PKCS7Signer) getRsaKey(size int) (*rsa.PrivateKey, error) {
	select {
	case key := <-s.rsaCache:
		if key.N.BitLen() != size {
			// it's theoritically impossible for this to happen
			// because the end entity has the same key size has
			// the signer, but we're paranoid so handling it
			log.Printf("WARNING: xpi rsa cache returned a key of size %d when %d was requested", key.N.BitLen(), size)
			return rsa.GenerateKey(rand.Reader, size)
		}
		return key, nil
	case <-time.After(100 * time.Millisecond):
		// generate a key if none available
		return rsa.GenerateKey(rand.Reader, size)
	}
}

// MakeEndEntity generates a private key and certificate ready to sign a given XPI.
// The subject CN of the certificate is taken from the `cn` string passed as argument.
// The type of key is identical to the key of the signer that issues the certificate,
// if the signer uses an RSA 2048 key, so will the end-entity. The signature algorithm
// and expiration date are also copied over from the issuer.
//
// The signed certificate and private key are returned.
func (s *PKCS7Signer) MakeEndEntity(cn string) (eeCert *x509.Certificate, eeKey crypto.PrivateKey, err error) {
	var derCert []byte
	template := x509.Certificate{
		// The maximum length of a serial number per rfc 5280 is 20 bytes / 160 bits
		// https://tools.ietf.org/html/rfc5280#section-4.1.2.2
		// Setting it to nanoseconds guarantees we'll never have two conflicting serials
		SerialNumber: big.NewInt(time.Now().UnixNano()),
		Subject: pkix.Name{
			CommonName:         cn,
			Organization:       []string{"Addons"},
			OrganizationalUnit: []string{s.OU},
			Country:            []string{"US"},
			Province:           []string{"CA"},
			Locality:           []string{"Mountain View"},
		},
		NotBefore:          time.Now(),
		NotAfter:           time.Now().Add(8760 * time.Hour), // one year
		SignatureAlgorithm: s.issuerCert.SignatureAlgorithm,
	}
	switch s.issuerKey.(type) {
	case *rsa.PrivateKey:
		size := s.issuerKey.(*rsa.PrivateKey).N.BitLen()
		eeKey, err = s.getRsaKey(size)
		if err != nil {
			err = errors.Wrapf(err, "xpi.MakeEndEntity: failed to generate rsa private key of size %d", size)
			return
		}
		derCert, err = x509.CreateCertificate(rand.Reader, &template, s.issuerCert, eeKey.(*rsa.PrivateKey).Public(), s.issuerKey.(*rsa.PrivateKey))
	case *ecdsa.PrivateKey:
		curve := s.issuerKey.(*ecdsa.PrivateKey).Curve
		eeKey, err = ecdsa.GenerateKey(curve, rand.Reader)
		if err != nil {
			err = errors.Wrapf(err, "xpi.MakeEndEntity: failed to generate ecdsa private key on curve %s", curve.Params().Name)
			return
		}
		derCert, err = x509.CreateCertificate(rand.Reader, &template, s.issuerCert, eeKey.(*ecdsa.PrivateKey).Public(), s.issuerKey.(*ecdsa.PrivateKey))
	}
	if err != nil {
		err = errors.Wrapf(err, "xpi.MakeEndEntity: failed to create certificate")
		return
	}
	if len(derCert) == 0 {
		err = errors.Errorf("xpi.MakeEndEntity: certificate creation failed for an unknown reason")
		return
	}
	eeCert, err = x509.ParseCertificate(derCert)
	if err != nil {
		err = errors.Wrapf(err, "xpi.MakeEndEntity: certificate parsing failed")
	}
	return
}
<commit_msg>xpi: add the add-on ID to the dns names of the EE cert<commit_after>package xpi

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"log"
	"math/big"
	"time"

	"github.com/pkg/errors"
)

// every minute, add an rsa key to the cache. This will block if
// the cache channel is already full, which is what we want anyway
func (s *PKCS7Signer) populateRsaCache(size int) {
	for {
		key, err := rsa.GenerateKey(rand.Reader, size)
		if err != nil {
			log.Fatalf("xpi.populateRsaCache: %v", err)
		}
		s.rsaCache <- key
		time.Sleep(time.Minute)
	}
}

// retrieve a key from the cache or generate one if it takes too long
// or if the size is wrong
func (s *PKCS7Signer) getRsaKey(size int) (*rsa.PrivateKey, error) {
	select {
	case key := <-s.rsaCache:
		if key.N.BitLen() != size {
			// it's theoritically impossible for this to happen
			// because the end entity has the same key size has
			// the signer, but we're paranoid so handling it
			log.Printf("WARNING: xpi rsa cache returned a key of size %d when %d was requested", key.N.BitLen(), size)
			return rsa.GenerateKey(rand.Reader, size)
		}
		return key, nil
	case <-time.After(100 * time.Millisecond):
		// generate a key if none available
		return rsa.GenerateKey(rand.Reader, size)
	}
}

// MakeEndEntity generates a private key and certificate ready to sign a given XPI.
// The subject CN of the certificate is taken from the `cn` string passed as argument.
// The type of key is identical to the key of the signer that issues the certificate,
// if the signer uses an RSA 2048 key, so will the end-entity. The signature algorithm
// and expiration date are also copied over from the issuer.
//
// The signed certificate and private key are returned.
func (s *PKCS7Signer) MakeEndEntity(cn string) (eeCert *x509.Certificate, eeKey crypto.PrivateKey, err error) {
	var derCert []byte
	template := x509.Certificate{
		// The maximum length of a serial number per rfc 5280 is 20 bytes / 160 bits
		// https://tools.ietf.org/html/rfc5280#section-4.1.2.2
		// Setting it to nanoseconds guarantees we'll never have two conflicting serials
		SerialNumber: big.NewInt(time.Now().UnixNano()),

		// having a dnsname or an email address field is required to comply with webpki
		// requirement due to the intermediates having constraints
		// see https://github.com/golang/go/issues/24151
		DNSNames: []string{cn},

		Subject: pkix.Name{
			CommonName:         cn,
			Organization:       []string{"Addons"},
			OrganizationalUnit: []string{s.OU},
			Country:            []string{"US"},
			Province:           []string{"CA"},
			Locality:           []string{"Mountain View"},
		},
		NotBefore:          time.Now(),
		NotAfter:           time.Now().Add(8760 * time.Hour), // one year
		SignatureAlgorithm: s.issuerCert.SignatureAlgorithm,
	}
	switch s.issuerKey.(type) {
	case *rsa.PrivateKey:
		size := s.issuerKey.(*rsa.PrivateKey).N.BitLen()
		eeKey, err = s.getRsaKey(size)
		if err != nil {
			err = errors.Wrapf(err, "xpi.MakeEndEntity: failed to generate rsa private key of size %d", size)
			return
		}
		derCert, err = x509.CreateCertificate(rand.Reader, &template, s.issuerCert, eeKey.(*rsa.PrivateKey).Public(), s.issuerKey.(*rsa.PrivateKey))
	case *ecdsa.PrivateKey:
		curve := s.issuerKey.(*ecdsa.PrivateKey).Curve
		eeKey, err = ecdsa.GenerateKey(curve, rand.Reader)
		if err != nil {
			err = errors.Wrapf(err, "xpi.MakeEndEntity: failed to generate ecdsa private key on curve %s", curve.Params().Name)
			return
		}
		derCert, err = x509.CreateCertificate(rand.Reader, &template, s.issuerCert, eeKey.(*ecdsa.PrivateKey).Public(), s.issuerKey.(*ecdsa.PrivateKey))
	}
	if err != nil {
		err = errors.Wrapf(err, "xpi.MakeEndEntity: failed to create certificate")
		return
	}
	if len(derCert) == 0 {
		err = errors.Errorf("xpi.MakeEndEntity: certificate creation failed for an unknown reason")
		return
	}
	eeCert, err = x509.ParseCertificate(derCert)
	if err != nil {
		err = errors.Wrapf(err, "xpi.MakeEndEntity: certificate parsing failed")
	}
	return
}
<|endoftext|>


<commit_before>// Package mysql implements reform.Dialect for MySQL.
package mysql

import (
	"fmt"
	"github.com/xaionaro/reform"
)

type mysql struct{}

func (mysql) String() string {
	return "mysql"
}

func (mysql) Placeholder(index int) string {
	return "?"
}

func (mysql) Placeholders(start, count int) []string {
	res := make([]string, count)
	for i := 0; i < count; i++ {
		res[i] = "?"
	}
	return res
}

func (mysql) QuoteIdentifier(identifier string) string {
	return "`" + identifier + "`"
}

func (mysql) LastInsertIdMethod() reform.LastInsertIdMethod {
	return reform.LastInsertId
}

func (mysql) SelectLimitMethod() reform.SelectLimitMethod {
	return reform.Limit
}

func (mysql) DefaultValuesMethod() reform.DefaultValuesMethod {
	return reform.EmptyLists
}

func (mysql) ColumnTypeForField(field reform.FieldInfo) string {
	if len(field.Type) == 0 {
		return "text"
	}
	if field.Type[:1] == "*" {
		field.Type = field.Type[1:]
	}
	switch field.Type {
	case "time.Time", "extime.Time":
		return "timestamp"
	case "int":
		return "integer"
	case "int64":
		return "bigint"
	case "string":
		if field.SQLSize > 0 && field.SQLSize < 256 {
			return fmt.Sprintf("varchar(%d)", field.SQLSize)
		}
		return "text"
	default:
		return "text"
	}
}

func (mysql) ColumnDefinitionForField(field reform.FieldInfo) string {
	canBeNull := false
	fieldType := field.Type
	if fieldType[0:1] == "*" {
		canBeNull = true
		fieldType = fieldType[1:]
	}

	columnType := Dialect.ColumnTypeForField(field)

	definition := field.Column + " " + columnType // TODO: escape everything

	if field.IsPK {
		definition += " PRIMARY KEY"
		if fieldType == "int" {
			definition += " AUTO_INCREMENT"
		}
	}

	if field.IsUnique {
		definition += " UNIQUE"
	}

	if !canBeNull {
		definition += " NOT NULL"
	}
	if field.HasIndex {
		definition += ", INDEX (`" + field.Column + "`)"
	}

	return definition
}

func (mysql) ColumnDefinitionPostQueryForField(structInfo reform.StructInfo, field reform.FieldInfo) string {
	return ""
}

// Dialect implements reform.Dialect for MySQL.
var Dialect mysql

// check interface
var _ reform.Dialect = Dialect
<commit_msg>Use datetime instead of timestamp in MySQL<commit_after>// Package mysql implements reform.Dialect for MySQL.
package mysql

import (
	"fmt"
	"github.com/xaionaro/reform"
)

type mysql struct{}

func (mysql) String() string {
	return "mysql"
}

func (mysql) Placeholder(index int) string {
	return "?"
}

func (mysql) Placeholders(start, count int) []string {
	res := make([]string, count)
	for i := 0; i < count; i++ {
		res[i] = "?"
	}
	return res
}

func (mysql) QuoteIdentifier(identifier string) string {
	return "`" + identifier + "`"
}

func (mysql) LastInsertIdMethod() reform.LastInsertIdMethod {
	return reform.LastInsertId
}

func (mysql) SelectLimitMethod() reform.SelectLimitMethod {
	return reform.Limit
}

func (mysql) DefaultValuesMethod() reform.DefaultValuesMethod {
	return reform.EmptyLists
}

func (mysql) ColumnTypeForField(field reform.FieldInfo) string {
	if len(field.Type) == 0 {
		return "text"
	}
	if field.Type[:1] == "*" {
		field.Type = field.Type[1:]
	}
	switch field.Type {
	case "time.Time", "extime.Time":
		return "datetime"
	case "int":
		return "integer"
	case "int64":
		return "bigint"
	case "string":
		if field.SQLSize > 0 && field.SQLSize < 256 {
			return fmt.Sprintf("varchar(%d)", field.SQLSize)
		}
		return "text"
	default:
		return "text"
	}
}

func (mysql) ColumnDefinitionForField(field reform.FieldInfo) string {
	canBeNull := false
	fieldType := field.Type
	if fieldType[0:1] == "*" {
		canBeNull = true
		fieldType = fieldType[1:]
	}

	columnType := Dialect.ColumnTypeForField(field)

	definition := field.Column + " " + columnType // TODO: escape everything

	if field.IsPK {
		definition += " PRIMARY KEY"
		if fieldType == "int" {
			definition += " AUTO_INCREMENT"
		}
	}

	if field.IsUnique {
		definition += " UNIQUE"
	}

	if !canBeNull {
		definition += " NOT NULL"
	}
	if field.HasIndex {
		definition += ", INDEX (`" + field.Column + "`)"
	}

	return definition
}

func (mysql) ColumnDefinitionPostQueryForField(structInfo reform.StructInfo, field reform.FieldInfo) string {
	return ""
}

// Dialect implements reform.Dialect for MySQL.
var Dialect mysql

// check interface
var _ reform.Dialect = Dialect
<|endoftext|>