<commit_before>package main

import (
	"crypto/sha256"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"runtime"
	"time"
	//"strconv"
	//"strings"
	"sync"

	elastigo "github.com/mattbaird/elastigo/lib"
	"github.com/streadway/amqp"
)

var signatureAlgorithm = [...]string{
	"UnknownSignatureAlgorithm",
	"MD2WithRSA",
	"MD5WithRSA",
	"SHA1WithRSA",
	"SHA256WithRSA",
	"SHA384WithRSA",
	"SHA512WithRSA",
	"DSAWithSHA1",
	"DSAWithSHA256",
	"ECDSAWithSHA1",
	"ECDSAWithSHA256",
	"ECDSAWithSHA384",
	"ECDSAWithSHA512",
}

var extKeyUsage = [...]string{
	"ExtKeyUsageAny",
	"ExtKeyUsageServerAuth",
	"ExtKeyUsageClientAuth",
	"ExtKeyUsageCodeSigning",
	"ExtKeyUsageEmailProtection",
	"ExtKeyUsageIPSECEndSystem",
	"ExtKeyUsageIPSECTunnel",
	"ExtKeyUsageIPSECUser",
	"ExtKeyUsageTimeStamping",
	"ExtKeyUsageOCSPSigning",
	"ExtKeyUsageMicrosoftServerGatedCrypto",
	"ExtKeyUsageNetscapeServerGatedCrypto",
}

var publicKeyAlgorithm = [...]string{
	"UnknownPublicKeyAlgorithm",
	"RSA",
	"DAS",
	"ECDSA",
}

type StoredCertificate struct {
	Version                float64                  `json:"version"`
	SignatureAlgorithm     string                   `json:"signatureAlgorithm"`
	Issuer                 certIssuer               `json:"issuer"`
	Validity               certValidity             `json:"validity"`
	Subject                certSubject              `json:"subject"`
	SubjectPublicKeyInfo   certSubjectPublicKeyInfo `json:"subjectPublicKeyInfo"`
	X509v3Extensions       certExtensions           `json:"x509v3Extensions"`
	X509v3BasicConstraints string                   `json:"x509v3BasicConstraints"`
	CA                     bool                     `json:"ca"`
	Analysis               interface{}              `json:"analysis"` //for future use...
	ParentSignature        string                   `json:"parentSignature"`
	IsChainValid		   string                   `json:"isChainValid"`//string displays error type along the validation chain...
	CollectionTimestamp    string                   `json:"collectionTimestamp"`
}

type certIssuer struct {
	Country      []string `json:"c"`
	Organisation []string `json:"o"`
	OrgUnit      []string `json:"ou"`
	CommonName   string   `json:"cn"`
}

type certValidity struct {
	NotBefore string `json:"notBefore"`
	NotAfter  string `json:"notAfter"`
}

type certSubject struct {
	Country      []string `json:"c"`
	Organisation []string `json:"o"`
	OrgUnit      []string `json:"ou"`
	CommonName   string   `json:"cn"`
}

type certSubjectPublicKeyInfo struct {
	PublicKeyAlgorithm string `json:"publicKeyAlgorithm"`
}

//Currently exporting extensions that are already decoded into the x509 Certificate structure

type certExtensions struct {
	AuthorityKeyId   []byte   `json:"authorityKeyId"`
	SubjectKeyId     []byte   `json:"subjectKeyId"`
	KeyUsage         []string `json:"keyUsage"`
	ExtendedKeyUsage []string `json:"extendedKeyUsage"`
	// Maybe need to create a struct to support Email and IPAddresses as long as DNSNames...
	SubjectAlternativeName []string `json:"subjectAlternativeName"`
	CRLDistributionPoints  []string `json:"crlDistributionPoints"`
}

type CertX509v3BasicConstraints struct {
	CA       bool        `json:"ca"`
	Analysis interface{} `json:"analysis"`
}

type CertChain struct {
	Domain string   `json:"domain"`
	Certs  []string `json:"certs"`
}

type ids struct {
	_type  string   `json:"type"`
	values []string `json:"values"`
}

type JsonRawCert struct{
	RawCert string `json:"rawCert"`
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func SHA256Hash(data []byte) string {
	h := sha256.New()
	h.Write(data)
	return fmt.Sprintf("%X", h.Sum(nil))
	//return hex.EncodeToString(h.Sum(nil))
}

func panicIf(err error) bool {
	if err != nil {
		log.Println(fmt.Sprintf("%s", err))
		return true
	}

	return false
}

func worker(msgs <-chan amqp.Delivery, es *elastigo.Conn) {

	forever := make(chan bool)
	defer wg.Done()

	for d := range msgs {

		chain := CertChain{}

		err := json.Unmarshal(d.Body, &chain)
		panicIf(err)

		log.Println(len(chain.Certs))

		analyseAndPushCertificates(&chain, es)

		// jsonCert, err := json.MarshalIndent(certtoStored(certif), "", "    ")
		// panicIf(err)
		// log.Println(string(jsonCert))
		// _, err = es.Index("certificates", "certificateInfo", SHA256Hash(certif.Raw), nil, jsonCert)
		// _, err = es.Index("certificates", "certificateRaw", SHA256Hash(certif.Raw), nil, base64.StdEncoding.EncodeToString(certif.Raw))
		// panicIf(err)
		d.Ack(false)
	}

	<-forever
}

func analyseAndPushCertificates(chain *CertChain, es *elastigo.Conn) {

	var certs []*x509.Certificate

	for i, data := range chain.Certs {

		certRaw, err := base64.StdEncoding.DecodeString(data)
		panicIf(err)

		var certif *x509.Certificate
		certif, err = x509.ParseCertificate(certRaw)
		panicIf(err)

		if i == 0{ // do this only for the leaf certificate as in most cases intermediate and CA certificate will already be in the DB
			searchJson := `{
		    "query" : {
		        "term" : { "_id" : "` + SHA256Hash(certif.Raw) + `" }
		    }
		}`
			res, e := es.Search("certificates", "certificateInfo", nil, searchJson)
			if !panicIf(e){
				if res.Hits.Total>0{//probably gonna be 0|1
					//certificate is already in the database. Abort??? return
				}
			}
		}

		certs = append(certs, certif)
	}

	inter:= x509.NewCertPool()

	for _,c := range certs[1:len(certs)]{
		if c.Issuer.CommonName != ""{

			log.Println("added to Intermediates"+c.Subject.CommonName)
			inter.AddCert(c)
		}
	}

	opts := x509.VerifyOptions{
		DNSName: chain.Domain,
		Intermediates: inter,
	}

	var chains [][]*x509.Certificate

	chains, err := certs[0].Verify(opts)

	if len(chains)>0 && !panicIf(err){
		CreateStoredChain(chains[0], "", es)
	}else{
		CreateStoredChain(certs,err.Error(), es)
	}
}

func CreateStoredChain(certs []*x509.Certificate, validation_err string, es *elastigo.Conn){
	for i, cert := range certs{

		searchJson := `{
		    "query" : {
		        "term" : { "_id" : "` + SHA256Hash(cert.Raw) + `" }
		    }
		}`

		res, e := es.Search("certificates", "certificateInfo", nil, searchJson)
		if !panicIf(e){
			if res.Hits.Total>0{//probably gonna be 0|1
				//certificate is already in the database. Abort??? return
			}
		}

		stored := certtoStored(cert)
		jsonCert, err := json.MarshalIndent(stored, "", "    ")
		panicIf(err)

		stored.IsChainValid = validation_err
		if cert.Issuer.CommonName != "" && len(certs)>i+1{
			stored.ParentSignature = SHA256Hash(certs[i+1].Raw)
		}
		_, err = es.Index("certificates", "certificateInfo", SHA256Hash(cert.Raw), nil, jsonCert)
		panicIf(err)

		raw := JsonRawCert{base64.StdEncoding.EncodeToString(cert.Raw)}
		jsonCert, err = json.MarshalIndent(raw, "", "    ")
		panicIf(err)
		_, err = es.Index("certificates", "certificateRaw", SHA256Hash(cert.Raw), nil, jsonCert)
		panicIf(err)
	}
}

//Alternate method of Verifying

// func analyseAndPushCertificates(chain *CertChain){

// 	for i, data := range chain.Certs {

// 		certRaw, err := base64.StdEncoding.DecodeString(data)
// 		panicIf(err)

// 		var certif *x509.Certificate
// 		certif, err = x509.ParseCertificate(certRaw)
// 		panicIf(err)

// 		if i+1<len(chain.Certs){
// 			certRaw, err = base64.StdEncoding.DecodeString(chain.Certs[i+1])
// 			panicIf(err)

// 			var certt *x509.Certificate
// 			certt, err = x509.ParseCertificate(certRaw)
// 			panicIf(err)

// 			log.Println(certif.Subject.CommonName + " - "+ certt.Subject.CommonName )

// 			err = certif.CheckSignatureFrom(certt)
// 			panicIf(err)
// 		}

// 		certs = append(certs, certif)
// 	}

// }

func getExtKeyUsageAsStringArray(cert *x509.Certificate) []string {

	usage := make([]string, len(cert.ExtKeyUsage))

	for _, eku := range cert.ExtKeyUsage {

		usage = append(usage, extKeyUsage[eku])
	}

	return usage
}

func getKeyUsageAsStringArray(cert *x509.Certificate) []string {

	var usage []string
	keyUsage := cert.KeyUsage

	//calculate included keyUsage from bitmap
	//String values taken from OpenSSL

	if keyUsage&x509.KeyUsageDigitalSignature != 0 {
		usage = append(usage, "Digital Signature")
	}
	if keyUsage&x509.KeyUsageContentCommitment != 0 {
		usage = append(usage, "Non Repudiation")
	}

	if keyUsage&x509.KeyUsageKeyEncipherment != 0 {
		usage = append(usage, "Key Encipherment")
	}

	if keyUsage&x509.KeyUsageDataEncipherment != 0 {
		usage = append(usage, "Data Encipherment")
	}

	if keyUsage&x509.KeyUsageKeyAgreement != 0 {
		usage = append(usage, "Key Agreement")
	}

	if keyUsage&x509.KeyUsageCertSign != 0 {
		usage = append(usage, "Certificate Sign")
	}

	if keyUsage&x509.KeyUsageCRLSign != 0 {
		usage = append(usage, "CRL Sign")
	}

	if keyUsage&x509.KeyUsageEncipherOnly != 0 {
		usage = append(usage, "Encipher Only")
	}

	if keyUsage&x509.KeyUsageDecipherOnly != 0 {
		usage = append(usage, "Decipher Only")
	}

	return usage
}

func getCertExtensions(cert *x509.Certificate) certExtensions {

	extensions := certExtensions{}

	extensions.AuthorityKeyId = []byte(base64.StdEncoding.EncodeToString(cert.AuthorityKeyId))
	extensions.SubjectKeyId = []byte(base64.StdEncoding.EncodeToString(cert.SubjectKeyId))

	extensions.KeyUsage = getKeyUsageAsStringArray(cert)

	extensions.ExtendedKeyUsage = getExtKeyUsageAsStringArray(cert)

	extensions.SubjectAlternativeName = cert.DNSNames

	extensions.CRLDistributionPoints = cert.CRLDistributionPoints

	return extensions

}

func certtoStored(cert *x509.Certificate) StoredCertificate {

	var stored = StoredCertificate{}

	//Print raw extension info

	// for i, extension := range cert.Extensions{

	// 	var numbers string
	// 	for num ,num2 := range  extension.Id{

	// 		numbers = numbers + " "+ "[" + strconv.Itoa(num)+ " " + strconv.Itoa(num2) + "]"

	// 	}

	// 	log.Println("//",strconv.Itoa(i),": {", numbers,"}",string(extension.Value) )

	// }

	stored.Version = float64(cert.Version)

	stored.SignatureAlgorithm = signatureAlgorithm[cert.SignatureAlgorithm]

	stored.SubjectPublicKeyInfo.PublicKeyAlgorithm = publicKeyAlgorithm[cert.PublicKeyAlgorithm]

	stored.Issuer.Country = cert.Issuer.Country
	stored.Issuer.Organisation = cert.Issuer.Organization
	stored.Issuer.OrgUnit = cert.Issuer.OrganizationalUnit
	stored.Issuer.CommonName = cert.Issuer.CommonName

	stored.Subject.Country = cert.Subject.Country
	stored.Subject.Organisation = cert.Subject.Organization
	stored.Subject.OrgUnit = cert.Subject.OrganizationalUnit
	stored.Subject.CommonName = cert.Subject.CommonName

	stored.Validity.NotBefore = cert.NotBefore.UTC().String()
	stored.Validity.NotAfter = cert.NotAfter.UTC().String()

	stored.X509v3Extensions = getCertExtensions(cert)

	if cert.BasicConstraintsValid {

		stored.X509v3BasicConstraints = "Critical"
		stored.CA = cert.IsCA
	} else {
		stored.X509v3BasicConstraints = ""
		stored.CA = false
	}

	stored.CollectionTimestamp = time.Now().UTC().String()
	return stored

}

var wg sync.WaitGroup

func main() {

	conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
	failOnError(err, "Failed to connect to RabbitMQ")
	defer conn.Close()

	es := elastigo.NewConn()
	es.Domain = "83.212.99.104:9200"

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	defer ch.Close()

	q, err := ch.QueueDeclare(
		"scan_results_queue", // name
		true,                 // durable
		false,                // delete when unused
		false,                // exclusive
		false,                // no-wait
		nil,                  // arguments
	)
	failOnError(err, "Failed to declare a queue")

	err = ch.Qos(
		3,     // prefetch count
		0,     // prefetch size
		false, // global
	)

	failOnError(err, "Failed to set QoS")

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		false,  // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)

	failOnError(err, "Failed to register a consumer")

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores)

	for i := 0; i < cores; i++ {
		wg.Add(1)
		go worker(msgs, es)
	}

	wg.Wait()
}
<commit_msg>further modifications to the way certificate chains are validated and stored<commit_after>package main

import (
	"crypto/sha256"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"runtime"
	"strconv"
	"time"
	//"strings"
	"sync"

	elastigo "github.com/mattbaird/elastigo/lib"
	"github.com/streadway/amqp"
)

var signatureAlgorithm = [...]string{
	"UnknownSignatureAlgorithm",
	"MD2WithRSA",
	"MD5WithRSA",
	"SHA1WithRSA",
	"SHA256WithRSA",
	"SHA384WithRSA",
	"SHA512WithRSA",
	"DSAWithSHA1",
	"DSAWithSHA256",
	"ECDSAWithSHA1",
	"ECDSAWithSHA256",
	"ECDSAWithSHA384",
	"ECDSAWithSHA512",
}

var extKeyUsage = [...]string{
	"ExtKeyUsageAny",
	"ExtKeyUsageServerAuth",
	"ExtKeyUsageClientAuth",
	"ExtKeyUsageCodeSigning",
	"ExtKeyUsageEmailProtection",
	"ExtKeyUsageIPSECEndSystem",
	"ExtKeyUsageIPSECTunnel",
	"ExtKeyUsageIPSECUser",
	"ExtKeyUsageTimeStamping",
	"ExtKeyUsageOCSPSigning",
	"ExtKeyUsageMicrosoftServerGatedCrypto",
	"ExtKeyUsageNetscapeServerGatedCrypto",
}

var publicKeyAlgorithm = [...]string{
	"UnknownPublicKeyAlgorithm",
	"RSA",
	"DAS",
	"ECDSA",
}

type StoredCertificate struct {
	Version                float64                  `json:"version"`
	SignatureAlgorithm     string                   `json:"signatureAlgorithm"`
	Issuer                 certIssuer               `json:"issuer"`
	Validity               certValidity             `json:"validity"`
	Subject                certSubject              `json:"subject"`
	SubjectPublicKeyInfo   certSubjectPublicKeyInfo `json:"subjectPublicKeyInfo"`
	X509v3Extensions       certExtensions           `json:"x509v3Extensions"`
	X509v3BasicConstraints string                   `json:"x509v3BasicConstraints"`
	CA                     bool                     `json:"ca"`
	Analysis               interface{}              `json:"analysis"` //for future use...
	ParentSignature        []string                 `json:"parentSignature"`
	IsChainValid           bool                     `json:"isChainValid"`
	ValidationError        string                   `json:"ValidationError"` //exists only if isChainValid is false
	CollectionTimestamp    string                   `json:"collectionTimestamp"`
	LastSeenTimestamp      string                   `json:"lastSeenTimestamp"`
}

type certIssuer struct {
	Country      []string `json:"c"`
	Organisation []string `json:"o"`
	OrgUnit      []string `json:"ou"`
	CommonName   string   `json:"cn"`
}

type certValidity struct {
	NotBefore string `json:"notBefore"`
	NotAfter  string `json:"notAfter"`
}

type certSubject struct {
	Country      []string `json:"c"`
	Organisation []string `json:"o"`
	OrgUnit      []string `json:"ou"`
	CommonName   string   `json:"cn"`
}

type certSubjectPublicKeyInfo struct {
	PublicKeyAlgorithm string `json:"publicKeyAlgorithm"`
}

//Currently exporting extensions that are already decoded into the x509 Certificate structure

type certExtensions struct {
	AuthorityKeyId         []byte   `json:"authorityKeyId"`
	SubjectKeyId           []byte   `json:"subjectKeyId"`
	KeyUsage               []string `json:"keyUsage"`
	ExtendedKeyUsage       []string `json:"extendedKeyUsage"`
	SubjectAlternativeName []string `json:"subjectAlternativeName"`
	CRLDistributionPoints  []string `json:"crlDistributionPoints"`
}

type CertX509v3BasicConstraints struct {
	CA       bool        `json:"ca"`
	Analysis interface{} `json:"analysis"`
}

type CertChain struct {
	Domain string   `json:"domain"`
	Certs  []string `json:"certs"`
}

type ids struct {
	_type  string   `json:"type"`
	values []string `json:"values"`
}

type JsonRawCert struct {
	RawCert string `json:"rawCert"`
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func SHA256Hash(data []byte) string {
	h := sha256.New()
	h.Write(data)
	return fmt.Sprintf("%X", h.Sum(nil))
	//return hex.EncodeToString(h.Sum(nil))
}

func panicIf(err error) bool {
	if err != nil {
		log.Println(fmt.Sprintf("%s", err))
		return true
	}

	return false
}

func worker(msgs <-chan amqp.Delivery, es *elastigo.Conn) {

	forever := make(chan bool)
	defer wg.Done()

	for d := range msgs {

		chain := CertChain{}

		err := json.Unmarshal(d.Body, &chain)
		panicIf(err)

		analyseAndPushCertificates(&chain, es)

		d.Ack(false)
	}

	<-forever
}

func analyseAndPushCertificates(chain *CertChain, es *elastigo.Conn) {

	var certs []*x509.Certificate

	for _, data := range chain.Certs { //create certificate chain from chain struct

		certRaw, err := base64.StdEncoding.DecodeString(data)
		panicIf(err)

		var certif *x509.Certificate
		certif, err = x509.ParseCertificate(certRaw)
		panicIf(err)

		//gotta decide if we're going to keep this here
		// 	searchJson := `{
		//     "query" : {
		//         "term" : { "_id" : "` + SHA256Hash(certif.Raw) + `" }
		//     }
		// }`
		// 	res, e := es.Search("certificates", "certificateInfo", nil, searchJson)
		// 	if !panicIf(e){
		// 		if res.Hits.Total>0{//Is certificate alreadycollected?
		// 			renewLastSeenTime(storedCert string)
		// 		}
		// 	}

		certs = append(certs, certif)
	}

	for i, c := range certs {

		inter := x509.NewCertPool()

		for _, cert := range certs[i+1 : len(certs)] {
			if cert.Issuer.CommonName != "" {
				inter.AddCert(cert)
			}
		}

		opts := x509.VerifyOptions{
			DNSName:       chain.Domain,
			Intermediates: inter,
			//will add rootCAs from cfg file
		}

		var chains [][]*x509.Certificate

		chains, err := c.Verify(opts)

		if err == nil {
			CreateStoredChain(chains[0], es)
			break
		} else {
			parentSignature := ""
			if c.Issuer.CommonName != "" && len(certs) > i+1 {
				parentSignature = SHA256Hash(certs[i+1].Raw)
			}
			pushCertificate(c, parentSignature, err.Error(), es)
		}
	}
}

func renewLastSeenTime(storedCert string) {
	//create StoredCert struct and change last seen time and resubmit
}

func CreateStoredChain(certs []*x509.Certificate, es *elastigo.Conn) {
	for i, cert := range certs {

		parentSignature := ""
		if cert.Issuer.CommonName != "" && len(certs) > i+1 {
			parentSignature = SHA256Hash(certs[i+1].Raw)
		}
		pushCertificate(cert, parentSignature, "", es)
	}
}

func pushCertificate(cert *x509.Certificate, parentSignature string, validationError string, es *elastigo.Conn) {

	stored := certtoStored(cert, parentSignature, validationError)
	jsonCert, err := json.MarshalIndent(stored, "", "    ")
	panicIf(err)

	_, err = es.Index("certificates", "certificateInfo", SHA256Hash(cert.Raw), nil, jsonCert)
	panicIf(err)

	raw := JsonRawCert{base64.StdEncoding.EncodeToString(cert.Raw)}
	jsonCert, err = json.MarshalIndent(raw, "", "    ")
	panicIf(err)
	_, err = es.Index("certificates", "certificateRaw", SHA256Hash(cert.Raw), nil, jsonCert)
	panicIf(err)

}

func getExtKeyUsageAsStringArray(cert *x509.Certificate) []string {

	usage := make([]string, len(cert.ExtKeyUsage))

	for _, eku := range cert.ExtKeyUsage {

		usage = append(usage, extKeyUsage[eku])
	}

	return usage
}

func getKeyUsageAsStringArray(cert *x509.Certificate) []string {

	var usage []string
	keyUsage := cert.KeyUsage

	//calculate included keyUsage from bitmap
	//String values taken from OpenSSL

	if keyUsage&x509.KeyUsageDigitalSignature != 0 {
		usage = append(usage, "Digital Signature")
	}
	if keyUsage&x509.KeyUsageContentCommitment != 0 {
		usage = append(usage, "Non Repudiation")
	}

	if keyUsage&x509.KeyUsageKeyEncipherment != 0 {
		usage = append(usage, "Key Encipherment")
	}

	if keyUsage&x509.KeyUsageDataEncipherment != 0 {
		usage = append(usage, "Data Encipherment")
	}

	if keyUsage&x509.KeyUsageKeyAgreement != 0 {
		usage = append(usage, "Key Agreement")
	}

	if keyUsage&x509.KeyUsageCertSign != 0 {
		usage = append(usage, "Certificate Sign")
	}

	if keyUsage&x509.KeyUsageCRLSign != 0 {
		usage = append(usage, "CRL Sign")
	}

	if keyUsage&x509.KeyUsageEncipherOnly != 0 {
		usage = append(usage, "Encipher Only")
	}

	if keyUsage&x509.KeyUsageDecipherOnly != 0 {
		usage = append(usage, "Decipher Only")
	}

	return usage
}

//This function currently stores only the extensions that are already exported by GoLang
//(in the x509 Certificate Struct)
func getCertExtensions(cert *x509.Certificate) certExtensions {

	extensions := certExtensions{}

	extensions.AuthorityKeyId = []byte(base64.StdEncoding.EncodeToString(cert.AuthorityKeyId))
	extensions.SubjectKeyId = []byte(base64.StdEncoding.EncodeToString(cert.SubjectKeyId))

	extensions.KeyUsage = getKeyUsageAsStringArray(cert)

	extensions.ExtendedKeyUsage = getExtKeyUsageAsStringArray(cert)

	extensions.SubjectAlternativeName = cert.DNSNames

	extensions.CRLDistributionPoints = cert.CRLDistributionPoints

	return extensions

}

func certtoStored(cert *x509.Certificate, parentSignature string, validationError string) StoredCertificate {

	var stored = StoredCertificate{}

	stored.Version = float64(cert.Version)

	stored.SignatureAlgorithm = signatureAlgorithm[cert.SignatureAlgorithm]

	stored.SubjectPublicKeyInfo.PublicKeyAlgorithm = publicKeyAlgorithm[cert.PublicKeyAlgorithm]

	stored.Issuer.Country = cert.Issuer.Country
	stored.Issuer.Organisation = cert.Issuer.Organization
	stored.Issuer.OrgUnit = cert.Issuer.OrganizationalUnit
	stored.Issuer.CommonName = cert.Issuer.CommonName

	stored.Subject.Country = cert.Subject.Country
	stored.Subject.Organisation = cert.Subject.Organization
	stored.Subject.OrgUnit = cert.Subject.OrganizationalUnit
	stored.Subject.CommonName = cert.Subject.CommonName

	stored.Validity.NotBefore = cert.NotBefore.UTC().String()
	stored.Validity.NotAfter = cert.NotAfter.UTC().String()

	stored.X509v3Extensions = getCertExtensions(cert)

	if cert.BasicConstraintsValid {

		stored.X509v3BasicConstraints = "Critical"
		stored.CA = cert.IsCA
	} else {
		stored.X509v3BasicConstraints = ""
		stored.CA = false
	}

	stored.CollectionTimestamp = time.Now().UTC().String()
	stored.LastSeenTimestamp = time.Now().UTC().String()

	stored.IsChainValid = true
	if validationError != "" {
		stored.IsChainValid = false
		stored.ValidationError = validationError
	}

	stored.ParentSignature = append(stored.ParentSignature, parentSignature)

	return stored

}

//Print raw extension info
//for debugging purposes
func printRawCertExtensions(cert *x509.Certificate) {

	for i, extension := range cert.Extensions {

		var numbers string
		for num, num2 := range extension.Id {

			numbers = numbers + " " + "[" + strconv.Itoa(num) + " " + strconv.Itoa(num2) + "]"

		}
		log.Println("//", strconv.Itoa(i), ": {", numbers, "}", string(extension.Value))
	}

}

var wg sync.WaitGroup

func main() {

	conn, err := amqp.Dial("amqp://guest:guest@localhost:5672/")
	failOnError(err, "Failed to connect to RabbitMQ")
	defer conn.Close()

	es := elastigo.NewConn()
	es.Domain = "83.212.99.104:9200"

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	defer ch.Close()

	q, err := ch.QueueDeclare(
		"scan_results_queue", // name
		true,                 // durable
		false,                // delete when unused
		false,                // exclusive
		false,                // no-wait
		nil,                  // arguments
	)
	failOnError(err, "Failed to declare a queue")

	err = ch.Qos(
		3,     // prefetch count
		0,     // prefetch size
		false, // global
	)

	failOnError(err, "Failed to set QoS")

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		false,  // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)

	failOnError(err, "Failed to register a consumer")

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores)

	for i := 0; i < cores; i++ {
		wg.Add(1)
		go worker(msgs, es)
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"

	"github.com/gorilla/mux"
	// postgres driver for migrate
	_ "github.com/mattes/migrate/driver/postgres"
	"github.com/nuveo/prest/config"
	"github.com/nuveo/prest/controllers"
	"github.com/nuveo/prest/middlewares"
	"github.com/spf13/cobra"
	"github.com/urfave/negroni"
)

var cfgFile string
var prestConfig config.Prest

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "prest",
	Short: "Serve a RESTful API from any PostgreSQL database",
	Long:  `Serve a RESTful API from any PostgreSQL database, start HTTP server`,
	Run: func(cmd *cobra.Command, args []string) {
		app()
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	config.InitConf()
}

func app() {
	cfg := config.Prest{}
	config.Parse(&cfg)

	n := negroni.Classic()
	n.Use(negroni.HandlerFunc(middlewares.HandlerSet))
	if cfg.JWTKey != "" {
		n.Use(middlewares.JwtMiddleware(cfg.JWTKey))
	}
	r := config.GetRouter()
	r.HandleFunc("/databases", controllers.GetDatabases).Methods("GET")
	r.HandleFunc("/schemas", controllers.GetSchemas).Methods("GET")
	r.HandleFunc("/tables", controllers.GetTables).Methods("GET")
	r.HandleFunc("/_QUERIES/{queriesLocation}/{script}", controllers.ExecuteFromScripts)
	r.HandleFunc("/{database}/{schema}", controllers.GetTablesByDatabaseAndSchema).Methods("GET")

	crudRoutes := mux.NewRouter().PathPrefix("/").Subrouter().StrictSlash(true)

	crudRoutes.HandleFunc("/{database}/{schema}/{table}", controllers.SelectFromTables).Methods("GET")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", controllers.InsertInTables).Methods("POST")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", controllers.DeleteFromTable).Methods("DELETE")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", controllers.UpdateTable).Methods("PUT", "PATCH")

	r.PathPrefix("/").Handler(negroni.New(
		negroni.HandlerFunc(middlewares.AccessControl),
		negroni.Wrap(crudRoutes),
	))
	n.UseHandler(r)
	n.Run(fmt.Sprintf(":%v", cfg.HTTPPort))
}
<commit_msg>updating from master<commit_after>package cmd

import (
	"fmt"
	"os"

	"github.com/gorilla/mux"
	"github.com/rs/cors"
	// postgres driver for migrate
	_ "github.com/mattes/migrate/driver/postgres"
	"github.com/nuveo/prest/config"
	"github.com/nuveo/prest/controllers"
	"github.com/nuveo/prest/middlewares"
	"github.com/spf13/cobra"
	"github.com/urfave/negroni"
)

var cfgFile string
var prestConfig config.Prest

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "prest",
	Short: "Serve a RESTful API from any PostgreSQL database",
	Long:  `Serve a RESTful API from any PostgreSQL database, start HTTP server`,
	Run: func(cmd *cobra.Command, args []string) {
		app()
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	config.InitConf()
}

func app() {
	cfg := config.Prest{}
	config.Parse(&cfg)

	n := negroni.Classic()
	n.Use(negroni.HandlerFunc(middlewares.HandlerSet))
	if cfg.JWTKey != "" {
		n.Use(middlewares.JwtMiddleware(cfg.JWTKey))
	}

	r := config.GetRouter()

	r.HandleFunc("/databases", controllers.GetDatabases).Methods("GET")
	r.HandleFunc("/schemas", controllers.GetSchemas).Methods("GET")
	r.HandleFunc("/tables", controllers.GetTables).Methods("GET")
	r.HandleFunc("/_QUERIES/{queriesLocation}/{script}", controllers.ExecuteFromScripts)
	r.HandleFunc("/{database}/{schema}", controllers.GetTablesByDatabaseAndSchema).Methods("GET")

	crudRoutes := mux.NewRouter().PathPrefix("/").Subrouter().StrictSlash(true)

	crudRoutes.HandleFunc("/{database}/{schema}/{table}", controllers.SelectFromTables).Methods("GET")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", controllers.InsertInTables).Methods("POST")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", controllers.DeleteFromTable).Methods("DELETE")
	crudRoutes.HandleFunc("/{database}/{schema}/{table}", controllers.UpdateTable).Methods("PUT", "PATCH")

	r.PathPrefix("/").Handler(negroni.New(
		negroni.HandlerFunc(middlewares.AccessControl),
		negroni.Wrap(crudRoutes),
	))

	if cfg.CORSAllowOrigin != nil {
		c := cors.New(cors.Options{
			AllowedOrigins: cfg.CORSAllowOrigin,
		})
		n.Use(c)
	}

	n.UseHandler(r)
	n.Run(fmt.Sprintf(":%v", cfg.HTTPPort))
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013-2014 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
	"container/list"
	"database/sql/driver"
	"io"
)

type firebirdsqlRows struct {
	stmt            *firebirdsqlStmt
	currentChunkRow *list.Element
	moreData        bool
	result          []driver.Value
}

func newFirebirdsqlRows(stmt *firebirdsqlStmt, result []driver.Value) *firebirdsqlRows {
	rows := new(firebirdsqlRows)
	rows.stmt = stmt
	rows.result = result
	if stmt.stmtType == isc_info_sql_stmt_select {
		rows.moreData = true
	}
	return rows
}

func (rows *firebirdsqlRows) Columns() []string {
	columns := make([]string, len(rows.stmt.xsqlda))
	for i, x := range rows.stmt.xsqlda {
		columns[i] = x.aliasname
	}
	return columns
}

func (rows *firebirdsqlRows) Close() (er error) {
	rows.stmt.Close()
	return
}

func (rows *firebirdsqlRows) Next(dest []driver.Value) (err error) {
	if rows.stmt.stmtType == isc_info_sql_stmt_exec_procedure {
		if rows.result == nil {
			err = io.EOF
			return
		}
		for i, v := range rows.result {
			dest[i] = v
		}
		rows.result = nil
	}

	if rows.currentChunkRow != nil {
		rows.currentChunkRow = rows.currentChunkRow.Next()
	}
	if rows.currentChunkRow == nil && rows.moreData == true {
		// Get one chunk
		var chunk *list.List
		rows.stmt.wp.opFetch(rows.stmt.stmtHandle, rows.stmt.blr)
		chunk, rows.moreData, err = rows.stmt.wp.opFetchResponse(rows.stmt.stmtHandle, rows.stmt.xsqlda)
		rows.currentChunkRow = chunk.Front()
	}

	if rows.currentChunkRow == nil {
		err = io.EOF
		return
	}
	row, _ := rows.currentChunkRow.Value.([]driver.Value)
	for i, v := range row {
		dest[i] = v
	}

	return
}
<commit_msg>retun correct value when procedure call<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013-2014 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
	"container/list"
	"database/sql/driver"
	"io"
)

type firebirdsqlRows struct {
	stmt            *firebirdsqlStmt
	currentChunkRow *list.Element
	moreData        bool
	result          []driver.Value
}

func newFirebirdsqlRows(stmt *firebirdsqlStmt, result []driver.Value) *firebirdsqlRows {
	rows := new(firebirdsqlRows)
	rows.stmt = stmt
	rows.result = result
	if stmt.stmtType == isc_info_sql_stmt_select {
		rows.moreData = true
	}
	return rows
}

func (rows *firebirdsqlRows) Columns() []string {
	columns := make([]string, len(rows.stmt.xsqlda))
	for i, x := range rows.stmt.xsqlda {
		columns[i] = x.aliasname
	}
	return columns
}

func (rows *firebirdsqlRows) Close() (er error) {
	rows.stmt.Close()
	return
}

func (rows *firebirdsqlRows) Next(dest []driver.Value) (err error) {
	if rows.stmt.stmtType == isc_info_sql_stmt_exec_procedure {
		if rows.result != nil {
			for i, v := range rows.result {
				dest[i] = v
			}
			rows.result = nil
		} else {
			err = io.EOF
		}

		return
	}

	if rows.currentChunkRow != nil {
		rows.currentChunkRow = rows.currentChunkRow.Next()
	}
	if rows.currentChunkRow == nil && rows.moreData == true {
		// Get one chunk
		var chunk *list.List
		rows.stmt.wp.opFetch(rows.stmt.stmtHandle, rows.stmt.blr)
		chunk, rows.moreData, err = rows.stmt.wp.opFetchResponse(rows.stmt.stmtHandle, rows.stmt.xsqlda)
		rows.currentChunkRow = chunk.Front()
	}

	if rows.currentChunkRow == nil {
		err = io.EOF
		return
	}
	row, _ := rows.currentChunkRow.Value.([]driver.Value)
	for i, v := range row {
		dest[i] = v
	}

	return
}
<|endoftext|>


<commit_before>// Package multipartutil provides helper functionality for using multipart.Writer.
// Steps are to call NewMultipartBuilder(), write parts, call builder.Close(), and
// retrieve Content-Type header from builder.Writer.FormDataContentType().

package multipartutil

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"path/filepath"
	"strings"

	hum "github.com/grokify/gotilla/net/httputilmore"
)

func NewReaderBodyBytes(body []byte, boundary string) *multipart.Reader {
	return multipart.NewReader(bytes.NewReader(body), boundary)
}

func NewMultipartReaderForHttpResponse(resp *http.Response) (*multipart.Reader, error) {
	contentType := resp.Header.Get(hum.HeaderContentType)
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return nil, err
	} else if !strings.HasPrefix(mediaType, "multipart/") {
		return nil, fmt.Errorf("MediaType is not multipart [%v]", mediaType)
	}
	if boundary, ok := params["boundary"]; !ok {
		return nil, fmt.Errorf("MIME Boundary not found in Content-Type header [%v]", contentType)
	} else {
		return multipart.NewReader(resp.Body, boundary), nil
	}
}

// MultipartBuilder is a multipart helper.
type MultipartBuilder struct {
	Buffer *bytes.Buffer
	Writer *multipart.Writer
}

// NewMultipartBuilder instantiates a new MultipartBuilder.
func NewMultipartBuilder() MultipartBuilder {
	builder := MultipartBuilder{}
	var b bytes.Buffer
	builder.Buffer = &b
	builder.Writer = multipart.NewWriter(&b)
	return builder
}

// WriteFieldString adds a text part.
func (builder *MultipartBuilder) WriteFieldString(partName string, data string) error {
	return builder.Writer.WriteField(partName, data)
}

// WriteFieldAsJSON adds a JSON part.
func (builder *MultipartBuilder) WriteFieldAsJSON(partName string, data interface{}, base64Encode bool) error {
	jsonBytes, err := json.Marshal(data)
	if err != nil {
		return err
	}

	header := textproto.MIMEHeader{}
	header.Add(hum.HeaderContentDisposition, fmt.Sprintf(`form-data; name="%s"`, partName))
	header.Add(hum.HeaderContentType, hum.ContentTypeAppJsonUtf8)
	if base64Encode {
		header.Add(hum.HeaderContentTransferEncoding, "base64")
	}

	partWriter, err := builder.Writer.CreatePart(header)
	if err != nil {
		return err
	}

	if base64Encode {
		str := base64.StdEncoding.EncodeToString(jsonBytes)
		_, err = bytes.NewBuffer([]byte(str)).WriteTo(partWriter)
	} else {
		_, err = bytes.NewBuffer(jsonBytes).WriteTo(partWriter)
	}
	return err
}

// WriteFilepathPlus adds a file part given a filename with the Content Type and
// other associated headers as needed. After builder.Close() has been called,
// use like `req, err := http.NewRequest("POST", url, builder.Buffer)`
func (builder *MultipartBuilder) WriteFilePathPlus(partName, srcFilepath string, base64Encode bool) error {
	_, filename := filepath.Split(srcFilepath)
	mimeType := mime.TypeByExtension(filepath.Ext(srcFilepath))

	header := textproto.MIMEHeader{}
	header.Add(hum.HeaderContentDisposition,
		fmt.Sprintf(`form-data; name="%s"; filename="%s"`, partName, filename))
	if len(mimeType) > 0 {
		header.Add(hum.HeaderContentType, mimeType)
	}
	if base64Encode {
		header.Add(hum.HeaderContentTransferEncoding, "base64")
	}

	partWriter, err := builder.Writer.CreatePart(header)
	if err != nil {
		return err
	}

	fileBytes, err := ioutil.ReadFile(srcFilepath)
	if err != nil {
		return err
	}

	if base64Encode {
		str := base64.StdEncoding.EncodeToString(fileBytes)
		_, err = bytes.NewBuffer([]byte(str)).WriteTo(partWriter)
	} else {
		_, err = bytes.NewBuffer(fileBytes).WriteTo(partWriter)
	}
	return err
}

// WriteFilePath adds a file part given a filename.
func (builder *MultipartBuilder) WriteFilePath(partName, srcFilepath string) error {
	file, err := os.Open(srcFilepath)
	if err != nil {
		return err
	}
	defer file.Close()
	_, filename := filepath.Split(srcFilepath)
	return builder.WriteFileReader(partName, filename, file)
}

// WriteFileHeader adds a file part given a part name and *multipart.FileHeader.
// See more at http://sanatgersappa.blogspot.com/2013/03/handling-multiple-file-uploads-in-go.html
// and https://gist.github.com/sanatgersappa/5127317#file-app-go
func (builder *MultipartBuilder) WriteFileHeader(partName string, fileHeader *multipart.FileHeader) error {
	file, err := fileHeader.Open()
	if err != nil {
		return err
	}
	defer file.Close()
	return builder.WriteFileReader(partName, fileHeader.Filename, file)
}

// WriteFileReader adds a file part given a filename and `io.Reader`.
func (builder *MultipartBuilder) WriteFileReader(partName, filename string, src io.Reader) error {
	fw, err := builder.Writer.CreateFormFile(partName, filename)
	if err != nil {
		return err
	}
	_, err = io.Copy(fw, src)
	return err
}

// Close closes the `multipart.Writer`.
func (builder *MultipartBuilder) Close() error {
	return builder.Writer.Close()
}

// ContentType returns the content type for the `Content-Type` header.
func (builder *MultipartBuilder) ContentType() string {
	return builder.Writer.FormDataContentType()
}
<commit_msg>update mime.WriteFilePathPlus<commit_after>// Package multipartutil provides helper functionality for using multipart.Writer.
// Steps are to call NewMultipartBuilder(), write parts, call builder.Close(), and
// retrieve Content-Type header from builder.Writer.FormDataContentType().

package multipartutil

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"path/filepath"
	"strings"

	hum "github.com/grokify/gotilla/net/httputilmore"
)

func NewReaderBodyBytes(body []byte, boundary string) *multipart.Reader {
	return multipart.NewReader(bytes.NewReader(body), boundary)
}

func NewMultipartReaderForHttpResponse(resp *http.Response) (*multipart.Reader, error) {
	contentType := resp.Header.Get(hum.HeaderContentType)
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return nil, err
	} else if !strings.HasPrefix(mediaType, "multipart/") {
		return nil, fmt.Errorf("MediaType is not multipart [%v]", mediaType)
	}
	if boundary, ok := params["boundary"]; !ok {
		return nil, fmt.Errorf("MIME Boundary not found in Content-Type header [%v]", contentType)
	} else {
		return multipart.NewReader(resp.Body, boundary), nil
	}
}

// MultipartBuilder is a multipart helper.
type MultipartBuilder struct {
	Buffer *bytes.Buffer
	Writer *multipart.Writer
}

// NewMultipartBuilder instantiates a new MultipartBuilder.
func NewMultipartBuilder() MultipartBuilder {
	builder := MultipartBuilder{}
	var b bytes.Buffer
	builder.Buffer = &b
	builder.Writer = multipart.NewWriter(&b)
	return builder
}

// WriteFieldString adds a text part.
func (builder *MultipartBuilder) WriteFieldString(partName string, data string) error {
	return builder.Writer.WriteField(partName, data)
}

// WriteFieldAsJSON adds a JSON part.
func (builder *MultipartBuilder) WriteFieldAsJSON(partName string, data interface{}, base64Encode bool) error {
	jsonBytes, err := json.Marshal(data)
	if err != nil {
		return err
	}

	header := textproto.MIMEHeader{}
	header.Add(hum.HeaderContentDisposition, fmt.Sprintf(`form-data; name="%s"`, partName))
	header.Add(hum.HeaderContentType, hum.ContentTypeAppJsonUtf8)
	if base64Encode {
		header.Add(hum.HeaderContentTransferEncoding, "base64")
	}

	partWriter, err := builder.Writer.CreatePart(header)
	if err != nil {
		return err
	}

	if base64Encode {
		str := base64.StdEncoding.EncodeToString(jsonBytes)
		_, err = bytes.NewBuffer([]byte(str)).WriteTo(partWriter)
	} else {
		_, err = bytes.NewBuffer(jsonBytes).WriteTo(partWriter)
	}
	return err
}

// WriteFilepathPlus adds a file part given a filename with the Content Type and
// other associated headers as needed. After builder.Close() has been called,
// use like `req, err := http.NewRequest("POST", url, builder.Buffer)`.
// Content-Dispostion uses optional attribute as defined here:
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
func (builder *MultipartBuilder) WriteFilePathPlus(partName, srcFilepath string, base64Encode bool) error {
	partName = strings.TrimSpace(partName)
	_, filename := filepath.Split(srcFilepath)
	filename = strings.TrimSpace(filename)
	mimeType := mime.TypeByExtension(filepath.Ext(srcFilepath))

	header := textproto.MIMEHeader{}
	cd := []string{"form-data"}
	if len(partName) > 0 {
		cd = append(cd, fmt.Sprintf(`name="%s"`, partName))
	}
	if len(filename) > 0 {
		cd = append(cd, fmt.Sprintf(`filename="%s"`, filename))
	}
	header.Add(hum.HeaderContentDisposition, strings.Join(cd, "; "))

	if len(mimeType) > 0 {
		header.Add(hum.HeaderContentType, mimeType)
	}
	if base64Encode {
		header.Add(hum.HeaderContentTransferEncoding, "base64")
	}

	partWriter, err := builder.Writer.CreatePart(header)
	if err != nil {
		return err
	}

	fileBytes, err := ioutil.ReadFile(srcFilepath)
	if err != nil {
		return err
	}

	if base64Encode {
		str := base64.StdEncoding.EncodeToString(fileBytes)
		_, err = bytes.NewBuffer([]byte(str)).WriteTo(partWriter)
	} else {
		_, err = bytes.NewBuffer(fileBytes).WriteTo(partWriter)
	}
	return err
}

// WriteFilePath adds a file part given a filename.
func (builder *MultipartBuilder) WriteFilePath(partName, srcFilepath string) error {
	file, err := os.Open(srcFilepath)
	if err != nil {
		return err
	}
	defer file.Close()
	_, filename := filepath.Split(srcFilepath)
	return builder.WriteFileReader(partName, filename, file)
}

// WriteFileHeader adds a file part given a part name and *multipart.FileHeader.
// See more at http://sanatgersappa.blogspot.com/2013/03/handling-multiple-file-uploads-in-go.html
// and https://gist.github.com/sanatgersappa/5127317#file-app-go
func (builder *MultipartBuilder) WriteFileHeader(partName string, fileHeader *multipart.FileHeader) error {
	file, err := fileHeader.Open()
	if err != nil {
		return err
	}
	defer file.Close()
	return builder.WriteFileReader(partName, fileHeader.Filename, file)
}

// WriteFileReader adds a file part given a filename and `io.Reader`.
func (builder *MultipartBuilder) WriteFileReader(partName, filename string, src io.Reader) error {
	fw, err := builder.Writer.CreateFormFile(partName, filename)
	if err != nil {
		return err
	}
	_, err = io.Copy(fw, src)
	return err
}

// Close closes the `multipart.Writer`.
func (builder *MultipartBuilder) Close() error {
	return builder.Writer.Close()
}

// ContentType returns the content type for the `Content-Type` header.
func (builder *MultipartBuilder) ContentType() string {
	return builder.Writer.FormDataContentType()
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy_test

import (
	"errors"
	"testing"
	"time"

	"github.com/googlecloudplatform/gcsfuse/gcsproxy"
	"github.com/googlecloudplatform/gcsfuse/lease/mock_lease"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestMutableContent(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Invariant-checking mutable content
////////////////////////////////////////////////////////////////////////

// A wrapper around MutableContent that calls CheckInvariants whenever
// invariants should hold. For catching logic errors early in the test.
type checkingMutableContent struct {
	ctx     context.Context
	wrapped *gcsproxy.MutableContent
}

func (mc *checkingMutableContent) Stat() (gcsproxy.StatResult, error) {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	return mc.wrapped.Stat(mc.ctx)
}

func (mc *checkingMutableContent) ReadAt(b []byte, o int64) (int, error) {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	return mc.wrapped.ReadAt(mc.ctx, b, o)
}

func (mc *checkingMutableContent) WriteAt(b []byte, o int64) (int, error) {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	return mc.wrapped.WriteAt(mc.ctx, b, o)
}

func (mc *checkingMutableContent) Truncate(n int64) error {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	return mc.wrapped.Truncate(mc.ctx, n)
}

func (mc *checkingMutableContent) Destroy() {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	mc.wrapped.Destroy()
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type mutableContentTest struct {
	ctx context.Context

	initialContent mock_lease.MockReadProxy
	rwl            mock_lease.MockReadWriteLease
	clock          timeutil.SimulatedClock

	mc checkingMutableContent
}

var _ SetUpInterface = &mutableContentTest{}

func (t *mutableContentTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the mock initial contents, including a default size.
	t.initialContent = mock_lease.NewMockReadProxy(
		ti.MockController,
		"initialContent")

	const initialContentSize = 11
	ExpectCall(t.initialContent, "Size")().
		WillRepeatedly(Return(initialContentSize))

	// Set up a mock read/write lease.
	t.rwl = mock_lease.NewMockReadWriteLease(
		ti.MockController,
		"rwl")

	// Ignore uninteresting calls.
	ExpectCall(t.initialContent, "CheckInvariants")().
		WillRepeatedly(Return())

	// Set up the clock.
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))

	// And the mutable content.
	t.mc.ctx = ti.Ctx
	t.mc.wrapped = gcsproxy.NewMutableContent(
		t.initialContent,
		&t.clock)
}

////////////////////////////////////////////////////////////////////////
// Clean state
////////////////////////////////////////////////////////////////////////

type CleanTest struct {
	mutableContentTest
}

func init() { RegisterTestSuite(&CleanTest{}) }

func (t *CleanTest) ReadAt_CallsProxy() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) ReadAt_ProxyFails() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) ReadAt_ProxySuceeds() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) Stat() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) WriteAt_UpgradeFails() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) WriteAt_UpgradeSucceeds() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) Truncate_UpgradeFails() {
	// Upgrade
	ExpectCall(t.initialContent, "Upgrade")(Any()).
		WillOnce(Return(nil, errors.New("taco")))

	// Call
	err := t.mc.Truncate(0)

	ExpectThat(err, Error(HasSubstr("Upgrade")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *CleanTest) Truncate_UpgradeSucceeds() {
	// Upgrade -- succeed.
	ExpectCall(t.initialContent, "Upgrade")(Any()).
		WillOnce(Return(t.rwl, nil))

	// The read/write lease should be called.
	ExpectCall(t.rwl, "Truncate")(17).
		WillOnce(Return(errors.New("")))

	// Call.
	t.mc.Truncate(17)

	// A further call should go right through to the read/write lease again.
	ExpectCall(t.rwl, "Truncate")(19).
		WillOnce(Return(errors.New("")))

	t.mc.Truncate(19)
}

////////////////////////////////////////////////////////////////////////
// Dirty state
////////////////////////////////////////////////////////////////////////

type DirtyTest struct {
	mutableContentTest
}

func init() { RegisterTestSuite(&DirtyTest{}) }

func (t *DirtyTest) ReadAt_CallsLease() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) ReadAt_LeaseFails() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) ReadAt_LeaseSuceeds() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) WriteAt_CallsLease() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) WriteAt_LeaseFails() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) WriteAt_LeaseSucceeds() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) WriteAt_DirtyThreshold() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) Truncate_CallsLease() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) Truncate_LeaseFails() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) Truncate_LeaseSucceeds() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) Truncate_DirtyThreshold() {
	AssertTrue(false, "TODO")
}
<commit_msg>CleanTest.WriteAt_UpgradeFails<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy_test

import (
	"errors"
	"testing"
	"time"

	"github.com/googlecloudplatform/gcsfuse/gcsproxy"
	"github.com/googlecloudplatform/gcsfuse/lease/mock_lease"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

func TestMutableContent(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Invariant-checking mutable content
////////////////////////////////////////////////////////////////////////

// A wrapper around MutableContent that calls CheckInvariants whenever
// invariants should hold. For catching logic errors early in the test.
type checkingMutableContent struct {
	ctx     context.Context
	wrapped *gcsproxy.MutableContent
}

func (mc *checkingMutableContent) Stat() (gcsproxy.StatResult, error) {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	return mc.wrapped.Stat(mc.ctx)
}

func (mc *checkingMutableContent) ReadAt(b []byte, o int64) (int, error) {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	return mc.wrapped.ReadAt(mc.ctx, b, o)
}

func (mc *checkingMutableContent) WriteAt(b []byte, o int64) (int, error) {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	return mc.wrapped.WriteAt(mc.ctx, b, o)
}

func (mc *checkingMutableContent) Truncate(n int64) error {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	return mc.wrapped.Truncate(mc.ctx, n)
}

func (mc *checkingMutableContent) Destroy() {
	mc.wrapped.CheckInvariants()
	defer mc.wrapped.CheckInvariants()
	mc.wrapped.Destroy()
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type mutableContentTest struct {
	ctx context.Context

	initialContent mock_lease.MockReadProxy
	rwl            mock_lease.MockReadWriteLease
	clock          timeutil.SimulatedClock

	mc checkingMutableContent
}

var _ SetUpInterface = &mutableContentTest{}

func (t *mutableContentTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the mock initial contents, including a default size.
	t.initialContent = mock_lease.NewMockReadProxy(
		ti.MockController,
		"initialContent")

	const initialContentSize = 11
	ExpectCall(t.initialContent, "Size")().
		WillRepeatedly(Return(initialContentSize))

	// Set up a mock read/write lease.
	t.rwl = mock_lease.NewMockReadWriteLease(
		ti.MockController,
		"rwl")

	// Ignore uninteresting calls.
	ExpectCall(t.initialContent, "CheckInvariants")().
		WillRepeatedly(Return())

	// Set up the clock.
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))

	// And the mutable content.
	t.mc.ctx = ti.Ctx
	t.mc.wrapped = gcsproxy.NewMutableContent(
		t.initialContent,
		&t.clock)
}

////////////////////////////////////////////////////////////////////////
// Clean state
////////////////////////////////////////////////////////////////////////

type CleanTest struct {
	mutableContentTest
}

func init() { RegisterTestSuite(&CleanTest{}) }

func (t *CleanTest) ReadAt_CallsProxy() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) ReadAt_ProxyFails() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) ReadAt_ProxySuceeds() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) Stat() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) WriteAt_UpgradeFails() {
	// Upgrade
	ExpectCall(t.initialContent, "Upgrade")(Any()).
		WillOnce(Return(nil, errors.New("taco")))

	// Call
	_, err := t.mc.WriteAt(make([]byte, 1), 0)

	ExpectThat(err, Error(HasSubstr("Upgrade")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *CleanTest) WriteAt_UpgradeSucceeds() {
	AssertTrue(false, "TODO")
}

func (t *CleanTest) Truncate_UpgradeFails() {
	// Upgrade
	ExpectCall(t.initialContent, "Upgrade")(Any()).
		WillOnce(Return(nil, errors.New("taco")))

	// Call
	err := t.mc.Truncate(0)

	ExpectThat(err, Error(HasSubstr("Upgrade")))
	ExpectThat(err, Error(HasSubstr("taco")))
}

func (t *CleanTest) Truncate_UpgradeSucceeds() {
	// Upgrade -- succeed.
	ExpectCall(t.initialContent, "Upgrade")(Any()).
		WillOnce(Return(t.rwl, nil))

	// The read/write lease should be called.
	ExpectCall(t.rwl, "Truncate")(17).
		WillOnce(Return(errors.New("")))

	// Call.
	t.mc.Truncate(17)

	// A further call should go right through to the read/write lease again.
	ExpectCall(t.rwl, "Truncate")(19).
		WillOnce(Return(errors.New("")))

	t.mc.Truncate(19)
}

////////////////////////////////////////////////////////////////////////
// Dirty state
////////////////////////////////////////////////////////////////////////

type DirtyTest struct {
	mutableContentTest
}

func init() { RegisterTestSuite(&DirtyTest{}) }

func (t *DirtyTest) ReadAt_CallsLease() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) ReadAt_LeaseFails() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) ReadAt_LeaseSuceeds() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) WriteAt_CallsLease() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) WriteAt_LeaseFails() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) WriteAt_LeaseSucceeds() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) WriteAt_DirtyThreshold() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) Truncate_CallsLease() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) Truncate_LeaseFails() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) Truncate_LeaseSucceeds() {
	AssertTrue(false, "TODO")
}

func (t *DirtyTest) Truncate_DirtyThreshold() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>// Copyright 2013 gopm authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// gopm(Go Package Manager) is a Go package manage tool for search, install, update and share packages in Go.
package main

import (
	"os"
	"runtime"

	"github.com/codegangsta/cli"

	"github.com/gpmgo/gopm/cmd"
)

// +build go1.1

// Test that go1.1 tag above is included in builds. main.go refers to this definition.
const go11tag = true

const APP_VER = "0.5.7.1205.7"

// 	//cmd.CmdSearch,
// 		cmdClean,
// 		cmdDoc,
// 		cmdEnv,
// 		cmdFix,
// 		cmdList,
// 		cmdTest,
// 		cmdTool,
// 		cmdVet,
// }

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
	app := cli.NewApp()
	app.Name = "gopm"
	app.Usage = "Go Package Manager"
	app.Version = APP_VER
	app.Commands = []cli.Command{
		cmd.CmdGet,
		cmd.CmdBin,
		cmd.CmdGen,
		cmd.CmdRun,
		cmd.CmdBuild,
		cmd.CmdInstall,
		//cmd.CmdUpdate,
		//cmd.CmdTest,
	}
	app.Flags = append(app.Flags, []cli.Flag{
		cli.BoolFlag{"noterm", "Disable color output"},
		cli.BoolFlag{"verbose, v", "show process details"},
	}...)
	app.Run(os.Args)
}
<commit_msg>Update<commit_after>// Copyright 2013 gopm authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// gopm(Go Package Manager) is a Go package manage tool for search, install, update and share packages in Go.
package main

import (
	"os"
	"runtime"

	"github.com/codegangsta/cli"

	"github.com/gpmgo/gopm/cmd"
)

// +build go1.1

// Test that go1.1 tag above is included in builds. main.go refers to this definition.
const go11tag = true

const APP_VER = "0.5.7.1205.7"

// 	//cmd.CmdSearch,
// 		cmdClean,
// 		cmdDoc,
// 		cmdEnv,
// 		cmdFix,
// 		cmdList,
// 		cmdTest,
// 		cmdTool,
// 		cmdVet,
// }

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
	app := cli.NewApp()
	app.Name = "gopm"
	app.Usage = "Go Package Manager"
	app.Version = APP_VER
	app.Commands = []cli.Command{
		cmd.CmdGet,
		cmd.CmdBin,
		cmd.CmdGen,
		cmd.CmdRun,
		cmd.CmdBuild,
		cmd.CmdInstall,
		//cmd.CmdUpdate,
		//cmd.CmdTest,
	}
	app.Flags = append(app.Flags, []cli.Flag{
		cli.BoolFlag{"noterm", "Disable color output"},
		cli.BoolFlag{"verbose", "show process details"},
	}...)
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	plug "github.com/jcline/goto/plugins"
	"github.com/jcline/goty"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

type UriFunc func(*string) (*string, error)
type WriteFunc func(*plug.IRCMessage, *string) (*plug.IRCMessage, error)

// Commands
var matchSpoilers = regexp.MustCompile(`(?i)(.*spoil.*)`)

func auth(con *goty.IRCConn, writeMessage chan plug.IRCMessage, user string) {
	var pswd string
	fmt.Printf("Password for NickServ:\n")
	_, err := fmt.Scanf("%s", &pswd)
	if err != nil {
		return
	}

	msg := plug.IRCMessage{Channel: "NickServ", Msg: "IDENTIFY " + user + " " + pswd}
	writeMessage <- msg
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, ErrConfNotFound
	}
	return false, err
}

type Settings struct {
	Server   string          `json:"server"`
	UserName string          `json:"userName"`
	RealName string          `json:"realName"`
	Channels []string        `json:"channels"`
	Plugins  plug.PluginConf `json:"plugin_conf"`
}

var ErrConfNotFound = errors.New("Conf does not exist")

func readConfig() (conf Settings, path string, err error) {
	args := os.Args
	path = ""
	if len(args) == 2 {
		path = filepath.Clean(args[1])
	} else {
		path = os.Getenv("XDG_CONFIG_HOME")
		if path == "" {
			path = filepath.Join("$HOME", ".config", "goto", "conf")
		} else {
			path = filepath.Join(path, "goto", "conf")
		}
	}

	path, err = filepath.Abs(os.ExpandEnv(path))
	if err != nil {
		return
	}

	log.Println(path)

	_, err = exists(path)
	if err != nil {
		return
	}

	file, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	err = json.Unmarshal(file, &conf)
	if err != nil {
		return
	}

	return
}

func getStr(prompt string, failurePrompt string, invalid func(string) bool) (result string, err error) {
	result = ""
	for {
		log.Println(prompt)
		_, err = fmt.Scanf("%s", &result)
		if err != nil {
			return
		}
		if invalid(result) {
			log.Println(failurePrompt)
		} else {
			return
		}
	}
}

func createConfig(path string, useMal bool, useYoutube bool) (conf Settings, err error) {

	_, err = exists(path)
	log.Println(exists, err)
	if err == ErrConfNotFound {
		err = os.MkdirAll(filepath.Dir(path), 0755)
		log.Println(path, ":", filepath.Dir(path))
		if err != nil && !os.IsPermission(err) {
			return
		}
	}

	isEmpty := func(s string) bool {
		return strings.TrimSpace(s) == ""
	}

	conf.Server, err = getStr("Server (e.g. irc.freenode.net:6666):", "You must include a port.", func(s string) bool { return !strings.Contains(s, ":") })
	if err != nil {
		return
	}

	conf.UserName, err = getStr("User name:", "User name must not be empty", isEmpty)
	if err != nil {
		return
	}

	conf.RealName, err = getStr("Real name:", "Real name must not be empty", isEmpty)
	if err != nil {
		return
	}

	chans, err := getStr("Channels to join (e.g. #chan1,#chan2 or #chan1):", "You must provide at least one channel", func(s string) bool {
		return isEmpty(s) || !strings.Contains(s, "#")
	})
	if err != nil {
		return
	}
	conf.Channels = strings.Split(chans, ",")

	if useMal {
		conf.Plugins.Mal.User, err = getStr("MAL user name:", "User name must not be empty", isEmpty)
		if err != nil {
			return
		}
		conf.Plugins.Mal.Password, err = getStr("MAL password:", "Password must not be empty", isEmpty)
		if err != nil {
			return
		}
	}

	if useYoutube {
		conf.Plugins.Youtube.Key, err = getStr("Youtube API Key:", "API key must not be empty", isEmpty)
		if err != nil {
			return
		}
	}

	js, err := json.Marshal(conf)
	if err != nil {
		return
	}

	log.Println("Writing to: ", path)
	err = ioutil.WriteFile(path, js, 0644)
	if err != nil {
		return
	}

	return
}

func main() {
	conf, path, err := readConfig()
	if err != nil {
		if err == ErrConfNotFound {
			log.Println("Could not read config, would you like to create one? [y/n]")
			var response string
			_, err := fmt.Scanf("%s", &response)
			if err != nil {
				log.Fatal(err)
			}
			if response == "y" || response == "Y" {
				conf, err = createConfig(path, true, true)
				if err != nil {
					log.Fatal(err)
				}
			} else {
				log.Fatal("I can't do anything without config.")
			}
		} else {
			log.Fatal(err)
		}
	}

	log.Println(conf)
	conf.Plugins.UserName = conf.UserName
	con, err := goty.Dial(conf.Server, conf.UserName, conf.RealName)
	if err != nil {
		log.Fatal(err)
	}

	rand.Seed(time.Now().UnixNano())

	writeMessage := make(chan plug.IRCMessage, 1000)
	go messageHandler(con, writeMessage, conf.Channels, 10, 2)

	var autobanner plug.Autoban
	var plugins []plug.Plugin
	plugins = append(plugins, new(plug.AmiAmi))
	plugins = append(plugins, new(plug.Help))
	plugins = append(plugins, new(plug.Mal))
	plugins = append(plugins, new(plug.MyFigureCollection))
	plugins = append(plugins, new(plug.Plamoya))
	plugins = append(plugins, new(plug.Reddit))
	plugins = append(plugins, new(plug.Vimeo))
	plugins = append(plugins, new(plug.Youtube))

	autobanner.Setup(writeMessage, conf.Plugins)
	for _, plugin := range plugins {
		plugin.Setup(writeMessage, conf.Plugins)
	}

	auth(con, writeMessage, conf.UserName)
	for _, channel := range conf.Channels {
		con.Write <- "JOIN " + channel
	}

	for msg := range con.Read {
		log.Println(msg)
		prepared, err := getMsgInfo(msg)
		if err != nil {
			continue
		}
		prepared.When = time.Now()

		if autobanner.Match(prepared) {
			continue
		}

		// half assed filtering
		_, notFound := plug.GetFirstMatch(matchSpoilers, &prepared.Msg)
		if notFound != nil {
			for _, plugin := range plugins {
				if plugin.Match(prepared) {
					plugin.Event() <- *prepared
				}
			}
		}
	}
	con.Close()
}

type unparsedMessage struct {
	msg  string
	when time.Time
}

func message(con *goty.IRCConn, msg plug.IRCMessage) {
	privmsg := "PRIVMSG " + msg.Channel + " :" + msg.Msg + "\r\n"
	log.Println(privmsg)
	con.Write <- privmsg
}

func messageHandler(con *goty.IRCConn, event chan plug.IRCMessage, channels []string, chanDelay, pmDelay int) {
	allBooks := map[string]time.Time{}
	//chanBooks := map[string]time.Time{}
	for msg := range event {
		now := time.Now()
		key := msg.Channel + ":" + msg.User
		delay := pmDelay
		for _, channel := range channels {
			if msg.Channel == channel {
				delay = chanDelay
				break
			}
		}
		if !msg.Unlimited && now.Sub(allBooks[key]) < time.Duration(delay)*time.Second { //|| now.Sub(chanBooks[key]) < time.Second*2 {
			continue
		}
		allBooks[key] = now
		//chanBooks[key] = now
		message(con, msg)
	}
}

var PRIVMSG = regexp.MustCompile(`:(.+![^ ]+) PRIVMSG ([^ ]+) :(.*)`)
var USER = regexp.MustCompile(`([^!]+)![^@]+@(.+)`)

func getMsgInfo(msg string) (*plug.IRCMessage, error) {
	// :nick!~realname@0.0.0.0 PRIVMSG #chan :msg
	imsg := new(plug.IRCMessage)
	match := PRIVMSG.FindAllStringSubmatch(msg, -1)
	if len(match) < 1 {
		return imsg, errors.New("could not parse message")
	}
	if len(match[0]) < 3 {
		return imsg, errors.New("could not parse message")
	}

	userMask := match[0][1]
	userMatch := USER.FindAllStringSubmatch(userMask, -1)

	imsg.User = userMatch[0][1]
	imsg.Mask = userMatch[0][2]
	imsg.Channel = match[0][2]
	imsg.Msg = match[0][3]
	return imsg, nil
}
<commit_msg>conf: support autoconf mal UA<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	plug "github.com/jcline/goto/plugins"
	"github.com/jcline/goty"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

type UriFunc func(*string) (*string, error)
type WriteFunc func(*plug.IRCMessage, *string) (*plug.IRCMessage, error)

// Commands
var matchSpoilers = regexp.MustCompile(`(?i)(.*spoil.*)`)

func auth(con *goty.IRCConn, writeMessage chan plug.IRCMessage, user string) {
	var pswd string
	fmt.Printf("Password for NickServ:\n")
	_, err := fmt.Scanf("%s", &pswd)
	if err != nil {
		return
	}

	msg := plug.IRCMessage{Channel: "NickServ", Msg: "IDENTIFY " + user + " " + pswd}
	writeMessage <- msg
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, ErrConfNotFound
	}
	return false, err
}

type Settings struct {
	Server   string          `json:"server"`
	UserName string          `json:"userName"`
	RealName string          `json:"realName"`
	Channels []string        `json:"channels"`
	Plugins  plug.PluginConf `json:"plugin_conf"`
}

var ErrConfNotFound = errors.New("Conf does not exist")

func readConfig() (conf Settings, path string, err error) {
	args := os.Args
	path = ""
	if len(args) == 2 {
		path = filepath.Clean(args[1])
	} else {
		path = os.Getenv("XDG_CONFIG_HOME")
		if path == "" {
			path = filepath.Join("$HOME", ".config", "goto", "conf")
		} else {
			path = filepath.Join(path, "goto", "conf")
		}
	}

	path, err = filepath.Abs(os.ExpandEnv(path))
	if err != nil {
		return
	}

	log.Println(path)

	_, err = exists(path)
	if err != nil {
		return
	}

	file, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	err = json.Unmarshal(file, &conf)
	if err != nil {
		return
	}

	return
}

func getStr(prompt string, failurePrompt string, invalid func(string) bool) (result string, err error) {
	result = ""
	for {
		log.Println(prompt)
		_, err = fmt.Scanf("%s", &result)
		if err != nil {
			return
		}
		if invalid(result) {
			log.Println(failurePrompt)
		} else {
			return
		}
	}
}

func createConfig(path string, useMal bool, useYoutube bool) (conf Settings, err error) {

	_, err = exists(path)
	log.Println(exists, err)
	if err == ErrConfNotFound {
		err = os.MkdirAll(filepath.Dir(path), 0755)
		log.Println(path, ":", filepath.Dir(path))
		if err != nil && !os.IsPermission(err) {
			return
		}
	}

	isEmpty := func(s string) bool {
		return strings.TrimSpace(s) == ""
	}

	conf.Server, err = getStr("Server (e.g. irc.freenode.net:6666):", "You must include a port.", func(s string) bool { return !strings.Contains(s, ":") })
	if err != nil {
		return
	}

	conf.UserName, err = getStr("User name:", "User name must not be empty", isEmpty)
	if err != nil {
		return
	}

	conf.RealName, err = getStr("Real name:", "Real name must not be empty", isEmpty)
	if err != nil {
		return
	}

	chans, err := getStr("Channels to join (e.g. #chan1,#chan2 or #chan1):", "You must provide at least one channel", func(s string) bool {
		return isEmpty(s) || !strings.Contains(s, "#")
	})
	if err != nil {
		return
	}
	conf.Channels = strings.Split(chans, ",")

	if useMal {
		conf.Plugins.Mal.User, err = getStr("MAL user name:", "User name must not be empty", isEmpty)
		if err != nil {
			return
		}
		conf.Plugins.Mal.Password, err = getStr("MAL password:", "Password must not be empty", isEmpty)
		if err != nil {
			return
		}
		conf.Plugins.Mal.UserAgent , err = getStr("MAL User Agent:", "User Agent must not be empty", isEmpty)
		if err != nil {
			return
		}
	}

	if useYoutube {
		conf.Plugins.Youtube.Key, err = getStr("Youtube API Key:", "API key must not be empty", isEmpty)
		if err != nil {
			return
		}
	}

	js, err := json.Marshal(conf)
	if err != nil {
		return
	}

	log.Println("Writing to: ", path)
	err = ioutil.WriteFile(path, js, 0644)
	if err != nil {
		return
	}

	return
}

func main() {
	conf, path, err := readConfig()
	if err != nil {
		if err == ErrConfNotFound {
			log.Println("Could not read config, would you like to create one? [y/n]")
			var response string
			_, err := fmt.Scanf("%s", &response)
			if err != nil {
				log.Fatal(err)
			}
			if response == "y" || response == "Y" {
				conf, err = createConfig(path, true, true)
				if err != nil {
					log.Fatal(err)
				}
			} else {
				log.Fatal("I can't do anything without config.")
			}
		} else {
			log.Fatal(err)
		}
	}

	log.Println(conf)
	conf.Plugins.UserName = conf.UserName
	con, err := goty.Dial(conf.Server, conf.UserName, conf.RealName)
	if err != nil {
		log.Fatal(err)
	}

	rand.Seed(time.Now().UnixNano())

	writeMessage := make(chan plug.IRCMessage, 1000)
	go messageHandler(con, writeMessage, conf.Channels, 10, 2)

	var autobanner plug.Autoban
	var plugins []plug.Plugin
	plugins = append(plugins, new(plug.AmiAmi))
	plugins = append(plugins, new(plug.Help))
	plugins = append(plugins, new(plug.Mal))
	plugins = append(plugins, new(plug.MyFigureCollection))
	plugins = append(plugins, new(plug.Plamoya))
	plugins = append(plugins, new(plug.Reddit))
	plugins = append(plugins, new(plug.Vimeo))
	plugins = append(plugins, new(plug.Youtube))

	autobanner.Setup(writeMessage, conf.Plugins)
	for _, plugin := range plugins {
		plugin.Setup(writeMessage, conf.Plugins)
	}

	auth(con, writeMessage, conf.UserName)
	for _, channel := range conf.Channels {
		con.Write <- "JOIN " + channel
	}

	for msg := range con.Read {
		log.Println(msg)
		prepared, err := getMsgInfo(msg)
		if err != nil {
			continue
		}
		prepared.When = time.Now()

		if autobanner.Match(prepared) {
			continue
		}

		// half assed filtering
		_, notFound := plug.GetFirstMatch(matchSpoilers, &prepared.Msg)
		if notFound != nil {
			for _, plugin := range plugins {
				if plugin.Match(prepared) {
					plugin.Event() <- *prepared
				}
			}
		}
	}
	con.Close()
}

type unparsedMessage struct {
	msg  string
	when time.Time
}

func message(con *goty.IRCConn, msg plug.IRCMessage) {
	privmsg := "PRIVMSG " + msg.Channel + " :" + msg.Msg + "\r\n"
	log.Println(privmsg)
	con.Write <- privmsg
}

func messageHandler(con *goty.IRCConn, event chan plug.IRCMessage, channels []string, chanDelay, pmDelay int) {
	allBooks := map[string]time.Time{}
	//chanBooks := map[string]time.Time{}
	for msg := range event {
		now := time.Now()
		key := msg.Channel + ":" + msg.User
		delay := pmDelay
		for _, channel := range channels {
			if msg.Channel == channel {
				delay = chanDelay
				break
			}
		}
		if !msg.Unlimited && now.Sub(allBooks[key]) < time.Duration(delay)*time.Second { //|| now.Sub(chanBooks[key]) < time.Second*2 {
			continue
		}
		allBooks[key] = now
		//chanBooks[key] = now
		message(con, msg)
	}
}

var PRIVMSG = regexp.MustCompile(`:(.+![^ ]+) PRIVMSG ([^ ]+) :(.*)`)
var USER = regexp.MustCompile(`([^!]+)![^@]+@(.+)`)

func getMsgInfo(msg string) (*plug.IRCMessage, error) {
	// :nick!~realname@0.0.0.0 PRIVMSG #chan :msg
	imsg := new(plug.IRCMessage)
	match := PRIVMSG.FindAllStringSubmatch(msg, -1)
	if len(match) < 1 {
		return imsg, errors.New("could not parse message")
	}
	if len(match[0]) < 3 {
		return imsg, errors.New("could not parse message")
	}

	userMask := match[0][1]
	userMatch := USER.FindAllStringSubmatch(userMask, -1)

	imsg.User = userMatch[0][1]
	imsg.Mask = userMatch[0][2]
	imsg.Channel = match[0][2]
	imsg.Msg = match[0][3]
	return imsg, nil
}
<|endoftext|>


<commit_before>package errors

import (
	"encoding/json"
	"fmt"
	"path/filepath"
	"reflect"
	"runtime"

	"testing"

	"github.com/cihangir/gene/testdata"
	"github.com/cihangir/schema"
)

const expected = `package errs

var ()
`

func TestConstructors(t *testing.T) {
	var s schema.Schema
	if err := json.Unmarshal([]byte(testdata.JSON1), &s); err != nil {
		t.Fatal(err.Error())
	}

	a, err := generate(&s)
	equals(t, nil, err)
	equals(t, expected, string(a))
}

func equals(tb testing.TB, exp, act interface{}) {
	if !reflect.DeepEqual(exp, act) {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
		tb.Fail()
	}
}
<commit_msg>Errors: update error tests with the new schema<commit_after>package errors

import (
	"encoding/json"
	"fmt"
	"path/filepath"
	"reflect"
	"runtime"

	"testing"

	"github.com/cihangir/gene/testdata"
	"github.com/cihangir/schema"
)

const expected = `package errs

var (
	ErrAccountCreatedAtNotSet              = errors.New("Account.CreatedAt not set")
	ErrAccountEmailAddressNotSet           = errors.New("Account.EmailAddress not set")
	ErrAccountEmailStatusConstantNotSet    = errors.New("Account.EmailStatusConstant not set")
	ErrAccountIdNotSet                     = errors.New("Account.Id not set")
	ErrAccountPasswordNotSet               = errors.New("Account.Password not set")
	ErrAccountPasswordStatusConstantNotSet = errors.New("Account.PasswordStatusConstant not set")
	ErrAccountProfileIdNotSet              = errors.New("Account.ProfileId not set")
	ErrAccountSaltNotSet                   = errors.New("Account.Salt not set")
	ErrAccountStatusConstantNotSet         = errors.New("Account.StatusConstant not set")
	ErrAccountURLNotSet                    = errors.New("Account.URL not set")
	ErrAccountURLNameNotSet                = errors.New("Account.URLName not set")
	ErrConfigMongoNotSet                   = errors.New("Config.Mongo not set")
	ErrConfigPostgresNotSet                = errors.New("Config.Postgres not set")
	ErrProfileAvatarURLNotSet              = errors.New("Profile.AvatarURL not set")
	ErrProfileCreatedAtNotSet              = errors.New("Profile.CreatedAt not set")
	ErrProfileFirstNameNotSet              = errors.New("Profile.FirstName not set")
	ErrProfileIdNotSet                     = errors.New("Profile.Id not set")
	ErrProfileLastNameNotSet               = errors.New("Profile.LastName not set")
	ErrProfileNickNotSet                   = errors.New("Profile.Nick not set")
)
`

func TestConstructors(t *testing.T) {
	var s schema.Schema
	if err := json.Unmarshal([]byte(testdata.JSON1), &s); err != nil {
		t.Fatal(err.Error())
	}

	a, err := generate(&s)
	equals(t, nil, err)
	equals(t, expected, string(a))
}

func equals(tb testing.TB, exp, act interface{}) {
	if !reflect.DeepEqual(exp, act) {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
		tb.Fail()
	}
}
<|endoftext|>