<commit_before>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package crypto

import (
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/x509"
	obcca "github.com/openblockchain/obc-peer/obc-ca/protos"
	protobuf "google/protobuf"
	"time"

	"crypto/rsa"
	"encoding/asn1"
	"errors"
	"github.com/golang/protobuf/proto"
	"github.com/openblockchain/obc-peer/openchain/crypto/utils"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"io/ioutil"
)

var (
	// ECertSubjectRole is the ASN1 object identifier of the subject's role.
	ECertSubjectRole = asn1.ObjectIdentifier{2, 1, 3, 4, 5, 6, 7}
)

func (node *nodeImpl) retrieveECACertsChain(userID string) error {
	// Retrieve ECA certificate and verify it
	ecaCertRaw, err := node.getECACertificate()
	if err != nil {
		node.log.Error("Failed getting ECA certificate [%s].", err.Error())

		return err
	}
	node.log.Debug("ECA certificate [%s].", utils.EncodeBase64(ecaCertRaw))

	// TODO: Test ECA cert againt root CA
	// TODO: check responce.Cert against rootCA
	x509ECACert, err := utils.DERToX509Certificate(ecaCertRaw)
	if err != nil {
		node.log.Error("Failed parsing ECA certificate [%s].", err.Error())

		return err
	}

	// Prepare ecaCertPool
	node.ecaCertPool = x509.NewCertPool()
	node.ecaCertPool.AddCert(x509ECACert)

	// Store ECA cert
	node.log.Debug("Storing ECA certificate for [%s]...", userID)

	if err := node.ks.storeCert(node.conf.getECACertsChainFilename(), ecaCertRaw); err != nil {
		node.log.Error("Failed storing eca certificate [%s].", err.Error())
		return err
	}

	return nil
}

func (node *nodeImpl) retrieveEnrollmentData(enrollID, enrollPWD string) error {
	key, enrollCertRaw, enrollChainKey, err := node.getEnrollmentCertificateFromECA(enrollID, enrollPWD)
	if err != nil {
		node.log.Error("Failed getting enrollment certificate [id=%s]: [%s]", enrollID, err)

		return err
	}
	node.log.Debug("Enrollment certificate [%s].", utils.EncodeBase64(enrollCertRaw))

	node.log.Debug("Storing enrollment data for user [%s]...", enrollID)

	// Store enrollment id
	err = ioutil.WriteFile(node.conf.getEnrollmentIDPath(), []byte(enrollID), 0700)
	if err != nil {
		node.log.Error("Failed storing enrollment certificate [id=%s]: [%s]", enrollID, err)
		return err
	}

	// Store enrollment key
	if err := node.ks.storePrivateKey(node.conf.getEnrollmentKeyFilename(), key); err != nil {
		node.log.Error("Failed storing enrollment key [id=%s]: [%s]", enrollID, err)
		return err
	}

	// Store enrollment cert
	if err := node.ks.storeCert(node.conf.getEnrollmentCertFilename(), enrollCertRaw); err != nil {
		node.log.Error("Failed storing enrollment certificate [id=%s]: [%s]", enrollID, err)
		return err
	}

	// Store enrollment chain key
	if err := node.ks.storeKey(node.conf.getEnrollmentChainKeyFilename(), enrollChainKey); err != nil {
		node.log.Error("Failed storing enrollment chain key [id=%s]: [%s]", enrollID, err)
		return err
	}

	return nil
}

func (node *nodeImpl) loadEnrollmentKey() error {
	node.log.Debug("Loading enrollment key...")

	enrollPrivKey, err := node.ks.loadPrivateKey(node.conf.getEnrollmentKeyFilename())
	if err != nil {
		node.log.Error("Failed loading enrollment private key [%s].", err.Error())

		return err
	}

	node.enrollPrivKey = enrollPrivKey.(*ecdsa.PrivateKey)

	return nil
}

func (node *nodeImpl) loadEnrollmentCertificate() error {
	node.log.Debug("Loading enrollment certificate...")

	cert, der, err := node.ks.loadCertX509AndDer(node.conf.getEnrollmentCertFilename())
	if err != nil {
		node.log.Error("Failed parsing enrollment certificate [%s].", err.Error())

		return err
	}
	node.enrollCert = cert

	// TODO: move this to retrieve
	pk := node.enrollCert.PublicKey.(*ecdsa.PublicKey)
	err = utils.VerifySignCapability(node.enrollPrivKey, pk)
	if err != nil {
		node.log.Error("Failed checking enrollment certificate against enrollment key [%s].", err.Error())

		return err
	}

	// Set node ID
	node.id = utils.Hash(der)
	node.log.Debug("Setting id to [%s].", utils.EncodeBase64(node.id))

	// Set eCertHash
	node.enrollCertHash = utils.Hash(der)
	node.log.Debug("Setting enrollCertHash to [%s].", utils.EncodeBase64(node.enrollCertHash))

	return nil
}

func (node *nodeImpl) loadEnrollmentID() error {
	node.log.Debug("Loading enrollment id at [%s]...", node.conf.getEnrollmentIDPath())

	enrollID, err := ioutil.ReadFile(node.conf.getEnrollmentIDPath())
	if err != nil {
		node.log.Error("Failed loading enrollment id [%s].", err.Error())

		return err
	}

	// Set enrollment ID
	node.enrollID = string(enrollID)
	node.log.Debug("Setting enrollment id to [%s].", node.enrollID)

	return nil
}

func (node *nodeImpl) loadEnrollmentChainKey() error {
	node.log.Debug("Loading enrollment chain key...")

	enrollChainKey, err := node.ks.loadKey(node.conf.getEnrollmentChainKeyFilename())
	if err != nil {
		node.log.Error("Failed loading enrollment chain key [%s].", err.Error())

		return err
	}
	node.enrollChainKey = enrollChainKey

	return nil
}

func (node *nodeImpl) loadECACertsChain() error {
	node.log.Debug("Loading ECA certificates chain...")

	pem, err := node.ks.loadCert(node.conf.getECACertsChainFilename())
	if err != nil {
		node.log.Error("Failed loading ECA certificates chain [%s].", err.Error())

		return err
	}

	ok := node.ecaCertPool.AppendCertsFromPEM(pem)
	if !ok {
		node.log.Error("Failed appending ECA certificates chain.")

		return errors.New("Failed appending ECA certificates chain.")
	}

	return nil
}

func (node *nodeImpl) getECAClient() (*grpc.ClientConn, obcca.ECAPClient, error) {
	node.log.Debug("Getting ECA client...")

	conn, err := node.getClientConn(node.conf.getECAPAddr(), node.conf.getECAServerName())
	if err != nil {
		node.log.Error("Failed getting client connection: [%s]", err)
	}

	client := obcca.NewECAPClient(conn)

	node.log.Debug("Getting ECA client...done")

	return conn, client, nil
}

func (node *nodeImpl) callECAReadCACertificate(ctx context.Context, opts ...grpc.CallOption) (*obcca.Cert, error) {
	// Get an ECA Client
	sock, ecaP, err := node.getECAClient()
	defer sock.Close()

	// Issue the request
	cert, err := ecaP.ReadCACertificate(ctx, &obcca.Empty{}, opts...)
	if err != nil {
		node.log.Error("Failed requesting read certificate [%s].", err.Error())

		return nil, err
	}

	return cert, nil
}

func (node *nodeImpl) callECAReadCertificate(ctx context.Context, in *obcca.ECertReadReq, opts ...grpc.CallOption) (*obcca.CertPair, error) {
	// Get an ECA Client
	sock, ecaP, err := node.getECAClient()
	defer sock.Close()

	// Issue the request
	resp, err := ecaP.ReadCertificatePair(ctx, in, opts...)
	if err != nil {
		node.log.Error("Failed requesting read certificate [%s].", err.Error())

		return nil, err
	}

	return resp, nil
}

func (node *nodeImpl) callECAReadCertificateByHash(ctx context.Context, in *obcca.Hash, opts ...grpc.CallOption) (*obcca.CertPair, error) {
	// Get an ECA Client
	sock, ecaP, err := node.getECAClient()
	defer sock.Close()

	// Issue the request
	resp, err := ecaP.ReadCertificateByHash(ctx, in, opts...)
	if err != nil {
		node.log.Error("Failed requesting read certificate [%s].", err.Error())

		return nil, err
	}

	return &obcca.CertPair{resp.Cert, nil}, nil
}

func (node *nodeImpl) getEnrollmentCertificateFromECA(id, pw string) (interface{}, []byte, []byte, error) {
	// Get a new ECA Client
	sock, ecaP, err := node.getECAClient()
	defer sock.Close()

	// Run the protocol
	signPriv, err := utils.NewECDSAKey()
	if err != nil {
		node.log.Error("Failed generating ECDSA key [%s].", err.Error())

		return nil, nil, nil, err
	}
	signPub, err := x509.MarshalPKIXPublicKey(&signPriv.PublicKey)
	if err != nil {
		node.log.Error("Failed mashalling ECDSA key [%s].", err.Error())

		return nil, nil, nil, err
	}

	encPriv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		node.log.Error("Failed generating RSA key [%s].", err.Error())

		return nil, nil, nil, err
	}
	encPub, err := x509.MarshalPKIXPublicKey(&encPriv.PublicKey)
	if err != nil {
		node.log.Error("Failed marshalling RSA key [%s].", err.Error())

		return nil, nil, nil, err
	}

	req := &obcca.ECertCreateReq{&protobuf.Timestamp{Seconds: time.Now().Unix(), Nanos: 0},
		&obcca.Identity{id},
		&obcca.Token{Tok: []byte(pw)},
		&obcca.PublicKey{obcca.CryptoType_ECDSA, signPub},
		&obcca.PublicKey{obcca.CryptoType_RSA, encPub},
		nil}

	resp, err := ecaP.CreateCertificatePair(context.Background(), req)
	if err != nil {
		node.log.Error("Failed invoking CreateCertficatePair [%s].", err.Error())

		return nil, nil, nil, err
	}

	out, err := rsa.DecryptPKCS1v15(rand.Reader, encPriv, resp.Tok.Tok)
	if err != nil {
		node.log.Error("Failed decrypting token [%s].", err.Error())

		return nil, nil, nil, err
	}

	req.Tok.Tok = out
	req.Sig = nil

	hash := utils.NewHash()
	raw, _ := proto.Marshal(req)
	hash.Write(raw)

	r, s, err := ecdsa.Sign(rand.Reader, signPriv, hash.Sum(nil))
	if err != nil {
		node.log.Error("Failed signing [%s].", err.Error())

		return nil, nil, nil, err
	}
	R, _ := r.MarshalText()
	S, _ := s.MarshalText()
	req.Sig = &obcca.Signature{obcca.CryptoType_ECDSA, R, S}

	resp, err = ecaP.CreateCertificatePair(context.Background(), req)
	if err != nil {
		node.log.Error("Failed invoking CreateCertificatePair [%s].", err.Error())

		return nil, nil, nil, err
	}

	// Verify responce

	// Verify cert for signing
	node.log.Debug("Enrollment certificate for signing [%s]", utils.EncodeBase64(utils.Hash(resp.Certs.Sign)))

	x509SignCert, err := utils.DERToX509Certificate(resp.Certs.Sign)
	if err != nil {
		node.log.Error("Failed parsing signing enrollment certificate for signing: [%s]", err)

		return nil, nil, nil, err
	}

	_, err = utils.GetCriticalExtension(x509SignCert, ECertSubjectRole)
	if err != nil {
		node.log.Error("Failed parsing ECertSubjectRole in enrollment certificate for signing: [%s]", err)

		return nil, nil, nil, err
	}

	err = utils.CheckCertAgainstSKAndRoot(x509SignCert, signPriv, node.ecaCertPool)
	if err != nil {
		node.log.Error("Failed checking signing enrollment certificate for signing: [%s]", err)

		return nil, nil, nil, err
	}

	// Verify cert for encrypting
	node.log.Debug("Enrollment certificate for encrypting [%s]", utils.EncodeBase64(utils.Hash(resp.Certs.Enc)))

	x509EncCert, err := utils.DERToX509Certificate(resp.Certs.Enc)
	if err != nil {
		node.log.Error("Failed parsing signing enrollment certificate for encrypting: [%s]", err)

		return nil, nil, nil, err
	}

	_, err = utils.GetCriticalExtension(x509EncCert, ECertSubjectRole)
	if err != nil {
		node.log.Error("Failed parsing ECertSubjectRole in enrollment certificate for encrypting: [%s]", err)

		return nil, nil, nil, err
	}

	err = utils.CheckCertAgainstSKAndRoot(x509EncCert, encPriv, node.ecaCertPool)
	if err != nil {
		node.log.Error("Failed checking signing enrollment certificate for encrypting: [%s]", err)

		return nil, nil, nil, err
	}

	// END

	return signPriv, resp.Certs.Sign, resp.Chain.Tok, nil
}

func (node *nodeImpl) getECACertificate() ([]byte, error) {
	responce, err := node.callECAReadCACertificate(context.Background())
	if err != nil {
		node.log.Error("Failed requesting ECA certificate [%s].", err.Error())

		return nil, err
	}

	return responce.Cert, nil
}
<commit_msg>client side aligned to the new eca protocol for getting an enrollment certificate<commit_after>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package crypto

import (
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/x509"
	obcca "github.com/openblockchain/obc-peer/obc-ca/protos"
	protobuf "google/protobuf"
	"time"

	"encoding/asn1"
	"errors"
	"github.com/golang/protobuf/proto"
	ecies "github.com/openblockchain/obc-peer/openchain/crypto/ecies/generic"
	"github.com/openblockchain/obc-peer/openchain/crypto/utils"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"io/ioutil"
)

var (
	// ECertSubjectRole is the ASN1 object identifier of the subject's role.
	ECertSubjectRole = asn1.ObjectIdentifier{2, 1, 3, 4, 5, 6, 7}
)

func (node *nodeImpl) retrieveECACertsChain(userID string) error {
	// Retrieve ECA certificate and verify it
	ecaCertRaw, err := node.getECACertificate()
	if err != nil {
		node.log.Error("Failed getting ECA certificate [%s].", err.Error())

		return err
	}
	node.log.Debug("ECA certificate [%s].", utils.EncodeBase64(ecaCertRaw))

	// TODO: Test ECA cert againt root CA
	// TODO: check responce.Cert against rootCA
	x509ECACert, err := utils.DERToX509Certificate(ecaCertRaw)
	if err != nil {
		node.log.Error("Failed parsing ECA certificate [%s].", err.Error())

		return err
	}

	// Prepare ecaCertPool
	node.ecaCertPool = x509.NewCertPool()
	node.ecaCertPool.AddCert(x509ECACert)

	// Store ECA cert
	node.log.Debug("Storing ECA certificate for [%s]...", userID)

	if err := node.ks.storeCert(node.conf.getECACertsChainFilename(), ecaCertRaw); err != nil {
		node.log.Error("Failed storing eca certificate [%s].", err.Error())
		return err
	}

	return nil
}

func (node *nodeImpl) retrieveEnrollmentData(enrollID, enrollPWD string) error {
	key, enrollCertRaw, enrollChainKey, err := node.getEnrollmentCertificateFromECA(enrollID, enrollPWD)
	if err != nil {
		node.log.Error("Failed getting enrollment certificate [id=%s]: [%s]", enrollID, err)

		return err
	}
	node.log.Debug("Enrollment certificate [%s].", utils.EncodeBase64(enrollCertRaw))

	node.log.Debug("Storing enrollment data for user [%s]...", enrollID)

	// Store enrollment id
	err = ioutil.WriteFile(node.conf.getEnrollmentIDPath(), []byte(enrollID), 0700)
	if err != nil {
		node.log.Error("Failed storing enrollment certificate [id=%s]: [%s]", enrollID, err)
		return err
	}

	// Store enrollment key
	if err := node.ks.storePrivateKey(node.conf.getEnrollmentKeyFilename(), key); err != nil {
		node.log.Error("Failed storing enrollment key [id=%s]: [%s]", enrollID, err)
		return err
	}

	// Store enrollment cert
	if err := node.ks.storeCert(node.conf.getEnrollmentCertFilename(), enrollCertRaw); err != nil {
		node.log.Error("Failed storing enrollment certificate [id=%s]: [%s]", enrollID, err)
		return err
	}

	// Store enrollment chain key
	if err := node.ks.storeKey(node.conf.getEnrollmentChainKeyFilename(), enrollChainKey); err != nil {
		node.log.Error("Failed storing enrollment chain key [id=%s]: [%s]", enrollID, err)
		return err
	}

	return nil
}

func (node *nodeImpl) loadEnrollmentKey() error {
	node.log.Debug("Loading enrollment key...")

	enrollPrivKey, err := node.ks.loadPrivateKey(node.conf.getEnrollmentKeyFilename())
	if err != nil {
		node.log.Error("Failed loading enrollment private key [%s].", err.Error())

		return err
	}

	node.enrollPrivKey = enrollPrivKey.(*ecdsa.PrivateKey)

	return nil
}

func (node *nodeImpl) loadEnrollmentCertificate() error {
	node.log.Debug("Loading enrollment certificate...")

	cert, der, err := node.ks.loadCertX509AndDer(node.conf.getEnrollmentCertFilename())
	if err != nil {
		node.log.Error("Failed parsing enrollment certificate [%s].", err.Error())

		return err
	}
	node.enrollCert = cert

	// TODO: move this to retrieve
	pk := node.enrollCert.PublicKey.(*ecdsa.PublicKey)
	err = utils.VerifySignCapability(node.enrollPrivKey, pk)
	if err != nil {
		node.log.Error("Failed checking enrollment certificate against enrollment key [%s].", err.Error())

		return err
	}

	// Set node ID
	node.id = utils.Hash(der)
	node.log.Debug("Setting id to [%s].", utils.EncodeBase64(node.id))

	// Set eCertHash
	node.enrollCertHash = utils.Hash(der)
	node.log.Debug("Setting enrollCertHash to [%s].", utils.EncodeBase64(node.enrollCertHash))

	return nil
}

func (node *nodeImpl) loadEnrollmentID() error {
	node.log.Debug("Loading enrollment id at [%s]...", node.conf.getEnrollmentIDPath())

	enrollID, err := ioutil.ReadFile(node.conf.getEnrollmentIDPath())
	if err != nil {
		node.log.Error("Failed loading enrollment id [%s].", err.Error())

		return err
	}

	// Set enrollment ID
	node.enrollID = string(enrollID)
	node.log.Debug("Setting enrollment id to [%s].", node.enrollID)

	return nil
}

func (node *nodeImpl) loadEnrollmentChainKey() error {
	node.log.Debug("Loading enrollment chain key...")

	enrollChainKey, err := node.ks.loadKey(node.conf.getEnrollmentChainKeyFilename())
	if err != nil {
		node.log.Error("Failed loading enrollment chain key [%s].", err.Error())

		return err
	}
	node.enrollChainKey = enrollChainKey

	return nil
}

func (node *nodeImpl) loadECACertsChain() error {
	node.log.Debug("Loading ECA certificates chain...")

	pem, err := node.ks.loadCert(node.conf.getECACertsChainFilename())
	if err != nil {
		node.log.Error("Failed loading ECA certificates chain [%s].", err.Error())

		return err
	}

	ok := node.ecaCertPool.AppendCertsFromPEM(pem)
	if !ok {
		node.log.Error("Failed appending ECA certificates chain.")

		return errors.New("Failed appending ECA certificates chain.")
	}

	return nil
}

func (node *nodeImpl) getECAClient() (*grpc.ClientConn, obcca.ECAPClient, error) {
	node.log.Debug("Getting ECA client...")

	conn, err := node.getClientConn(node.conf.getECAPAddr(), node.conf.getECAServerName())
	if err != nil {
		node.log.Error("Failed getting client connection: [%s]", err)
	}

	client := obcca.NewECAPClient(conn)

	node.log.Debug("Getting ECA client...done")

	return conn, client, nil
}

func (node *nodeImpl) callECAReadCACertificate(ctx context.Context, opts ...grpc.CallOption) (*obcca.Cert, error) {
	// Get an ECA Client
	sock, ecaP, err := node.getECAClient()
	defer sock.Close()

	// Issue the request
	cert, err := ecaP.ReadCACertificate(ctx, &obcca.Empty{}, opts...)
	if err != nil {
		node.log.Error("Failed requesting read certificate [%s].", err.Error())

		return nil, err
	}

	return cert, nil
}

func (node *nodeImpl) callECAReadCertificate(ctx context.Context, in *obcca.ECertReadReq, opts ...grpc.CallOption) (*obcca.CertPair, error) {
	// Get an ECA Client
	sock, ecaP, err := node.getECAClient()
	defer sock.Close()

	// Issue the request
	resp, err := ecaP.ReadCertificatePair(ctx, in, opts...)
	if err != nil {
		node.log.Error("Failed requesting read certificate [%s].", err.Error())

		return nil, err
	}

	return resp, nil
}

func (node *nodeImpl) callECAReadCertificateByHash(ctx context.Context, in *obcca.Hash, opts ...grpc.CallOption) (*obcca.CertPair, error) {
	// Get an ECA Client
	sock, ecaP, err := node.getECAClient()
	defer sock.Close()

	// Issue the request
	resp, err := ecaP.ReadCertificateByHash(ctx, in, opts...)
	if err != nil {
		node.log.Error("Failed requesting read certificate [%s].", err.Error())

		return nil, err
	}

	return &obcca.CertPair{resp.Cert, nil}, nil
}

func (node *nodeImpl) getEnrollmentCertificateFromECA(id, pw string) (interface{}, []byte, []byte, error) {
	// Get a new ECA Client
	sock, ecaP, err := node.getECAClient()
	defer sock.Close()

	// Run the protocol

	signPriv, err := utils.NewECDSAKey()
	if err != nil {
		node.log.Error("Failed generating ECDSA key [%s].", err.Error())

		return nil, nil, nil, err
	}
	signPub, err := x509.MarshalPKIXPublicKey(&signPriv.PublicKey)
	if err != nil {
		node.log.Error("Failed mashalling ECDSA key [%s].", err.Error())

		return nil, nil, nil, err
	}

	encPriv, err := utils.NewECDSAKey()
	if err != nil {
		node.log.Error("Failed generating Encryption key [%s].", err.Error())

		return nil, nil, nil, err
	}
	encPub, err := x509.MarshalPKIXPublicKey(&encPriv.PublicKey)
	if err != nil {
		node.log.Error("Failed marshalling Encryption key [%s].", err.Error())

		return nil, nil, nil, err
	}

	req := &obcca.ECertCreateReq{&protobuf.Timestamp{Seconds: time.Now().Unix(), Nanos: 0},
		&obcca.Identity{id},
		&obcca.Token{Tok: []byte(pw)},
		&obcca.PublicKey{obcca.CryptoType_ECDSA, signPub},
		&obcca.PublicKey{obcca.CryptoType_ECDSA, encPub},
		nil}

	resp, err := ecaP.CreateCertificatePair(context.Background(), req)
	if err != nil {
		node.log.Error("Failed invoking CreateCertficatePair [%s].", err.Error())

		return nil, nil, nil, err
	}

	//out, err := rsa.DecryptPKCS1v15(rand.Reader, encPriv, resp.Tok.Tok)
	spi := ecies.NewSPI()
	eciesKey, err := spi.NewPrivateKey(nil, encPriv)
	if err != nil {
		node.log.Error("Failed parsing decrypting key [%s].", err.Error())

		return nil, nil, nil, err
	}

	ecies, err := spi.NewAsymmetricCipherFromPublicKey(eciesKey)
	if err != nil {
		node.log.Error("Failed creating asymmetrinc cipher [%s].", err.Error())

		return nil, nil, nil, err
	}

	out, err := ecies.Process(resp.Tok.Tok)
	if err != nil {
		node.log.Error("Failed decrypting toke [%s].", err.Error())

		return nil, nil, nil, err
	}

	req.Tok.Tok = out
	req.Sig = nil

	hash := utils.NewHash()
	raw, _ := proto.Marshal(req)
	hash.Write(raw)

	r, s, err := ecdsa.Sign(rand.Reader, signPriv, hash.Sum(nil))
	if err != nil {
		node.log.Error("Failed signing [%s].", err.Error())

		return nil, nil, nil, err
	}
	R, _ := r.MarshalText()
	S, _ := s.MarshalText()
	req.Sig = &obcca.Signature{obcca.CryptoType_ECDSA, R, S}

	resp, err = ecaP.CreateCertificatePair(context.Background(), req)
	if err != nil {
		node.log.Error("Failed invoking CreateCertificatePair [%s].", err.Error())

		return nil, nil, nil, err
	}

	// Verify response

	// Verify cert for signing
	node.log.Debug("Enrollment certificate for signing [%s]", utils.EncodeBase64(utils.Hash(resp.Certs.Sign)))

	x509SignCert, err := utils.DERToX509Certificate(resp.Certs.Sign)
	if err != nil {
		node.log.Error("Failed parsing signing enrollment certificate for signing: [%s]", err)

		return nil, nil, nil, err
	}

	_, err = utils.GetCriticalExtension(x509SignCert, ECertSubjectRole)
	if err != nil {
		node.log.Error("Failed parsing ECertSubjectRole in enrollment certificate for signing: [%s]", err)

		return nil, nil, nil, err
	}

	err = utils.CheckCertAgainstSKAndRoot(x509SignCert, signPriv, node.ecaCertPool)
	if err != nil {
		node.log.Error("Failed checking signing enrollment certificate for signing: [%s]", err)

		return nil, nil, nil, err
	}

	// Verify cert for encrypting
	node.log.Debug("Enrollment certificate for encrypting [%s]", utils.EncodeBase64(utils.Hash(resp.Certs.Enc)))

	x509EncCert, err := utils.DERToX509Certificate(resp.Certs.Enc)
	if err != nil {
		node.log.Error("Failed parsing signing enrollment certificate for encrypting: [%s]", err)

		return nil, nil, nil, err
	}

	_, err = utils.GetCriticalExtension(x509EncCert, ECertSubjectRole)
	if err != nil {
		node.log.Error("Failed parsing ECertSubjectRole in enrollment certificate for encrypting: [%s]", err)

		return nil, nil, nil, err
	}

	err = utils.CheckCertAgainstSKAndRoot(x509EncCert, encPriv, node.ecaCertPool)
	if err != nil {
		node.log.Error("Failed checking signing enrollment certificate for encrypting: [%s]", err)

		return nil, nil, nil, err
	}

	// END

	return signPriv, resp.Certs.Sign, resp.Chain.Tok, nil
}

func (node *nodeImpl) getECACertificate() ([]byte, error) {
	responce, err := node.callECAReadCACertificate(context.Background())
	if err != nil {
		node.log.Error("Failed requesting ECA certificate [%s].", err.Error())

		return nil, err
	}

	return responce.Cert, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Ted Goddard. All rights reserved.
// Use of this source code is governed the Apache 2.0
// license that can be found in the LICENSE file.

package main

import "fmt"
import "net"
import "os"
import "bufio"
import "net/http"
import "websocket"
import "log"
import "fits"
import "encoding/json"

func main() {
	fmt.Println("Starting server")

    previousImagePath := ""
    currentImagePath := ""

    wsClientHTML :=
        "" +
            "" +
                "" +
            "" +
            "" +
            "
" +
                "" +
                "<svg id='marker' width='20' height='20' style='position: absolute; top: 0; left: 0;'>" +
                "    <rect x='0' y='0' width='20' height='20' stroke='green' stroke-width='4' fill='none' />" +
                "</svg>" +
            "
" +
            "<button style='position:fixed;bottom:0;left:0' onclick='guide()'>GUIDE</button>" +

            "" +
        ""

    http.HandleFunc("/phdremote/", func(w http.ResponseWriter, r *http.Request) {
        fmt.Fprintf(w, wsClientHTML)
    })

    //only one websocket allowed for now
    var wsConn *websocket.Conn
    var phdWrite *bufio.Writer
    phdDone := make(chan bool)

    GuideWatch := func (conn *net.Conn)  {
        connRead := bufio.NewReader(*conn)
        status := ""
        var err error
        for (err == nil)  {
            status, err = connRead.ReadString('\n')
            log.Print(status)
            var phdMessage map[string]interface{}
            err = json.Unmarshal([]byte(status), &phdMessage)
            if (nil == err)  {
                if (nil != phdMessage["jsonrpc"]) {
                    log.Print("jsonrpc contents", status)
                    switch result := phdMessage["result"].(type)  {
                        case map[string]interface{}:
                            previousImagePath = currentImagePath
                            currentImagePath = result["filename"].(string)
                            if ("" != previousImagePath)  {
                                os.Remove(previousImagePath)
                            }
                        case float64:
                            log.Print("float64 jsonrpc result")
                    }
                }
                fmt.Println(phdMessage["jsonrpc"])
            }

            if (nil != wsConn)  {
                log.Print("writing to WebSocket")
                (*wsConn).Write([]byte(status))
            }
        }
        phdDone <- true
    }

    SocketWatch := func()  {
        var err error
        for (err == nil)  {
            var msg = make([]byte, 512)
            var n int
            n, err = wsConn.Read(msg)
            fmt.Printf("WEBSOCKET Received: %s.\n", msg[:n])
            if (nil != phdWrite)  {
                fmt.Fprintf(phdWrite, string(msg))
                phdWrite.Flush()
            }
        }
    }

    EchoServer := func(newConn *websocket.Conn) {
        log.Print("EchoServer started")
        wsConn = newConn
        go SocketWatch ()
        echoDone := <-phdDone
        if (echoDone)  {
            log.Print("EchoServer done")
       }
    }

    log.Print("websocket.Handler")
    wsHandler := websocket.Handler(EchoServer)
	http.Handle("/echo/", wsHandler)

    conn, err := net.Dial("tcp", "localhost:4400")
    if (err == nil) {
        phdWrite = bufio.NewWriter(conn)
        go GuideWatch (&conn)
    } else {
        log.Print("Unable to connect to PHD")
    }

    http.HandleFunc("/phdremote/cam.png", func(w http.ResponseWriter, r *http.Request) {
log.Print("returning png image")
        if (nil != phdWrite)  {
            fmt.Fprintf(phdWrite, "{\"method\":\"save_image\",\"id\":123}\n")
            phdWrite.Flush()
        }
        w.Header().Set("Content-Type", "image/png")
        momentaryImagePath := currentImagePath
        if ("" == momentaryImagePath)  {
            momentaryImagePath = "RCA.fit"
        }
        fits.ConvertPNG(momentaryImagePath, w)
    })

    http.HandleFunc("/phdremote/cam.jpg", func(w http.ResponseWriter, r *http.Request) {
log.Print("returning jpg image")
        if (nil != phdWrite)  {
            fmt.Fprintf(phdWrite, "{\"method\":\"save_image\",\"id\":123}\n")
            phdWrite.Flush()
        }
        w.Header().Set("Content-Type", "image/jpeg")
        momentaryImagePath := currentImagePath
        if ("" == momentaryImagePath)  {
            momentaryImagePath = "RCA.fit"
        }
        fits.ConvertJPG(momentaryImagePath, w)
    })

    log.Print("http.ListenAndServe")
    log.Fatal(http.ListenAndServe(":8080", nil))

}

<commit_msg>yellow and solid guide indicators #1<commit_after>// Copyright 2014 Ted Goddard. All rights reserved.
// Use of this source code is governed the Apache 2.0
// license that can be found in the LICENSE file.

package main

import "fmt"
import "net"
import "os"
import "bufio"
import "net/http"
import "websocket"
import "log"
import "fits"
import "encoding/json"

func main() {
	fmt.Println("Starting server")

    previousImagePath := ""
    currentImagePath := ""

    wsClientHTML :=
        "" +
            "" +
                "" +
            "" +
            "" +
            "
" +
                "" +
                "<svg id='marker' width='20' height='20' style='position: absolute; top: 0; left: 0;'>" +
                "    <rect x='0' y='0' width='20' height='20' stroke='green' stroke-width='4' fill='none' />" +
                "</svg>" +
            "
" +
            "<button style='position:fixed;bottom:0;left:0' onclick='guide()'>GUIDE</button>" +

            "" +
        ""

    http.HandleFunc("/phdremote/", func(w http.ResponseWriter, r *http.Request) {
        fmt.Fprintf(w, wsClientHTML)
    })

    //only one websocket allowed for now
    var wsConn *websocket.Conn
    var phdWrite *bufio.Writer
    phdDone := make(chan bool)

    GuideWatch := func (conn *net.Conn)  {
        connRead := bufio.NewReader(*conn)
        status := ""
        var err error
        for (err == nil)  {
            status, err = connRead.ReadString('\n')
            log.Print(status)
            var phdMessage map[string]interface{}
            err = json.Unmarshal([]byte(status), &phdMessage)
            if (nil == err)  {
                if (nil != phdMessage["jsonrpc"]) {
                    log.Print("jsonrpc contents", status)
                    switch result := phdMessage["result"].(type)  {
                        case map[string]interface{}:
                            previousImagePath = currentImagePath
                            currentImagePath = result["filename"].(string)
                            if ("" != previousImagePath)  {
                                os.Remove(previousImagePath)
                            }
                        case float64:
                            log.Print("float64 jsonrpc result")
                    }
                }
                fmt.Println(phdMessage["jsonrpc"])
            }

            if (nil != wsConn)  {
                log.Print("writing to WebSocket")
                (*wsConn).Write([]byte(status))
            }
        }
        phdDone <- true
    }

    SocketWatch := func()  {
        var err error
        for (err == nil)  {
            var msg = make([]byte, 512)
            var n int
            n, err = wsConn.Read(msg)
            fmt.Printf("WEBSOCKET Received: %s.\n", msg[:n])
            if (nil != phdWrite)  {
                fmt.Fprintf(phdWrite, string(msg))
                phdWrite.Flush()
            }
        }
    }

    EchoServer := func(newConn *websocket.Conn) {
        log.Print("EchoServer started")
        wsConn = newConn
        go SocketWatch ()
        echoDone := <-phdDone
        if (echoDone)  {
            log.Print("EchoServer done")
       }
    }

    log.Print("websocket.Handler")
    wsHandler := websocket.Handler(EchoServer)
	http.Handle("/echo/", wsHandler)

    conn, err := net.Dial("tcp", "localhost:4400")
    if (err == nil) {
        phdWrite = bufio.NewWriter(conn)
        go GuideWatch (&conn)
    } else {
        log.Print("Unable to connect to PHD")
    }

    http.HandleFunc("/phdremote/cam.png", func(w http.ResponseWriter, r *http.Request) {
log.Print("returning png image")
        if (nil != phdWrite)  {
            fmt.Fprintf(phdWrite, "{\"method\":\"save_image\",\"id\":123}\n")
            phdWrite.Flush()
        }
        w.Header().Set("Content-Type", "image/png")
        momentaryImagePath := currentImagePath
        if ("" == momentaryImagePath)  {
            momentaryImagePath = "RCA.fit"
        }
        fits.ConvertPNG(momentaryImagePath, w)
    })

    http.HandleFunc("/phdremote/cam.jpg", func(w http.ResponseWriter, r *http.Request) {
log.Print("returning jpg image")
        if (nil != phdWrite)  {
            fmt.Fprintf(phdWrite, "{\"method\":\"save_image\",\"id\":123}\n")
            phdWrite.Flush()
        }
        w.Header().Set("Content-Type", "image/jpeg")
        momentaryImagePath := currentImagePath
        if ("" == momentaryImagePath)  {
            momentaryImagePath = "RCA.fit"
        }
        fits.ConvertJPG(momentaryImagePath, w)
    })

    log.Print("http.ListenAndServe")
    log.Fatal(http.ListenAndServe(":8080", nil))

}

<|endoftext|>


<commit_before>package main

import (
	"flag"
	"net"
	"netio"
	"os"
	"strings"
)

var (
	host    string
	netType string
	receive bool
)

func init() {
	flag.StringVar(&host, "h", "", "Host \"127.0.0.1:301234\"")
	flag.StringVar(&netType, "n", "tcp", "Net type \"tcp\", \"tcp4\" (IPv4-only), \"tcp6\" (IPv6-only), \"udp\", \"udp4\" (IPv4-only), \"udp6\" (IPv6-only), \"ip\", \"ip4\" (IPv4-only), \"ip6\" (IPv6-only), \"unix\", \"unixgram\" and \"unixpacket\"")
	flag.BoolVar(&receive, "r", false, "Set receive mode")
	flag.Parse()

	h, port, _ := net.SplitHostPort(host)
	if port == "" && h != "" {
		host += ":30123"
	}
	if host == "" && !receive {
		flag.Usage()
		os.Exit(1)
	}

	netType = strings.TrimSpace(strings.ToLower(netType))
}

func main() {
	if !receive {
		ww := netio.NewWriter(host, netType)
		err := ww.Connect()
		if err != nil {
			netio.Log("Error connect:", err)
			os.Exit(1)
		}
		_, err = ww.ReadFrom(os.Stdin)
		if err != nil {
			netio.Log("Error send:", err)
			os.Exit(1)
		}
		ww.Close()
	} else {
		rr := netio.NewReader(host, netType)
		err := rr.Connect()
		if err != nil {
			netio.Log("Error listen:", err)
			os.Exit(1)
		}
		_, err = rr.WriteTo(os.Stdout)
		if err != nil {
			netio.Log("Error recv:", err)
			os.Exit(1)
		}
		rr.Close()
	}
}
<commit_msg>fix empty host err<commit_after>package main

import (
	"flag"
	"net"
	"netio"
	"os"
	"strings"
)

var (
	host    string
	netType string
	receive bool
)

func init() {
	flag.StringVar(&host, "h", "", "Host \"127.0.0.1:301234\"")
	flag.StringVar(&netType, "n", "tcp", "Net type \"tcp\", \"tcp4\" (IPv4-only), \"tcp6\" (IPv6-only), \"udp\", \"udp4\" (IPv4-only), \"udp6\" (IPv6-only), \"ip\", \"ip4\" (IPv4-only), \"ip6\" (IPv6-only), \"unix\", \"unixgram\" and \"unixpacket\"")
	flag.BoolVar(&receive, "r", false, "Set receive mode")
	flag.Parse()

	if host == "" && !receive {
		flag.Usage()
		os.Exit(1)
	}

	h, port, err := net.SplitHostPort(host)
	if (port == "" && h != "") || err != nil {
		if host == "" || host[len(host)-1] != ':' {
			host += ":"
		}
		host += "30123"
	}

	netType = strings.TrimSpace(strings.ToLower(netType))
}

func main() {
	if !receive {
		ww := netio.NewWriter(host, netType)
		err := ww.Connect()
		if err != nil {
			netio.Log("Error connect:", err)
			os.Exit(1)
		}
		_, err = ww.ReadFrom(os.Stdin)
		if err != nil {
			netio.Log("Error send:", err)
			os.Exit(1)
		}
		ww.Close()
	} else {
		rr := netio.NewReader(host, netType)
		err := rr.Connect()
		if err != nil {
			netio.Log("Error listen:", err)
			os.Exit(1)
		}
		_, err = rr.WriteTo(os.Stdout)
		if err != nil {
			netio.Log("Error recv:", err)
			os.Exit(1)
		}
		rr.Close()
	}
}
<|endoftext|>


<commit_before>/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package cmd

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"v.io/x/lib/cmdline"
)

var (
	cmdSingle = &cmdline.Command{
		Name:   "single",
		Short:  "Generates an alluxio tarball",
		Long:   "Generates an alluxio tarball",
		Runner: cmdline.RunnerFunc(single),
	}

	hadoopDistributionFlag string
	targetFlag             string
	mvnArgsFlag            string

	webappDir = "core/server/common/src/main/webapp"
	webappWar = "assembly/webapp.war"
)

func init() {
	cmdSingle.Flags.StringVar(&hadoopDistributionFlag, "hadoop-distribution", "hadoop-2.2", "the hadoop distribution to build this Alluxio distribution tarball")
	cmdSingle.Flags.StringVar(&targetFlag, "target", fmt.Sprintf("alluxio-%v.tar.gz", versionMarker),
		fmt.Sprintf("an optional target name for the generated tarball. The default is alluxio-%v.tar.gz. The string %q will be substituted with the built version. "+
			`Note that trailing ".tar.gz" will be stripped to determine the name for the Root directory of the generated tarball`, versionMarker, versionMarker))
	cmdSingle.Flags.StringVar(&mvnArgsFlag, "mvn-args", "", `a comma-separated list of additional Maven arguments to build with, e.g. -mvn-args "-Pspark,-Dhadoop.version=2.2.0"`)
}

func single(_ *cmdline.Env, _ []string) error {
	if err := generateTarball(hadoopDistributionFlag); err != nil {
		return err
	}
	return nil
}

func replace(path, old, new string) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ReadFile() failed: %v\n", err)
		os.Exit(1)
	}
	data = bytes.Replace(data, []byte(old), []byte(new), -1)
	if err := ioutil.WriteFile(path, data, os.FileMode(0644)); err != nil {
		fmt.Fprintf(os.Stderr, "WriteFile() failed: %v\n", err)
		os.Exit(1)
	}
}

func mkdir(path string) {
	if err := os.MkdirAll(path, os.FileMode(0755)); err != nil {
		fmt.Fprintf(os.Stderr, "MkdirAll() failed: %v\n", err)
		os.Exit(1)
	}
}

func chdir(path string) {
	if err := os.Chdir(path); err != nil {
		fmt.Fprintf(os.Stderr, "Chdir() failed: %v\n", err)
		os.Exit(1)
	}
}

func symlink(oldname, newname string) {
	if err := os.Symlink(oldname, newname); err != nil {
		fmt.Fprintf(os.Stderr, "Symlink(%v, %v) failed: %v\n", oldname, newname, err)
		os.Exit(1)
	}
}

func getCommonMvnArgs(hadoopVersion version) []string {
	args := []string{"clean", "install", "-DskipTests", "-Dfindbugs.skip", "-Dmaven.javadoc.skip", "-Dcheckstyle.skip", "-Pmesos"}
	if mvnArgsFlag != "" {
		for _, arg := range strings.Split(mvnArgsFlag, ",") {
			args = append(args, arg)
		}
	}

	args = append(args, fmt.Sprintf("-Dhadoop.version=%v", hadoopVersion), fmt.Sprintf("-P%v", hadoopVersion.hadoopProfile()))
	if includeYarnIntegration(hadoopVersion) {
		args = append(args, "-Pyarn")
	}
	if hadoopVersion.major == 1 {
		// checker requires hadoop 2+ to compile.
		args = append(args, "-Dchecker.hadoop.version=2.2.0")
	}
	return args
}

func includeYarnIntegration(hadoopVersion version) bool {
	return hadoopVersion.major >= 2 && hadoopVersion.minor >= 4
}

func getVersion() (string, error) {
	versionLine := run("grepping for the version", "grep", "-m1", "<version>", "pom.xml")
	re := regexp.MustCompile(".*<version>(.*)</version>.*")
	match := re.FindStringSubmatch(versionLine)
	if len(match) < 2 {
		return "", errors.New("failed to find version")
	}
	return match[1], nil
}

func addAdditionalFiles(srcPath, dstPath string, hadoopVersion version, version string) {
	chdir(srcPath)
	pathsToCopy := []string{
		"bin/alluxio",
		"bin/alluxio-masters.sh",
		"bin/alluxio-mount.sh",
		"bin/alluxio-start.sh",
		"bin/alluxio-stop.sh",
		"bin/alluxio-workers.sh",
		fmt.Sprintf("client/alluxio-%v-client.jar", version),
		"conf/alluxio-env.sh.template",
		"conf/alluxio-site.properties.template",
		"conf/core-site.xml.template",
		"conf/log4j.properties",
		"conf/masters",
		"conf/metrics.properties.template",
		"conf/workers",
		"integration/docker/Dockerfile",
		"integration/docker/entrypoint.sh",
		"integration/docker/bin/alluxio-master.sh",
		"integration/docker/bin/alluxio-proxy.sh",
		"integration/docker/bin/alluxio-worker.sh",
		"integration/docker/conf/alluxio-site.properties.template",
		"integration/docker/conf/alluxio-env.sh.template",
		"integration/fuse/bin/alluxio-fuse",
		"integration/kubernetes/alluxio-journal-volume.yaml.template",
		"integration/kubernetes/alluxio-master.yaml.template",
		"integration/kubernetes/alluxio-worker.yaml.template",
		"integration/kubernetes/conf/alluxio.properties.template",
		"integration/mesos/bin/alluxio-env-mesos.sh",
		"integration/mesos/bin/alluxio-mesos-start.sh",
		"integration/mesos/bin/alluxio-master-mesos.sh",
		"integration/mesos/bin/alluxio-mesos-stop.sh",
		"integration/mesos/bin/alluxio-worker-mesos.sh",
		"integration/mesos/bin/common.sh",
		fmt.Sprintf("lib/alluxio-underfs-gcs-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-hdfs-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-local-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-oss-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-s3a-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-swift-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-wasb-%v.jar", version),
		"libexec/alluxio-config.sh",
	}
	if includeYarnIntegration(hadoopVersion) {
		pathsToCopy = append(pathsToCopy, []string{
			"integration/yarn/bin/alluxio-application-master.sh",
			"integration/yarn/bin/alluxio-master-yarn.sh",
			"integration/yarn/bin/alluxio-worker-yarn.sh",
			"integration/yarn/bin/alluxio-yarn.sh",
			"integration/yarn/bin/alluxio-yarn-setup.sh",
			"integration/yarn/bin/common.sh",
		}...)
	}
	for _, path := range pathsToCopy {
		mkdir(filepath.Join(dstPath, filepath.Dir(path)))
		run(fmt.Sprintf("adding %v", path), "cp", path, filepath.Join(dstPath, path))
	}

	// Create empty directories for default UFS and Docker integration.
	mkdir(filepath.Join(dstPath, "underFSStorage"))
	mkdir(filepath.Join(dstPath, "integration/docker/conf"))

	// Add links for previous jar locations for backwards compatibility
	for _, jar := range []string{"client", "server"} {
		oldLocation := filepath.Join(dstPath, fmt.Sprintf("assembly/%v/target/alluxio-assembly-%v-%v-jar-with-dependencies.jar", jar, jar, version))
		mkdir(filepath.Dir(oldLocation))
		symlink(fmt.Sprintf("../../alluxio-%v-%v.jar", jar, version), oldLocation)
	}
	mkdir(filepath.Join(dstPath, "assembly/server/target"))
}

func generateTarball(hadoopDistribution string) error {
	hadoopVersion := hadoopDistributions[hadoopDistribution]
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}

	srcPath, err := ioutil.TempDir("", "alluxio")
	if err != nil {
		return fmt.Errorf("Failed to create temp directory: %v", err)
	}

	_, file, _, ok := runtime.Caller(0)
	if !ok {
		return errors.New("Failed to determine file of the go script")
	}
	// Relative path from this class to the root directory.
	repoPath := filepath.Join(filepath.Dir(file), "../../../../../../")
	run(fmt.Sprintf("copying source from %v to %v", repoPath, srcPath), "cp", "-R", repoPath+"/.", srcPath)

	chdir(srcPath)
	run("running git clean -fdx", "git", "clean", "-fdx")

	version, err := getVersion()
	if err != nil {
		return err
	}

	// Update the web app location.
	replace("core/common/src/main/java/alluxio/PropertyKey.java", webappDir, webappWar)
	// Update the assembly jar paths.
	replace("libexec/alluxio-config.sh", "assembly/client/target/alluxio-assembly-client-${VERSION}-jar-with-dependencies.jar", "assembly/alluxio-client-${VERSION}.jar")
	replace("libexec/alluxio-config.sh", "assembly/server/target/alluxio-assembly-server-${VERSION}-jar-with-dependencies.jar", "assembly/alluxio-server-${VERSION}.jar")
	// Update the FUSE jar path
	replace("integration/fuse/bin/alluxio-fuse", "target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar", "alluxio-fuse-${VERSION}.jar")

	mvnArgs := getCommonMvnArgs(hadoopVersion)
	run("compiling repo", "mvn", mvnArgs...)

	tarball := strings.Replace(targetFlag, versionMarker, version, 1)
	dstDir := strings.TrimSuffix(filepath.Base(tarball), ".tar.gz")
	dstPath := filepath.Join(cwd, dstDir)
	run(fmt.Sprintf("removing any existing %v", dstPath), "rm", "-rf", dstPath)
	fmt.Printf("Creating %s:\n", tarball)

	// Create the directory for the server jar.
	mkdir(filepath.Join(dstPath, "assembly"))
	// Create directories for the client jar.
	mkdir(filepath.Join(dstPath, "client"))
	mkdir(filepath.Join(dstPath, "logs"))
	// Create directories for the fuse connector
	mkdir(filepath.Join(dstPath, "integration", "fuse"))

	run("adding Alluxio client assembly jar", "mv", fmt.Sprintf("assembly/client/target/alluxio-assembly-client-%v-jar-with-dependencies.jar", version), filepath.Join(dstPath, "assembly", fmt.Sprintf("alluxio-client-%v.jar", version)))
	run("adding Alluxio server assembly jar", "mv", fmt.Sprintf("assembly/server/target/alluxio-assembly-server-%v-jar-with-dependencies.jar", version), filepath.Join(dstPath, "assembly", fmt.Sprintf("alluxio-server-%v.jar", version)))
	run("adding Alluxio FUSE jar", "mv", fmt.Sprintf("integration/fuse/target/alluxio-integration-fuse-%v-jar-with-dependencies.jar", version), filepath.Join(dstPath, "integration", "fuse", fmt.Sprintf("alluxio-fuse-%v.jar", version)))
	// Condense the webapp into a single .war file.
	run("jarring up webapp", "jar", "-cf", filepath.Join(dstPath, webappWar), "-C", webappDir, ".")

	if includeYarnIntegration(hadoopVersion) {
		// Update the YARN jar path
		replace("integration/yarn/bin/alluxio-yarn.sh", "target/alluxio-integration-yarn-${VERSION}-jar-with-dependencies.jar", "alluxio-yarn-${VERSION}.jar")
		// Create directories for the yarn integration
		mkdir(filepath.Join(dstPath, "integration", "yarn"))
		run("adding Alluxio YARN jar", "mv", fmt.Sprintf("integration/yarn/target/alluxio-integration-yarn-%v-jar-with-dependencies.jar", version), filepath.Join(dstPath, "integration", "yarn", fmt.Sprintf("alluxio-yarn-%v.jar", version)))
	}

	addAdditionalFiles(srcPath, dstPath, hadoopVersion, version)

	chdir(cwd)
	run("creating the distribution tarball", "tar", "-czvf", tarball, dstDir)
	run("removing the temporary repositories", "rm", "-rf", srcPath, dstPath)

	return nil
}
<commit_msg>Add checker to tarball generation (#7428)<commit_after>/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package cmd

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"v.io/x/lib/cmdline"
)

var (
	cmdSingle = &cmdline.Command{
		Name:   "single",
		Short:  "Generates an alluxio tarball",
		Long:   "Generates an alluxio tarball",
		Runner: cmdline.RunnerFunc(single),
	}

	hadoopDistributionFlag string
	targetFlag             string
	mvnArgsFlag            string

	webappDir = "core/server/common/src/main/webapp"
	webappWar = "assembly/webapp.war"
)

func init() {
	cmdSingle.Flags.StringVar(&hadoopDistributionFlag, "hadoop-distribution", "hadoop-2.2", "the hadoop distribution to build this Alluxio distribution tarball")
	cmdSingle.Flags.StringVar(&targetFlag, "target", fmt.Sprintf("alluxio-%v.tar.gz", versionMarker),
		fmt.Sprintf("an optional target name for the generated tarball. The default is alluxio-%v.tar.gz. The string %q will be substituted with the built version. "+
			`Note that trailing ".tar.gz" will be stripped to determine the name for the Root directory of the generated tarball`, versionMarker, versionMarker))
	cmdSingle.Flags.StringVar(&mvnArgsFlag, "mvn-args", "", `a comma-separated list of additional Maven arguments to build with, e.g. -mvn-args "-Pspark,-Dhadoop.version=2.2.0"`)
}

func single(_ *cmdline.Env, _ []string) error {
	if err := generateTarball(hadoopDistributionFlag); err != nil {
		return err
	}
	return nil
}

func replace(path, old, new string) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ReadFile() failed: %v\n", err)
		os.Exit(1)
	}
	data = bytes.Replace(data, []byte(old), []byte(new), -1)
	if err := ioutil.WriteFile(path, data, os.FileMode(0644)); err != nil {
		fmt.Fprintf(os.Stderr, "WriteFile() failed: %v\n", err)
		os.Exit(1)
	}
}

func mkdir(path string) {
	if err := os.MkdirAll(path, os.FileMode(0755)); err != nil {
		fmt.Fprintf(os.Stderr, "MkdirAll() failed: %v\n", err)
		os.Exit(1)
	}
}

func chdir(path string) {
	if err := os.Chdir(path); err != nil {
		fmt.Fprintf(os.Stderr, "Chdir() failed: %v\n", err)
		os.Exit(1)
	}
}

func symlink(oldname, newname string) {
	if err := os.Symlink(oldname, newname); err != nil {
		fmt.Fprintf(os.Stderr, "Symlink(%v, %v) failed: %v\n", oldname, newname, err)
		os.Exit(1)
	}
}

func getCommonMvnArgs(hadoopVersion version) []string {
	args := []string{"clean", "install", "-DskipTests", "-Dfindbugs.skip", "-Dmaven.javadoc.skip", "-Dcheckstyle.skip", "-Pmesos"}
	if mvnArgsFlag != "" {
		for _, arg := range strings.Split(mvnArgsFlag, ",") {
			args = append(args, arg)
		}
	}

	args = append(args, fmt.Sprintf("-Dhadoop.version=%v", hadoopVersion), fmt.Sprintf("-P%v", hadoopVersion.hadoopProfile()))
	if includeYarnIntegration(hadoopVersion) {
		args = append(args, "-Pyarn")
	}
	if hadoopVersion.major == 1 {
		// checker requires hadoop 2+ to compile.
		args = append(args, "-Dchecker.hadoop.version=2.2.0")
	}
	return args
}

func includeYarnIntegration(hadoopVersion version) bool {
	return hadoopVersion.major >= 2 && hadoopVersion.minor >= 4
}

func getVersion() (string, error) {
	versionLine := run("grepping for the version", "grep", "-m1", "<version>", "pom.xml")
	re := regexp.MustCompile(".*<version>(.*)</version>.*")
	match := re.FindStringSubmatch(versionLine)
	if len(match) < 2 {
		return "", errors.New("failed to find version")
	}
	return match[1], nil
}

func addAdditionalFiles(srcPath, dstPath string, hadoopVersion version, version string) {
	chdir(srcPath)
	pathsToCopy := []string{
		"bin/alluxio",
		"bin/alluxio-masters.sh",
		"bin/alluxio-mount.sh",
		"bin/alluxio-start.sh",
		"bin/alluxio-stop.sh",
		"bin/alluxio-workers.sh",
		fmt.Sprintf("client/alluxio-%v-client.jar", version),
		"conf/alluxio-env.sh.template",
		"conf/alluxio-site.properties.template",
		"conf/core-site.xml.template",
		"conf/log4j.properties",
		"conf/masters",
		"conf/metrics.properties.template",
		"conf/workers",
		"integration/checker/bin/alluxio-checker.sh",
		"integration/checker/bin/hive-checker.sh",
		"integration/checker/bin/mapreduce-checker.sh",
		"integration/checker/bin/spark-checker.sh",
		"integration/docker/Dockerfile",
		"integration/docker/entrypoint.sh",
		"integration/docker/bin/alluxio-master.sh",
		"integration/docker/bin/alluxio-proxy.sh",
		"integration/docker/bin/alluxio-worker.sh",
		"integration/docker/conf/alluxio-site.properties.template",
		"integration/docker/conf/alluxio-env.sh.template",
		"integration/fuse/bin/alluxio-fuse",
		"integration/kubernetes/alluxio-journal-volume.yaml.template",
		"integration/kubernetes/alluxio-master.yaml.template",
		"integration/kubernetes/alluxio-worker.yaml.template",
		"integration/kubernetes/conf/alluxio.properties.template",
		"integration/mesos/bin/alluxio-env-mesos.sh",
		"integration/mesos/bin/alluxio-mesos-start.sh",
		"integration/mesos/bin/alluxio-master-mesos.sh",
		"integration/mesos/bin/alluxio-mesos-stop.sh",
		"integration/mesos/bin/alluxio-worker-mesos.sh",
		"integration/mesos/bin/common.sh",
		fmt.Sprintf("lib/alluxio-underfs-gcs-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-hdfs-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-local-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-oss-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-s3a-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-swift-%v.jar", version),
		fmt.Sprintf("lib/alluxio-underfs-wasb-%v.jar", version),
		"libexec/alluxio-config.sh",
	}
	if includeYarnIntegration(hadoopVersion) {
		pathsToCopy = append(pathsToCopy, []string{
			"integration/yarn/bin/alluxio-application-master.sh",
			"integration/yarn/bin/alluxio-master-yarn.sh",
			"integration/yarn/bin/alluxio-worker-yarn.sh",
			"integration/yarn/bin/alluxio-yarn.sh",
			"integration/yarn/bin/alluxio-yarn-setup.sh",
			"integration/yarn/bin/common.sh",
		}...)
	}
	for _, path := range pathsToCopy {
		mkdir(filepath.Join(dstPath, filepath.Dir(path)))
		run(fmt.Sprintf("adding %v", path), "cp", path, filepath.Join(dstPath, path))
	}

	// Create empty directories for default UFS and Docker integration.
	mkdir(filepath.Join(dstPath, "underFSStorage"))
	mkdir(filepath.Join(dstPath, "integration/docker/conf"))

	// Add links for previous jar locations for backwards compatibility
	for _, jar := range []string{"client", "server"} {
		oldLocation := filepath.Join(dstPath, fmt.Sprintf("assembly/%v/target/alluxio-assembly-%v-%v-jar-with-dependencies.jar", jar, jar, version))
		mkdir(filepath.Dir(oldLocation))
		symlink(fmt.Sprintf("../../alluxio-%v-%v.jar", jar, version), oldLocation)
	}
	mkdir(filepath.Join(dstPath, "assembly/server/target"))
}

func generateTarball(hadoopDistribution string) error {
	hadoopVersion := hadoopDistributions[hadoopDistribution]
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}

	srcPath, err := ioutil.TempDir("", "alluxio")
	if err != nil {
		return fmt.Errorf("Failed to create temp directory: %v", err)
	}

	_, file, _, ok := runtime.Caller(0)
	if !ok {
		return errors.New("Failed to determine file of the go script")
	}
	// Relative path from this class to the root directory.
	repoPath := filepath.Join(filepath.Dir(file), "../../../../../../")
	run(fmt.Sprintf("copying source from %v to %v", repoPath, srcPath), "cp", "-R", repoPath+"/.", srcPath)

	chdir(srcPath)
	run("running git clean -fdx", "git", "clean", "-fdx")

	version, err := getVersion()
	if err != nil {
		return err
	}

	// Update the web app location.
	replace("core/common/src/main/java/alluxio/PropertyKey.java", webappDir, webappWar)
	// Update the assembly jar paths.
	replace("libexec/alluxio-config.sh", "assembly/client/target/alluxio-assembly-client-${VERSION}-jar-with-dependencies.jar", "assembly/alluxio-client-${VERSION}.jar")
	replace("libexec/alluxio-config.sh", "assembly/server/target/alluxio-assembly-server-${VERSION}-jar-with-dependencies.jar", "assembly/alluxio-server-${VERSION}.jar")
	// Update the FUSE jar path
	replace("integration/fuse/bin/alluxio-fuse", "target/alluxio-integration-fuse-${VERSION}-jar-with-dependencies.jar", "alluxio-fuse-${VERSION}.jar")
	// Update the checker jar paths
	for _, file := range []string{"bin/hive-checker.sh", "bin/mapreduce-checker.sh", "bin/spark-checker.sh"} {
		replace(filepath.Join("integration/checker", file), "target/alluxio-checker-${VERSION}-jar-with-dependencies.jar", "alluxio-checker-${VERSION}.jar")
	}

	mvnArgs := getCommonMvnArgs(hadoopVersion)
	run("compiling repo", "mvn", mvnArgs...)

	tarball := strings.Replace(targetFlag, versionMarker, version, 1)
	dstDir := strings.TrimSuffix(filepath.Base(tarball), ".tar.gz")
	dstPath := filepath.Join(cwd, dstDir)
	run(fmt.Sprintf("removing any existing %v", dstPath), "rm", "-rf", dstPath)
	fmt.Printf("Creating %s:\n", tarball)

	for _, dir := range []string{
		"assembly", "client", "logs", "integration/fuse", "integration/checker",
	} {
		mkdir(filepath.Join(dstPath, dir))
	}

	run("adding Alluxio client assembly jar", "mv", fmt.Sprintf("assembly/client/target/alluxio-assembly-client-%v-jar-with-dependencies.jar", version), filepath.Join(dstPath, "assembly", fmt.Sprintf("alluxio-client-%v.jar", version)))
	run("adding Alluxio server assembly jar", "mv", fmt.Sprintf("assembly/server/target/alluxio-assembly-server-%v-jar-with-dependencies.jar", version), filepath.Join(dstPath, "assembly", fmt.Sprintf("alluxio-server-%v.jar", version)))
	run("adding Alluxio FUSE jar", "mv", fmt.Sprintf("integration/fuse/target/alluxio-integration-fuse-%v-jar-with-dependencies.jar", version), filepath.Join(dstPath, "integration", "fuse", fmt.Sprintf("alluxio-fuse-%v.jar", version)))
	run("adding Alluxio checker jar", "mv", fmt.Sprintf("integration/checker/target/alluxio-checker-%v-jar-with-dependencies.jar", version), filepath.Join(dstPath, "integration", "checker", fmt.Sprintf("alluxio-checker-%v.jar", version)))
	// Condense the webapp into a single .war file.
	run("jarring up webapp", "jar", "-cf", filepath.Join(dstPath, webappWar), "-C", webappDir, ".")

	if includeYarnIntegration(hadoopVersion) {
		// Update the YARN jar path
		replace("integration/yarn/bin/alluxio-yarn.sh", "target/alluxio-integration-yarn-${VERSION}-jar-with-dependencies.jar", "alluxio-yarn-${VERSION}.jar")
		// Create directories for the yarn integration
		mkdir(filepath.Join(dstPath, "integration", "yarn"))
		run("adding Alluxio YARN jar", "mv", fmt.Sprintf("integration/yarn/target/alluxio-integration-yarn-%v-jar-with-dependencies.jar", version), filepath.Join(dstPath, "integration", "yarn", fmt.Sprintf("alluxio-yarn-%v.jar", version)))
	}

	addAdditionalFiles(srcPath, dstPath, hadoopVersion, version)

	chdir(cwd)
	run("creating the distribution tarball", "tar", "-czvf", tarball, dstDir)
	run("removing the temporary repositories", "rm", "-rf", srcPath, dstPath)

	return nil
}
<|endoftext|>


<commit_before>//
// util.go -- various utilities
//

package srnd

import (
	"crypto/sha1"
	"crypto/sha512"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"github.com/majestrate/nacl"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

func DelFile(fname string) {
	if CheckFile(fname) {
		os.Remove(fname)
	}
}

func CheckFile(fname string) bool {
	if _, err := os.Stat(fname); os.IsNotExist(err) {
		return false
	}
	return true
}

func IsDir(dirname string) bool {
	stat, err := os.Stat(dirname)
	if err != nil {
		log.Fatal(err)
	}
	return stat.IsDir()
}

// ensure a directory exists
func EnsureDir(dirname string) {
	stat, err := os.Stat(dirname)
	if os.IsNotExist(err) {
		os.Mkdir(dirname, 0755)
	} else if !stat.IsDir() {
		os.Remove(dirname)
		os.Mkdir(dirname, 0755)
	}
}

// TODO make this work better
func ValidMessageID(id string) bool {
	id_len := len(id)

	if id_len < 5 {
		return false
	}

	at_idx := strings.Index(id, "@")
	if at_idx < 3 {
		return false
	}

	for idx, c := range id {
		if idx == 0 {
			if c == '<' {
				continue
			}
		} else if idx == id_len-1 {
			if c == '>' {
				continue
			}
		} else {
			if idx == at_idx {
				continue
			}
			if c >= 'a' && c <= 'z' {
				continue
			}
			if c >= 'A' && c <= 'Z' {
				continue
			}
			if c >= '0' && c <= '9' {
				continue
			}
			if c == '.' {
				continue
			}
			if c == '$' {
				continue
			}
		}
		log.Printf("bad message ID: len=%d %s , invalid char at %d: %c", id_len, id, idx, c)
		return false
	}
	return true
}

// message id hash
func HashMessageID(msgid string) string {
	return fmt.Sprintf("%x", sha1.Sum([]byte(msgid)))
}

// short message id hash
func ShortHashMessageID(msgid string) string {
	return strings.ToLower(HashMessageID(msgid)[:18])
}

// will this message id produce quads?
func MessageIDWillDoQuads(msgid string) bool {
	h := HashMessageID(msgid)
	return h[0] == h[1] && h[1] == h[2] && h[2] == h[3]
}

// will this message id produce trips?
func MessageIDWillDoTrips(msgid string) bool {
	h := HashMessageID(msgid)
	return h[0] == h[1] && h[1] == h[2]
}

// will this message id produce dubs?
func MessageIDWillDoDubs(msgid string) bool {
	h := HashMessageID(msgid)
	return h[0] == h[1]
}

// shorter message id hash
func ShorterHashMessageID(msgid string) string {
	return strings.ToLower(HashMessageID(msgid)[:10])
}

type lineWriter struct {
	io.Writer
	wr    io.Writer
	delim []byte
}

func NewLineWriter(wr io.Writer, delim string) io.Writer {
	return lineWriter{wr, wr, []byte(delim)}
}

func (self lineWriter) Write(data []byte) (n int, err error) {
	n, err = self.wr.Write(data)
	self.wr.Write(self.delim)
	return n, err
}

func OpenFileWriter(fname string) (io.WriteCloser, error) {
	return os.Create(fname)
}

// make a random string
func randStr(length int) string {
	return hex.EncodeToString(nacl.RandBytes(length))[length:]
}

// time for right now as int64
func timeNow() int64 {
	return time.Now().Unix()
}

// sanitize data for nntp
func nntpSanitize(data string) string {
	parts := strings.Split(data, "\n.\n")
	return parts[0]
}

type int64Sorter []int64

func (self int64Sorter) Len() int {
	return len(self)
}

func (self int64Sorter) Less(i, j int) bool {
	return self[i] < self[j]
}

func (self int64Sorter) Swap(i, j int) {
	tmp := self[j]
	self[j] = self[i]
	self[i] = tmp
}

// obtain the "real" ip address
func getRealIP(name string) string {
	if len(name) > 0 {
		ip, err := net.ResolveIPAddr("ip", name)
		if err == nil {
			if ip.IP.IsGlobalUnicast() {
				return ip.IP.String()
			}
		}
	}
	return ""
}

// check that we have permission to access this
// fatal on fail
func checkPerms(fname string) {
	fstat, err := os.Stat(fname)
	if err != nil {
		log.Fatalf("Cannot access %s, %s", fname, err)
	}
	// check if we can access this dir
	if fstat.IsDir() {
		tmpfname := filepath.Join(fname, ".test")
		f, err := os.Create(tmpfname)
		if err != nil {
			log.Fatalf("No Write access in %s, %s", fname, err)
		}
		err = f.Close()
		if err != nil {
			log.Fatalf("failed to close test file %s !? %s", tmpfname, err)
		}
		err = os.Remove(tmpfname)
		if err != nil {
			log.Fatalf("failed to remove test file %s, %s", tmpfname, err)
		}
	} else {
		// this isn't a dir, treat it like a regular file
		f, err := os.Open(fname)
		if err != nil {
			log.Fatalf("cannot read file %s, %s", fname, err)
		}
		f.Close()
	}
}

// number of bytes to use in otp
func encAddrBytes() int {
	return 64
}

// length of an encrypted clearnet address
func encAddrLen() int {
	return 88
}

// length of an i2p dest hash
func i2pDestHashLen() int {
	return 44
}

// given an address
// generate a new encryption key for it
// return the encryption key and the encrypted address
func newAddrEnc(addr string) (string, string) {
	key_bytes := nacl.RandBytes(encAddrBytes())
	key := base64.StdEncoding.EncodeToString(key_bytes)
	return key, encAddr(addr, key)
}

// xor address with a one time pad
// if the address isn't long enough it's padded with spaces
func encAddr(addr, key string) string {
	key_bytes, err := base64.StdEncoding.DecodeString(key)

	if err != nil {
		log.Println("encAddr() key base64 decode", err)
		return ""
	}

	if len(addr) > len(key_bytes) {
		log.Println("encAddr() len(addr) > len(key_bytes)")
		return ""
	}

	// pad with spaces
	for len(addr) < len(key_bytes) {
		addr += " "
	}

	addr_bytes := []byte(addr)
	res_bytes := make([]byte, len(addr_bytes))
	for idx, b := range key_bytes {
		res_bytes[idx] = addr_bytes[idx] ^ b
	}

	return base64.StdEncoding.EncodeToString(res_bytes)
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// decrypt an address
// strips any whitespaces
func decAddr(encaddr, key string) string {
	encaddr_bytes, err := base64.StdEncoding.DecodeString(encaddr)
	if err != nil {
		log.Println("decAddr() encaddr base64 decode", err)
		return ""
	}
	if len(encaddr_bytes) != len(key) {
		log.Println("decAddr() len(encaddr_bytes) != len(key)")
		return ""
	}
	key_bytes, err := base64.StdEncoding.DecodeString(key)
	if err != nil {
		log.Println("decAddr() key base64 decode", err)
	}
	res_bytes := make([]byte, len(key))
	for idx, b := range key_bytes {
		res_bytes[idx] = encaddr_bytes[idx] ^ b
	}
	res := string(res_bytes)
	return strings.Trim(res, " ")
}

func newsgroupValidFormat(newsgroup string) bool {
	// too long newsgroup
	if len(newsgroup) > 128 {
		return false
	}
	for _, ch := range newsgroup {
		if ch >= 'a' && ch <= 'z' {
			continue
		}
		if ch >= '0' && ch <= '9' {
			continue
		}
		if ch >= 'A' && ch <= 'Z' {
			continue
		}
		if ch == '.' {
			continue
		}
		return false
	}
	return true
}

// generate a new signing keypair
// public, secret
func newSignKeypair() (string, string) {
	kp := nacl.GenSignKeypair()
	defer kp.Free()
	pk := kp.Public()
	sk := kp.Seed()
	return hex.EncodeToString(pk), hex.EncodeToString(sk)
}

// make a utf-8 tripcode
func makeTripcode(pk string) string {
	data, err := hex.DecodeString(pk)
	if err == nil {
		tripcode := ""
		//  here is the python code this is based off of
		//  i do something slightly different but this is the base
		//
		//  for x in range(0, length / 2):
		//    pub_short += '&#%i;' % (9600 + int(full_pubkey_hex[x*2:x*2+2], 16))
		//  length -= length / 2
		//  for x in range(0, length):
		//    pub_short += '&#%i;' % (9600 + int(full_pubkey_hex[-(length*2):][x*2:x*2+2], 16))
		//
		for _, c := range data {
			ch := 9600
			ch += int(c)
			tripcode += fmt.Sprintf("&#%04d;", ch)
		}
		return tripcode
	}
	return "[invalid]"
}

// generate a new message id with base name
func genMessageID(name string) string {
	return fmt.Sprintf("<%s%d@%s>", randStr(5), timeNow(), name)
}

// time now as a string timestamp
func timeNowStr() string {
	return time.Unix(timeNow(), 0).UTC().Format(time.RFC1123Z)
}

// get from a map an int given a key or fall back to a default value
func mapGetInt(m map[string]string, key string, fallback int) int {
	val, ok := m[key]
	if ok {
		i, err := strconv.ParseInt(val, 10, 32)
		if err == nil {
			return int(i)
		}
	}
	return fallback
}

func isSage(str string) bool {
	str = strings.ToLower(str)
	return str == "sage" || strings.HasPrefix(str, "sage ")
}

func unhex(str string) []byte {
	buff, _ := hex.DecodeString(str)
	return buff
}

func hexify(data []byte) string {
	return hex.EncodeToString(data)
}

// extract pubkey from secret key
// return as base32
func getSignPubkey(sk []byte) string {
	k, _ := nacl.GetSignPubkey(sk)
	return hexify(k)
}

// sign data with secret key the fucky srnd way
// return signature as base32
func cryptoSign(data, sk []byte) string {
	// hash
	hash := sha512.Sum512(data)
	log.Printf("hash=%s len=%s", hexify(hash[:]), len(data))
	// sign
	sig := nacl.CryptoSignFucky(hash[:], sk)
	return hexify(sig)
}

// given a tripcode after the #
// make a seed byteslice
func parseTripcodeSecret(str string) []byte {
	// try decoding hex
	raw := unhex(str)
	keylen := nacl.CryptoSignSeedLen()
	if raw == nil || len(raw) != keylen {
		// treat this as a "regular" chan tripcode
		// decode as bytes then pad the rest with 0s if it doesn't fit
		raw = make([]byte, keylen)
		str_bytes := []byte(str)
		if len(str_bytes) > keylen {
			copy(raw, str_bytes[:keylen])
		} else {
			copy(raw, str_bytes)
		}
	}
	return raw
}

// generate a login salt for nntp users
func genLoginCredSalt() (salt string) {
	salt = randStr(128)
	return
}

// do nntp login credential hash given password and salt
func nntpLoginCredHash(passwd, salt string) (str string) {
	var b []byte
	b = append(b, []byte(passwd)...)
	b = append(b, []byte(salt)...)
	h := sha512.Sum512(b)
	str = base64.StdEncoding.EncodeToString(h[:])
	return
}
<commit_msg>Add IP helper functions<commit_after>//
// util.go -- various utilities
//

package srnd

import (
	"crypto/sha1"
	"crypto/sha512"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"github.com/majestrate/nacl"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

func DelFile(fname string) {
	if CheckFile(fname) {
		os.Remove(fname)
	}
}

func CheckFile(fname string) bool {
	if _, err := os.Stat(fname); os.IsNotExist(err) {
		return false
	}
	return true
}

func IsDir(dirname string) bool {
	stat, err := os.Stat(dirname)
	if err != nil {
		log.Fatal(err)
	}
	return stat.IsDir()
}

// ensure a directory exists
func EnsureDir(dirname string) {
	stat, err := os.Stat(dirname)
	if os.IsNotExist(err) {
		os.Mkdir(dirname, 0755)
	} else if !stat.IsDir() {
		os.Remove(dirname)
		os.Mkdir(dirname, 0755)
	}
}

// TODO make this work better
func ValidMessageID(id string) bool {
	id_len := len(id)

	if id_len < 5 {
		return false
	}

	at_idx := strings.Index(id, "@")
	if at_idx < 3 {
		return false
	}

	for idx, c := range id {
		if idx == 0 {
			if c == '<' {
				continue
			}
		} else if idx == id_len-1 {
			if c == '>' {
				continue
			}
		} else {
			if idx == at_idx {
				continue
			}
			if c >= 'a' && c <= 'z' {
				continue
			}
			if c >= 'A' && c <= 'Z' {
				continue
			}
			if c >= '0' && c <= '9' {
				continue
			}
			if c == '.' {
				continue
			}
			if c == '$' {
				continue
			}
		}
		log.Printf("bad message ID: len=%d %s , invalid char at %d: %c", id_len, id, idx, c)
		return false
	}
	return true
}

// message id hash
func HashMessageID(msgid string) string {
	return fmt.Sprintf("%x", sha1.Sum([]byte(msgid)))
}

// short message id hash
func ShortHashMessageID(msgid string) string {
	return strings.ToLower(HashMessageID(msgid)[:18])
}

// will this message id produce quads?
func MessageIDWillDoQuads(msgid string) bool {
	h := HashMessageID(msgid)
	return h[0] == h[1] && h[1] == h[2] && h[2] == h[3]
}

// will this message id produce trips?
func MessageIDWillDoTrips(msgid string) bool {
	h := HashMessageID(msgid)
	return h[0] == h[1] && h[1] == h[2]
}

// will this message id produce dubs?
func MessageIDWillDoDubs(msgid string) bool {
	h := HashMessageID(msgid)
	return h[0] == h[1]
}

// shorter message id hash
func ShorterHashMessageID(msgid string) string {
	return strings.ToLower(HashMessageID(msgid)[:10])
}

type lineWriter struct {
	io.Writer
	wr    io.Writer
	delim []byte
}

func NewLineWriter(wr io.Writer, delim string) io.Writer {
	return lineWriter{wr, wr, []byte(delim)}
}

func (self lineWriter) Write(data []byte) (n int, err error) {
	n, err = self.wr.Write(data)
	self.wr.Write(self.delim)
	return n, err
}

func OpenFileWriter(fname string) (io.WriteCloser, error) {
	return os.Create(fname)
}

// make a random string
func randStr(length int) string {
	return hex.EncodeToString(nacl.RandBytes(length))[length:]
}

// time for right now as int64
func timeNow() int64 {
	return time.Now().Unix()
}

// sanitize data for nntp
func nntpSanitize(data string) string {
	parts := strings.Split(data, "\n.\n")
	return parts[0]
}

type int64Sorter []int64

func (self int64Sorter) Len() int {
	return len(self)
}

func (self int64Sorter) Less(i, j int) bool {
	return self[i] < self[j]
}

func (self int64Sorter) Swap(i, j int) {
	tmp := self[j]
	self[j] = self[i]
	self[i] = tmp
}

// obtain the "real" ip address
func getRealIP(name string) string {
	if len(name) > 0 {
		ip, err := net.ResolveIPAddr("ip", name)
		if err == nil {
			if ip.IP.IsGlobalUnicast() {
				return ip.IP.String()
			}
		}
	}
	return ""
}

// check that we have permission to access this
// fatal on fail
func checkPerms(fname string) {
	fstat, err := os.Stat(fname)
	if err != nil {
		log.Fatalf("Cannot access %s, %s", fname, err)
	}
	// check if we can access this dir
	if fstat.IsDir() {
		tmpfname := filepath.Join(fname, ".test")
		f, err := os.Create(tmpfname)
		if err != nil {
			log.Fatalf("No Write access in %s, %s", fname, err)
		}
		err = f.Close()
		if err != nil {
			log.Fatalf("failed to close test file %s !? %s", tmpfname, err)
		}
		err = os.Remove(tmpfname)
		if err != nil {
			log.Fatalf("failed to remove test file %s, %s", tmpfname, err)
		}
	} else {
		// this isn't a dir, treat it like a regular file
		f, err := os.Open(fname)
		if err != nil {
			log.Fatalf("cannot read file %s, %s", fname, err)
		}
		f.Close()
	}
}

// number of bytes to use in otp
func encAddrBytes() int {
	return 64
}

// length of an encrypted clearnet address
func encAddrLen() int {
	return 88
}

// length of an i2p dest hash
func i2pDestHashLen() int {
	return 44
}

// given an address
// generate a new encryption key for it
// return the encryption key and the encrypted address
func newAddrEnc(addr string) (string, string) {
	key_bytes := nacl.RandBytes(encAddrBytes())
	key := base64.StdEncoding.EncodeToString(key_bytes)
	return key, encAddr(addr, key)
}

// xor address with a one time pad
// if the address isn't long enough it's padded with spaces
func encAddr(addr, key string) string {
	key_bytes, err := base64.StdEncoding.DecodeString(key)

	if err != nil {
		log.Println("encAddr() key base64 decode", err)
		return ""
	}

	if len(addr) > len(key_bytes) {
		log.Println("encAddr() len(addr) > len(key_bytes)")
		return ""
	}

	// pad with spaces
	for len(addr) < len(key_bytes) {
		addr += " "
	}

	addr_bytes := []byte(addr)
	res_bytes := make([]byte, len(addr_bytes))
	for idx, b := range key_bytes {
		res_bytes[idx] = addr_bytes[idx] ^ b
	}

	return base64.StdEncoding.EncodeToString(res_bytes)
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// decrypt an address
// strips any whitespaces
func decAddr(encaddr, key string) string {
	encaddr_bytes, err := base64.StdEncoding.DecodeString(encaddr)
	if err != nil {
		log.Println("decAddr() encaddr base64 decode", err)
		return ""
	}
	if len(encaddr_bytes) != len(key) {
		log.Println("decAddr() len(encaddr_bytes) != len(key)")
		return ""
	}
	key_bytes, err := base64.StdEncoding.DecodeString(key)
	if err != nil {
		log.Println("decAddr() key base64 decode", err)
	}
	res_bytes := make([]byte, len(key))
	for idx, b := range key_bytes {
		res_bytes[idx] = encaddr_bytes[idx] ^ b
	}
	res := string(res_bytes)
	return strings.Trim(res, " ")
}

func newsgroupValidFormat(newsgroup string) bool {
	// too long newsgroup
	if len(newsgroup) > 128 {
		return false
	}
	for _, ch := range newsgroup {
		if ch >= 'a' && ch <= 'z' {
			continue
		}
		if ch >= '0' && ch <= '9' {
			continue
		}
		if ch >= 'A' && ch <= 'Z' {
			continue
		}
		if ch == '.' {
			continue
		}
		return false
	}
	return true
}

// generate a new signing keypair
// public, secret
func newSignKeypair() (string, string) {
	kp := nacl.GenSignKeypair()
	defer kp.Free()
	pk := kp.Public()
	sk := kp.Seed()
	return hex.EncodeToString(pk), hex.EncodeToString(sk)
}

// make a utf-8 tripcode
func makeTripcode(pk string) string {
	data, err := hex.DecodeString(pk)
	if err == nil {
		tripcode := ""
		//  here is the python code this is based off of
		//  i do something slightly different but this is the base
		//
		//  for x in range(0, length / 2):
		//    pub_short += '&#%i;' % (9600 + int(full_pubkey_hex[x*2:x*2+2], 16))
		//  length -= length / 2
		//  for x in range(0, length):
		//    pub_short += '&#%i;' % (9600 + int(full_pubkey_hex[-(length*2):][x*2:x*2+2], 16))
		//
		for _, c := range data {
			ch := 9600
			ch += int(c)
			tripcode += fmt.Sprintf("&#%04d;", ch)
		}
		return tripcode
	}
	return "[invalid]"
}

// generate a new message id with base name
func genMessageID(name string) string {
	return fmt.Sprintf("<%s%d@%s>", randStr(5), timeNow(), name)
}

// time now as a string timestamp
func timeNowStr() string {
	return time.Unix(timeNow(), 0).UTC().Format(time.RFC1123Z)
}

// get from a map an int given a key or fall back to a default value
func mapGetInt(m map[string]string, key string, fallback int) int {
	val, ok := m[key]
	if ok {
		i, err := strconv.ParseInt(val, 10, 32)
		if err == nil {
			return int(i)
		}
	}
	return fallback
}

func isSage(str string) bool {
	str = strings.ToLower(str)
	return str == "sage" || strings.HasPrefix(str, "sage ")
}

func unhex(str string) []byte {
	buff, _ := hex.DecodeString(str)
	return buff
}

func hexify(data []byte) string {
	return hex.EncodeToString(data)
}

// extract pubkey from secret key
// return as base32
func getSignPubkey(sk []byte) string {
	k, _ := nacl.GetSignPubkey(sk)
	return hexify(k)
}

// sign data with secret key the fucky srnd way
// return signature as base32
func cryptoSign(data, sk []byte) string {
	// hash
	hash := sha512.Sum512(data)
	log.Printf("hash=%s len=%s", hexify(hash[:]), len(data))
	// sign
	sig := nacl.CryptoSignFucky(hash[:], sk)
	return hexify(sig)
}

// given a tripcode after the #
// make a seed byteslice
func parseTripcodeSecret(str string) []byte {
	// try decoding hex
	raw := unhex(str)
	keylen := nacl.CryptoSignSeedLen()
	if raw == nil || len(raw) != keylen {
		// treat this as a "regular" chan tripcode
		// decode as bytes then pad the rest with 0s if it doesn't fit
		raw = make([]byte, keylen)
		str_bytes := []byte(str)
		if len(str_bytes) > keylen {
			copy(raw, str_bytes[:keylen])
		} else {
			copy(raw, str_bytes)
		}
	}
	return raw
}

// generate a login salt for nntp users
func genLoginCredSalt() (salt string) {
	salt = randStr(128)
	return
}

// do nntp login credential hash given password and salt
func nntpLoginCredHash(passwd, salt string) (str string) {
	var b []byte
	b = append(b, []byte(passwd)...)
	b = append(b, []byte(salt)...)
	h := sha512.Sum512(b)
	str = base64.StdEncoding.EncodeToString(h[:])
	return
}

func IsSubnet(cidr string) (bool, *net.IPNet) {
	_, ipnet, err := net.ParseCIDR(cidr)
	if err == nil {
		return true, ipnet
	}
	return false, nil
}

func IPNet2MinMax(inet *net.IPNet) (min, max net.IP) {
	netb := []byte(inet.IP)
	maskb := []byte(inet.Mask)
	maxb := make([]byte, len(netb))

	for i, _ := range maxb {
		maxb[i] = netb[i] | (^maskb[i])
	}
	min = net.IP(netb)
	max = net.IP(maxb)
	return
}

func ZeroIPString(ip net.IP) string {
	p := ip

	if len(ip) == 0 {
		return "<nil>"
	}

	if p4 := p.To4(); len(p4) == net.IPv4len {
		return fmt.Sprintf("%03d.%03d.%03d.%03d", p4[0], p4[1], p4[2], p4[3])
	}
	if len(p) == net.IPv6len {
		//>IPv6
		//ishygddt
		return fmt.Sprintf("%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x", p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15])
	}
	return "?"
}
<|endoftext|>


<commit_before>package jsonstore

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"strings"
	"sync"
)

type JSONStore struct {
	Data     map[string]interface{}
	location string
	gzip     bool
	sync.RWMutex
}

func (s *JSONStore) Init() {
	s.Lock()
	defer s.Unlock()
	s.location = "data.json.gz"
	s.Data = make(map[string]interface{})
	s.gzip = true
}

func (s *JSONStore) SetGzip(on bool) {
	s.Lock()
	defer s.Unlock()
	s.gzip = on
}

func (s *JSONStore) SetLocation(location string) {
	s.Lock()
	s.location = location
	s.Unlock()
}

func (s *JSONStore) Load() error {
	s.Lock()
	defer s.Unlock()
	var err error
	if _, err = os.Stat(s.location); os.IsNotExist(err) {
		err = errors.New("Location does not exist")
	} else {
		var b []byte
		if s.gzip {
			if !strings.Contains(s.location, ".gz") {
				s.location = s.location + ".gz"
			}
			b, err = readGzFile(s.location)
			if err != nil {
				return err
			}
		} else {
			b, err = ioutil.ReadFile(s.location)
			if err != nil {
				return err
			}
		}
		err = json.Unmarshal(b, &s.Data)
	}
	return err
}

func (s *JSONStore) Save() error {
	s.RLock()
	defer s.RUnlock()
	var err error
	b, err := json.MarshalIndent(s.Data, "", " ")
	if err != nil {
		return err
	}

	if s.gzip {
		var b2 bytes.Buffer
		w := gzip.NewWriter(&b2)
		w.Write(b)
		w.Close()
		err = ioutil.WriteFile(s.location, b2.Bytes(), 0644)
	} else {
		err = ioutil.WriteFile(s.location, b, 0644)
	}
	return err
}

func (s *JSONStore) Set(key string, value interface{}) error {
	s.set(key, value)
	s.Save()
	return nil
}

func (s *JSONStore) SetMem(key string, value interface{}) error {
	s.set(key, value)
	return nil
}

func (s *JSONStore) set(key string, value interface{}) error {
	s.Lock()
	defer s.Unlock()
	s.Data[key] = value
	return nil
}

func (s *JSONStore) Get(key string) (interface{}, error) {
	if strings.Contains(key, "*") {
		return s.getmany(key)
	} else {
		return s.getone(key)
	}
}

func (s *JSONStore) getmany(key string) (interface{}, error) {
	s.RLock()
	defer s.RUnlock()
	possible := []string{}
	for _, substring := range strings.Split(key, "*") {
		if strings.Contains(substring, "*") || len(substring) == 0 {
			continue
		}
		possible = append(possible, substring)
	}

	m := make(map[string]interface{})
	for key := range s.Data {
		for _, substring := range possible {
			if strings.Contains(key, substring) {
				m[key] = s.Data[key]
			}
		}
	}

	if len(m) == 0 {
		return -1, errors.New(key + " not found")
	}
	return m, nil
}

func (s *JSONStore) getone(key string) (interface{}, error) {
	s.RLock()
	defer s.RUnlock()
	val, ok := s.Data[key]
	if !ok {
		return -1, errors.New(key + " not found")
	}
	return val, nil
}

// utils

// from http://stackoverflow.com/questions/16890648/how-can-i-use-golangs-compress-gzip-package-to-gzip-a-file
func readGzFile(filename string) ([]byte, error) {
	fi, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer fi.Close()

	fz, err := gzip.NewReader(fi)
	if err != nil {
		return nil, err
	}
	defer fz.Close()

	s, err := ioutil.ReadAll(fz)
	if err != nil {
		return nil, err
	}
	return s, nil
}
<commit_msg>Added comments<commit_after>package jsonstore

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"strings"
	"sync"
)

type JSONStore struct {
	Data     map[string]interface{}
	location string
	gzip     bool
	sync.RWMutex
}

// Init initializes the JSON store so that it will save to `data.json.gz`
// with GZIP enabled automatically
func (s *JSONStore) Init() {
	s.Lock()
	defer s.Unlock()
	s.location = "data.json.gz"
	s.Data = make(map[string]interface{})
	s.gzip = true
}

// SetGzip will toggle Gzip compression
func (s *JSONStore) SetGzip(on bool) {
	s.Lock()
	defer s.Unlock()
	s.gzip = on
}

// SetLocation determines where the file will be saved for persistence
func (s *JSONStore) SetLocation(location string) {
	s.Lock()
	s.location = location
	s.Unlock()
}

// Load will load the data from the current file
func (s *JSONStore) Load() error {
	s.Lock()
	defer s.Unlock()
	var err error
	if _, err = os.Stat(s.location); os.IsNotExist(err) {
		err = errors.New("Location does not exist")
	} else {
		var b []byte
		if s.gzip {
			if !strings.Contains(s.location, ".gz") {
				s.location = s.location + ".gz"
			}
			b, err = readGzFile(s.location)
			if err != nil {
				return err
			}
		} else {
			b, err = ioutil.ReadFile(s.location)
			if err != nil {
				return err
			}
		}
		err = json.Unmarshal(b, &s.Data)
	}
	return err
}

// Save will save the current data to the location, adding Gzip compression if
// is enabled (it is by default)
func (s *JSONStore) Save() error {
	s.RLock()
	defer s.RUnlock()
	var err error
	b, err := json.MarshalIndent(s.Data, "", " ")
	if err != nil {
		return err
	}

	if s.gzip {
		var b2 bytes.Buffer
		w := gzip.NewWriter(&b2)
		w.Write(b)
		w.Close()
		err = ioutil.WriteFile(s.location, b2.Bytes(), 0644)
	} else {
		err = ioutil.WriteFile(s.location, b, 0644)
	}
	return err
}

// Set will set a key to a value, and then save go disk
func (s *JSONStore) Set(key string, value interface{}) error {
	s.set(key, value)
	s.Save()
	return nil
}

// SetMem will set a key to a value, but not save to disk
func (s *JSONStore) SetMem(key string, value interface{}) error {
	s.set(key, value)
	return nil
}

func (s *JSONStore) set(key string, value interface{}) error {
	s.Lock()
	defer s.Unlock()
	s.Data[key] = value
	return nil
}

// Get will return the value associated with a key
// if the key contains a `*`, like `name:*`, it will a map[string]interface{}
// where each key is a key containing `*` and its corresponding value
func (s *JSONStore) Get(key string) (interface{}, error) {
	if strings.Contains(key, "*") {
		return s.getmany(key)
	} else {
		return s.getone(key)
	}
}

func (s *JSONStore) getmany(key string) (interface{}, error) {
	s.RLock()
	defer s.RUnlock()
	possible := []string{}
	for _, substring := range strings.Split(key, "*") {
		if strings.Contains(substring, "*") || len(substring) == 0 {
			continue
		}
		possible = append(possible, substring)
	}

	m := make(map[string]interface{})
	for key := range s.Data {
		for _, substring := range possible {
			if strings.Contains(key, substring) {
				m[key] = s.Data[key]
			}
		}
	}

	if len(m) == 0 {
		return -1, errors.New(key + " not found")
	}
	return m, nil
}

func (s *JSONStore) getone(key string) (interface{}, error) {
	s.RLock()
	defer s.RUnlock()
	val, ok := s.Data[key]
	if !ok {
		return -1, errors.New(key + " not found")
	}
	return val, nil
}

// utils

// from http://stackoverflow.com/questions/16890648/how-can-i-use-golangs-compress-gzip-package-to-gzip-a-file
func readGzFile(filename string) ([]byte, error) {
	fi, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer fi.Close()

	fz, err := gzip.NewReader(fi)
	if err != nil {
		return nil, err
	}
	defer fz.Close()

	s, err := ioutil.ReadAll(fz)
	if err != nil {
		return nil, err
	}
	return s, nil
}
<|endoftext|>


<commit_before>package juju

import (
	"fmt"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"regexp"
	"sync"
)

var (
	ValidService = regexp.MustCompile("^[a-z][a-z0-9]*(-[a-z0-9]*[a-z][a-z0-9]*)*$")
	ValidUnit    = regexp.MustCompile("^[a-z][a-z0-9]*(-[a-z0-9]*[a-z][a-z0-9]*)*/[0-9]+$")
)

// Conn holds a connection to a juju.
type Conn struct {
	Environ environs.Environ
	state   *state.State
	mu      sync.Mutex
}

// NewConn returns a Conn pointing at the environName environment, or the
// default environment if not specified.
func NewConn(environName string) (*Conn, error) {
	environs, err := environs.ReadEnvirons("")
	if err != nil {
		return nil, err
	}
	environ, err := environs.Open(environName)
	if err != nil {
		return nil, err
	}
	return &Conn{Environ: environ}, nil
}

// NewConnFromAttrs returns a Conn pointing at the environment
// created with the given attributes, as created with environs.NewFromAttrs.
func NewConnFromAttrs(attrs map[string]interface{}) (*Conn, error) {
	environ, err := environs.NewFromAttrs(attrs)
	if err != nil {
		return nil, err
	}
	return &Conn{Environ: environ}, nil
}

// Bootstrap initializes the Conn's environment and makes it ready to deploy
// services.
func (c *Conn) Bootstrap(uploadTools bool) error {
	return c.Environ.Bootstrap(uploadTools)
}

// Destroy destroys the Conn's environment and all its instances.
func (c *Conn) Destroy() error {
	return c.Environ.Destroy(nil)
}

// State returns the environment state associated with c. Closing the
// obtained state will have undefined consequences; Close c instead.
func (c *Conn) State() (*state.State, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.state == nil {
		info, err := c.Environ.StateInfo()
		if err != nil {
			return nil, err
		}
		st, err := state.Open(info)
		if err != nil {
			return nil, err
		}
		c.state = st
		if err := c.updateSecrets(); err != nil {
			c.state = nil
			return nil, fmt.Errorf("unable to push secrets: %v", err)
		}
	}
	return c.state, nil
}

// updateSecrets updates the sensitive parts of the environment
// from the local configuration.
func (c *Conn) updateSecrets() error {
	cfg := c.Environ.Config()
	env, err := c.state.EnvironConfig()
	if err != nil {
		return err
	}
	// This is wrong. This will _always_ overwrite the secrets
	// in the state with the local secrets. To fix this properly
	// we need to ensure that the config, minus secrets, is always
	// pushed on bootstrap, then we can fill in the secrets here.
	secrets, err := c.Environ.Provider().SecretAttrs(cfg)
	if err != nil {
		return err
	}
	env.Update(secrets)
	n, err := env.Write()
	if err != nil {
		return err
	}
	if len(n) > 0 {
		log.Debugf("Updating %d secret(s) in environment %q", len(n), c.Environ.Name())
	}
	return nil
}

// Close terminates the connection to the environment and releases
// any associated resources.
func (c *Conn) Close() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	state := c.state
	c.state = nil
	if state != nil {
		return state.Close()
	}
	return nil
}
<commit_msg>Push the entire environment when conn.State is called.<commit_after>package juju

import (
	"fmt"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"regexp"
	"sync"
)

var (
	ValidService = regexp.MustCompile("^[a-z][a-z0-9]*(-[a-z0-9]*[a-z][a-z0-9]*)*$")
	ValidUnit    = regexp.MustCompile("^[a-z][a-z0-9]*(-[a-z0-9]*[a-z][a-z0-9]*)*/[0-9]+$")
)

// Conn holds a connection to a juju.
type Conn struct {
	Environ environs.Environ
	state   *state.State
	mu      sync.Mutex
}

// NewConn returns a Conn pointing at the environName environment, or the
// default environment if not specified.
func NewConn(environName string) (*Conn, error) {
	environs, err := environs.ReadEnvirons("")
	if err != nil {
		return nil, err
	}
	environ, err := environs.Open(environName)
	if err != nil {
		return nil, err
	}
	return &Conn{Environ: environ}, nil
}

// NewConnFromAttrs returns a Conn pointing at the environment
// created with the given attributes, as created with environs.NewFromAttrs.
func NewConnFromAttrs(attrs map[string]interface{}) (*Conn, error) {
	environ, err := environs.NewFromAttrs(attrs)
	if err != nil {
		return nil, err
	}
	return &Conn{Environ: environ}, nil
}

// Bootstrap initializes the Conn's environment and makes it ready to deploy
// services.
func (c *Conn) Bootstrap(uploadTools bool) error {
	return c.Environ.Bootstrap(uploadTools)
}

// Destroy destroys the Conn's environment and all its instances.
func (c *Conn) Destroy() error {
	return c.Environ.Destroy(nil)
}

// State returns the environment state associated with c. Closing the
// obtained state will have undefined consequences; Close c instead.
func (c *Conn) State() (*state.State, error) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.state == nil {
		info, err := c.Environ.StateInfo()
		if err != nil {
			return nil, err
		}
		st, err := state.Open(info)
		if err != nil {
			return nil, err
		}
		c.state = st
		if err := c.updateSecrets(); err != nil {
			c.state = nil
			return nil, fmt.Errorf("unable to push secrets: %v", err)
		}
	}
	return c.state, nil
}

// updateSecrets updates the sensitive parts of the environment
// from the local configuration.
func (c *Conn) updateSecrets() error {
	cfg := c.Environ.Config()
	env, err := c.state.EnvironConfig()
	if err != nil {
		return err
	}
	// This is wrong. This will _always_ overwrite the secrets
	// in the state with the local secrets. To fix this properly
	// we need to ensure that the config, minus secrets, is always
	// pushed on bootstrap, then we can fill in the secrets here.
	env.Update(cfg.AllAttrs())
	n, err := env.Write()
	if err != nil {
		return err
	}
	if len(n) > 0 {
		log.Debugf("Updating %d secret(s) in environment %q", len(n), c.Environ.Name())
	}
	return nil
}

// Close terminates the connection to the environment and releases
// any associated resources.
func (c *Conn) Close() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	state := c.state
	c.state = nil
	if state != nil {
		return state.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>package grpcHandlers

import (
	"reflect"
	"testing"
	"time"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
	"github.com/seadsystem/Backend/DB/landingzone/database"
	"github.com/seadsystem/Backend/DB/landingzone/decoders/grpcDecoders"

	sqlmock "github.com/DATA-DOG/go-sqlmock"
	pb "github.com/seadsystem/Backend/DB/landingzone/proto/packet"
)

func TestHandle(t *testing.T) {
	ctx := context.Background()

	db, mock, err := database.NewMock()
	if err != nil {
		t.Fatalf("Creating mock DB: %v", err)
	}

	mock.ExpectBegin()
	query := "COPY \\\"data_raw\\\" \\(\\\"serial\\\", \\\"type\\\", \\\"data\\\", \\\"time\\\", \\\"device\\\"\\) FROM STDIN"
	stmt := mock.ExpectPrepare(query)
	stmt.ExpectExec().WithArgs(64, "T", 0, time.Unix(500, 0), "fooo").WillReturnResult(sqlmock.NewResult(0, 1))
	stmt.ExpectExec().WithArgs(64, "T", 1, time.Unix(501, 0), "fooo").WillReturnResult(sqlmock.NewResult(0, 1))
	stmt.ExpectExec().WithArgs(64, "T", 2, time.Unix(502, 0), "fooo").WillReturnResult(sqlmock.NewResult(0, 1))
	stmt.ExpectExec().WillReturnResult(sqlmock.NewResult(0, 0))
	mock.ExpectCommit()

	in := &pb.Packet{
		Serial: 64,
		Time:   &pb.Timestamp{Seconds: 500},
		Delta:  int64(time.Second),
		Type:   "T",
		Device: "fooo",
		Data:   []int64{0, 1, 2},
	}
	wantStatus := &pb.Status{Success: true}

	s := &server{db}
	if status, err := s.SendPacket(ctx, in); err != nil || !proto.Equal(status, wantStatus) {
		t.Errorf(
			"got s.SendPacket(ctx, %s) = %s, %v, want = %s, %v",
			proto.MarshalTextString(in),
			proto.MarshalTextString(status),
			err,
			proto.MarshalTextString(wantStatus),
			nil,
		)
	}

	if err := mock.ExpectationsWereMet(); err != nil {
		t.Error("mock expectations were not met")
	}
}

func TestHandleError(t *testing.T) {
	ctx := context.Background()

	tests := []struct {
		name   string
		in     *pb.Packet
		status *pb.Status
		err    error
	}{
		{"nil packet", nil, &pb.Status{Success: false, Msg: grpcDecoders.NillPacket.Error()}, nil},
		{"nil Time", &pb.Packet{}, &pb.Status{Success: false, Msg: grpcDecoders.NillPacket.Error()}, nil},
		{"too short type", &pb.Packet{Time: &pb.Timestamp{}}, &pb.Status{Success: false, Msg: grpcDecoders.InvalidType.Error()}, nil},
		{"too long type", &pb.Packet{Type: "fo", Time: &pb.Timestamp{}}, &pb.Status{Success: false, Msg: grpcDecoders.InvalidType.Error()}, nil},
		{"valid packet", &pb.Packet{Type: "T", Time: &pb.Timestamp{}}, &pb.Status{Success: false, Msg: "call to database transaction Begin, was not expected, next expectation is: ExpectedClose => expecting database Close"}, nil},
	}
	for _, test := range tests {
		db, mock, err := database.NewMock()
		if err != nil {
			t.Fatalf("Creating mock DB: %v", err)
		}
		mock.ExpectClose()

		s := &server{db}
		if status, err := s.SendPacket(ctx, test.in); !reflect.DeepEqual(err, test.err) || !proto.Equal(status, test.status) {
			t.Errorf(
				"%s: got SendPacket(ctx, %s) = %s, %v, want = %s, %v",
				test.name,
				proto.MarshalTextString(test.in),
				proto.MarshalTextString(status),
				err,
				proto.MarshalTextString(test.status),
				test.err,
			)
		}

		if err := db.Close(); err != nil {
			t.Errorf("%s: got db.Close() = %v, want = nil", test.name, err)
		}
	}
}

func TestRegister(t *testing.T) {
	db, mock, err := database.NewMock()
	if err != nil {
		t.Fatalf("Creating mock DB: %v", err)
	}
	mock.ExpectClose()

	s := grpc.NewServer()
	Register(s, db)

	s.Stop()

	if err := db.Close(); err != nil {
		t.Fatalf("got db.Close() = %v, want = nil", err)
	}
}

func TestHandleVerbose(t *testing.T) {
	oldVerbosity := constants.Verbose
	constants.Verbose = true
	defer func() { constants.Verbose = oldVerbosity }()

	t.Log("TestHandle")
	TestHandle(t)

	t.Log("TestHandleError")
	TestHandleError(t)

	t.Log("TestRegister")
	TestRegister(t)
}
<commit_msg>valid packet + database error<commit_after>package grpcHandlers

import (
	"reflect"
	"testing"
	"time"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/seadsystem/Backend/DB/landingzone/constants"
	"github.com/seadsystem/Backend/DB/landingzone/database"
	"github.com/seadsystem/Backend/DB/landingzone/decoders/grpcDecoders"

	sqlmock "github.com/DATA-DOG/go-sqlmock"
	pb "github.com/seadsystem/Backend/DB/landingzone/proto/packet"
)

func TestHandle(t *testing.T) {
	ctx := context.Background()

	db, mock, err := database.NewMock()
	if err != nil {
		t.Fatalf("Creating mock DB: %v", err)
	}

	mock.ExpectBegin()
	query := "COPY \\\"data_raw\\\" \\(\\\"serial\\\", \\\"type\\\", \\\"data\\\", \\\"time\\\", \\\"device\\\"\\) FROM STDIN"
	stmt := mock.ExpectPrepare(query)
	stmt.ExpectExec().WithArgs(64, "T", 0, time.Unix(500, 0), "fooo").WillReturnResult(sqlmock.NewResult(0, 1))
	stmt.ExpectExec().WithArgs(64, "T", 1, time.Unix(501, 0), "fooo").WillReturnResult(sqlmock.NewResult(0, 1))
	stmt.ExpectExec().WithArgs(64, "T", 2, time.Unix(502, 0), "fooo").WillReturnResult(sqlmock.NewResult(0, 1))
	stmt.ExpectExec().WillReturnResult(sqlmock.NewResult(0, 0))
	mock.ExpectCommit()

	in := &pb.Packet{
		Serial: 64,
		Time:   &pb.Timestamp{Seconds: 500},
		Delta:  int64(time.Second),
		Type:   "T",
		Device: "fooo",
		Data:   []int64{0, 1, 2},
	}
	wantStatus := &pb.Status{Success: true}

	s := &server{db}
	if status, err := s.SendPacket(ctx, in); err != nil || !proto.Equal(status, wantStatus) {
		t.Errorf(
			"got s.SendPacket(ctx, %s) = %s, %v, want = %s, %v",
			proto.MarshalTextString(in),
			proto.MarshalTextString(status),
			err,
			proto.MarshalTextString(wantStatus),
			nil,
		)
	}

	if err := mock.ExpectationsWereMet(); err != nil {
		t.Error("mock expectations were not met")
	}
}

func TestHandleError(t *testing.T) {
	ctx := context.Background()

	tests := []struct {
		name   string
		in     *pb.Packet
		status *pb.Status
		err    error
	}{
		{"nil packet", nil, &pb.Status{Success: false, Msg: grpcDecoders.NillPacket.Error()}, nil},
		{"nil Time", &pb.Packet{}, &pb.Status{Success: false, Msg: grpcDecoders.NillPacket.Error()}, nil},
		{"too short type", &pb.Packet{Time: &pb.Timestamp{}}, &pb.Status{Success: false, Msg: grpcDecoders.InvalidType.Error()}, nil},
		{"too long type", &pb.Packet{Type: "fo", Time: &pb.Timestamp{}}, &pb.Status{Success: false, Msg: grpcDecoders.InvalidType.Error()}, nil},
		{"valid packet + database error", &pb.Packet{Type: "T", Time: &pb.Timestamp{}}, &pb.Status{Success: false, Msg: "call to database transaction Begin, was not expected, next expectation is: ExpectedClose => expecting database Close"}, nil},
	}
	for _, test := range tests {
		db, mock, err := database.NewMock()
		if err != nil {
			t.Fatalf("Creating mock DB: %v", err)
		}
		mock.ExpectClose()

		s := &server{db}
		if status, err := s.SendPacket(ctx, test.in); !reflect.DeepEqual(err, test.err) || !proto.Equal(status, test.status) {
			t.Errorf(
				"%s: got SendPacket(ctx, %s) = %s, %v, want = %s, %v",
				test.name,
				proto.MarshalTextString(test.in),
				proto.MarshalTextString(status),
				err,
				proto.MarshalTextString(test.status),
				test.err,
			)
		}

		if err := db.Close(); err != nil {
			t.Errorf("%s: got db.Close() = %v, want = nil", test.name, err)
		}
	}
}

func TestRegister(t *testing.T) {
	db, mock, err := database.NewMock()
	if err != nil {
		t.Fatalf("Creating mock DB: %v", err)
	}
	mock.ExpectClose()

	s := grpc.NewServer()
	Register(s, db)

	s.Stop()

	if err := db.Close(); err != nil {
		t.Fatalf("got db.Close() = %v, want = nil", err)
	}
}

func TestHandleVerbose(t *testing.T) {
	oldVerbosity := constants.Verbose
	constants.Verbose = true
	defer func() { constants.Verbose = oldVerbosity }()

	t.Log("TestHandle")
	TestHandle(t)

	t.Log("TestHandleError")
	TestHandleError(t)

	t.Log("TestRegister")
	TestRegister(t)
}
<|endoftext|>