<commit_before>package hap

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net"
	"sync"

	"github.com/brutella/hc/db"
	"github.com/brutella/hc/event"
	"github.com/brutella/hc/model/accessory"
	"github.com/brutella/hc/model/characteristic"
	"github.com/brutella/hc/model/container"
	"github.com/brutella/hc/netio"
	"github.com/brutella/hc/server"
	"github.com/brutella/hc/util"
	"github.com/brutella/log"
	"github.com/gosexy/to"
)

// Config provides basic configuration for an IP transport
type Config struct {
	// Path to the storage
	// When empty, the tranport stores the data inside a folder named exactly like the accessory
	StoragePath string

	// Port on which transport is reachable e.g. 12345
	// When empty, the transport uses a random port
	Port string

	// IP on which clients can connect.
	IP string

	// Pin with has to be entered on iOS client to pair with the accessory
	// When empty, the pin 00102003 is used
	Pin string
}

type ipTransport struct {
	config  Config
	context netio.HAPContext
	server  server.Server
	mutex   *sync.Mutex
	mdns    *MDNSService

	storage  util.Storage
	database db.Database

	name      string
	device    netio.SecuredDevice
	container *container.Container

	// Used to communicate between different parts of the program (e.g. successful pairing with HomeKit)
	emitter event.Emitter
}

// NewIPTransport creates a transport to provide accessories over IP.
//
// The IP transports stores the crypto keys inside a database, which
// is by default inside a folder at the current working directory.
// The folder is named exactly as the accessory name.
//
// The transports can contain more than one accessory. If this is the
// case, the first accessory acts as the HomeKit bridge.
//
// *Important:* Changing the name of the accessory, or letting multiple
// transports store the data inside the same database lead to
// unexpected behavior – don't do that.
//
// The transport is secured with an 8-digit pin, which must be entered
// by an iOS client to successfully pair with the accessory. If the
// provided transport config does not specify any pin, 00102003 is used.
func NewIPTransport(config Config, a *accessory.Accessory, as ...*accessory.Accessory) (Transport, error) {
	// Find transport name which is visible in mDNS
	name := a.Name()
	if len(name) == 0 {
		log.Fatal("Invalid empty name for first accessory")
	}

	ip, err := getFirstLocalIPAddr()
	if err != nil {
		return nil, err
	}

	default_config := Config{
		StoragePath: name,
		Pin:         "00102003",
		Port:        "",
		IP:          ip.String(),
	}

	if dir := config.StoragePath; len(dir) > 0 {
		default_config.StoragePath = dir
	}

	if pin := config.Pin; len(pin) > 0 {
		default_config.Pin = pin
	}

	if port := config.Port; len(port) > 0 {
		default_config.Port = ":" + port
	}

	if ip := config.IP; len(ip) > 0 {
		default_config.IP = ip
	}

	storage, err := util.NewFileStorage(default_config.StoragePath)
	if err != nil {
		return nil, err
	}

	// Find transport uuid which appears as "id" txt record in mDNS and
	// must be unique and stay the same over time
	uuid := transportUUIDInStorage(storage)
	database := db.NewDatabaseWithStorage(storage)

	hap_pin, err := NewPin(default_config.Pin)
	if err != nil {
		return nil, err
	}

	device, err := netio.NewSecuredDevice(uuid, hap_pin, database)

	t := &ipTransport{
		database:  database,
		name:      name,
		device:    device,
		config:    default_config,
		container: container.NewContainer(),
		mutex:     &sync.Mutex{},
		context:   netio.NewContextForSecuredDevice(device),
		emitter:   event.NewEmitter(),
	}

	t.addAccessory(a)
	for _, a := range as {
		t.addAccessory(a)
	}

	t.emitter.AddListener(t)

	return t, err
}

func (t *ipTransport) Start() {

	// Create server which handles incoming tcp connections
	config := server.Config{
		Port:      t.config.Port,
		Context:   t.context,
		Database:  t.database,
		Container: t.container,
		Device:    t.device,
		Mutex:     t.mutex,
		Emitter:   t.emitter,
	}

	s := server.NewServer(config)
	t.server = s

	// Publish accessory ip
	host := t.config.IP

	// Publish server port which might be different then `t.config.Port`
	portInt64 := to.Int64(s.Port())

	mdns := NewMDNSService(t.name, t.device.Name(), host, int(portInt64), int64(t.container.AccessoryType()))
	t.mdns = mdns

	// Paired accessories must not be reachable for other clients since iOS 9
	if t.isPaired() {
		mdns.SetReachable(false)
	}

	mdns.Publish()

	// Listen until server.Stop() is called
	s.ListenAndServe()
}

// Stop stops the ip transport by unpublishing the mDNS service.
func (t *ipTransport) Stop() {
	if t.mdns != nil {
		t.mdns.Stop()
	}

	if t.server != nil {
		t.server.Stop()
	}
}

// isPaired returns true when the transport is already paired
func (t *ipTransport) isPaired() bool {

	// If more than one entity is stored in the database, we are paired with a device.
	// The transport itself is a device and is stored in the database, therefore
	// we have to check for more than one entity.
	if es, err := t.database.Entities(); err == nil && len(es) > 1 {
		return true
	}

	return false
}

func (t *ipTransport) updateMDNSReachability() {
	if mdns := t.mdns; mdns != nil {
		mdns.SetReachable(t.isPaired() == false)
		mdns.Update()
	}
}

func (t *ipTransport) addAccessory(a *accessory.Accessory) {
	t.container.AddAccessory(a)

	for _, s := range a.Services {
		for _, c := range s.Characteristics {
			// When a characteristic value changes and events are enabled for this characteristic
			// all listeners are notified. Since we don't track which client is interested in
			// which characteristic change event, we send them to all active connections.
			onConnChange := func(conn net.Conn, c *characteristic.Characteristic, new, old interface{}) {
				if c.Events == true {
					t.notifyListener(a, c, conn)
				}
			}
			c.OnConnChange(onConnChange)

			onChange := func(c *characteristic.Characteristic, new, old interface{}) {
				if c.Events == true {
					t.notifyListener(a, c, nil)
				}
			}
			c.OnChange(onChange)
		}
	}
}

func (t *ipTransport) notifyListener(a *accessory.Accessory, c *characteristic.Characteristic, except net.Conn) {
	conns := t.context.ActiveConnections()
	for _, conn := range conns {
		if conn == except {
			continue
		}
		resp, err := netio.New(a, c)
		if err != nil {
			log.Fatal(err)
		}

		// Write response into buffer to replace HTTP protocol
		// specifier with EVENT as required by HAP
		var buffer = new(bytes.Buffer)
		resp.Write(buffer)
		bytes, err := ioutil.ReadAll(buffer)
		bytes = netio.FixProtocolSpecifier(bytes)
		log.Printf("[VERB] %s <- %s", conn.RemoteAddr(), string(bytes))
		conn.Write(bytes)
	}
}

// transportUUIDInStorage returns the uuid stored in storage or
// creates a new random uuid and stores it.
func transportUUIDInStorage(storage util.Storage) string {
	uuid, err := storage.Get("uuid")
	if len(uuid) == 0 || err != nil {
		str := util.RandomHexString()
		uuid = []byte(netio.MAC48Address(str))
		err := storage.Set("uuid", uuid)
		if err != nil {
			log.Fatal(err)
		}
	}
	return string(uuid)
}

// Handles event which are sent when pairing with a device is added or removed
func (t *ipTransport) Handle(ev interface{}) {
	switch ev.(type) {
	case event.DevicePaired:
		log.Printf("[INFO] Event: paired with device")
		t.updateMDNSReachability()
	case event.DeviceUnpaired:
		log.Printf("[INFO] Event: unpaired with device")
		t.updateMDNSReachability()
	default:
		break
	}
}

// GetFirstLocalIPAddress returns the first available IP address of the local machine
// This is a fix for Beaglebone Black where net.LookupIP(hostname) return no IP address.
func getFirstLocalIPAddr() (net.IP, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}

	for _, addr := range addrs {
		var ip net.IP
		switch v := addr.(type) {
		case *net.IPNet:
			ip = v.IP
		case *net.IPAddr:
			ip = v.IP
		}
		if ip == nil || ip.IsLoopback() {
			continue
		}
		ip = ip.To4()
		if ip == nil {
			continue // not an ipv4 address
		}
		return ip, nil
	}

	return nil, errors.New("Could not determine ip address")
}
<commit_msg>Log accessory ip to console #45<commit_after>package hap

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net"
	"sync"

	"github.com/brutella/hc/db"
	"github.com/brutella/hc/event"
	"github.com/brutella/hc/model/accessory"
	"github.com/brutella/hc/model/characteristic"
	"github.com/brutella/hc/model/container"
	"github.com/brutella/hc/netio"
	"github.com/brutella/hc/server"
	"github.com/brutella/hc/util"
	"github.com/brutella/log"
	"github.com/gosexy/to"
)

// Config provides basic configuration for an IP transport
type Config struct {
	// Path to the storage
	// When empty, the tranport stores the data inside a folder named exactly like the accessory
	StoragePath string

	// Port on which transport is reachable e.g. 12345
	// When empty, the transport uses a random port
	Port string

	// IP on which clients can connect.
	IP string

	// Pin with has to be entered on iOS client to pair with the accessory
	// When empty, the pin 00102003 is used
	Pin string
}

type ipTransport struct {
	config  Config
	context netio.HAPContext
	server  server.Server
	mutex   *sync.Mutex
	mdns    *MDNSService

	storage  util.Storage
	database db.Database

	name      string
	device    netio.SecuredDevice
	container *container.Container

	// Used to communicate between different parts of the program (e.g. successful pairing with HomeKit)
	emitter event.Emitter
}

// NewIPTransport creates a transport to provide accessories over IP.
//
// The IP transports stores the crypto keys inside a database, which
// is by default inside a folder at the current working directory.
// The folder is named exactly as the accessory name.
//
// The transports can contain more than one accessory. If this is the
// case, the first accessory acts as the HomeKit bridge.
//
// *Important:* Changing the name of the accessory, or letting multiple
// transports store the data inside the same database lead to
// unexpected behavior – don't do that.
//
// The transport is secured with an 8-digit pin, which must be entered
// by an iOS client to successfully pair with the accessory. If the
// provided transport config does not specify any pin, 00102003 is used.
func NewIPTransport(config Config, a *accessory.Accessory, as ...*accessory.Accessory) (Transport, error) {
	// Find transport name which is visible in mDNS
	name := a.Name()
	if len(name) == 0 {
		log.Fatal("Invalid empty name for first accessory")
	}

	ip, err := getFirstLocalIPAddr()
	if err != nil {
		return nil, err
	}

	default_config := Config{
		StoragePath: name,
		Pin:         "00102003",
		Port:        "",
		IP:          ip.String(),
	}

	if dir := config.StoragePath; len(dir) > 0 {
		default_config.StoragePath = dir
	}

	if pin := config.Pin; len(pin) > 0 {
		default_config.Pin = pin
	}

	if port := config.Port; len(port) > 0 {
		default_config.Port = ":" + port
	}

	if ip := config.IP; len(ip) > 0 {
		default_config.IP = ip
	}

	storage, err := util.NewFileStorage(default_config.StoragePath)
	if err != nil {
		return nil, err
	}

	// Find transport uuid which appears as "id" txt record in mDNS and
	// must be unique and stay the same over time
	uuid := transportUUIDInStorage(storage)
	database := db.NewDatabaseWithStorage(storage)

	hap_pin, err := NewPin(default_config.Pin)
	if err != nil {
		return nil, err
	}

	device, err := netio.NewSecuredDevice(uuid, hap_pin, database)

	t := &ipTransport{
		database:  database,
		name:      name,
		device:    device,
		config:    default_config,
		container: container.NewContainer(),
		mutex:     &sync.Mutex{},
		context:   netio.NewContextForSecuredDevice(device),
		emitter:   event.NewEmitter(),
	}

	t.addAccessory(a)
	for _, a := range as {
		t.addAccessory(a)
	}

	t.emitter.AddListener(t)

	return t, err
}

func (t *ipTransport) Start() {

	// Create server which handles incoming tcp connections
	config := server.Config{
		Port:      t.config.Port,
		Context:   t.context,
		Database:  t.database,
		Container: t.container,
		Device:    t.device,
		Mutex:     t.mutex,
		Emitter:   t.emitter,
	}

	s := server.NewServer(config)
	t.server = s

	// Publish accessory ip
	ip := t.config.IP
	log.Println("[INFO] Accessory IP is", ip)

	// Publish server port which might be different then `t.config.Port`
	portInt64 := to.Int64(s.Port())

	mdns := NewMDNSService(t.name, t.device.Name(), ip, int(portInt64), int64(t.container.AccessoryType()))
	t.mdns = mdns

	// Paired accessories must not be reachable for other clients since iOS 9
	if t.isPaired() {
		mdns.SetReachable(false)
	}

	mdns.Publish()

	// Listen until server.Stop() is called
	s.ListenAndServe()
}

// Stop stops the ip transport by unpublishing the mDNS service.
func (t *ipTransport) Stop() {
	if t.mdns != nil {
		t.mdns.Stop()
	}

	if t.server != nil {
		t.server.Stop()
	}
}

// isPaired returns true when the transport is already paired
func (t *ipTransport) isPaired() bool {

	// If more than one entity is stored in the database, we are paired with a device.
	// The transport itself is a device and is stored in the database, therefore
	// we have to check for more than one entity.
	if es, err := t.database.Entities(); err == nil && len(es) > 1 {
		return true
	}

	return false
}

func (t *ipTransport) updateMDNSReachability() {
	if mdns := t.mdns; mdns != nil {
		mdns.SetReachable(t.isPaired() == false)
		mdns.Update()
	}
}

func (t *ipTransport) addAccessory(a *accessory.Accessory) {
	t.container.AddAccessory(a)

	for _, s := range a.Services {
		for _, c := range s.Characteristics {
			// When a characteristic value changes and events are enabled for this characteristic
			// all listeners are notified. Since we don't track which client is interested in
			// which characteristic change event, we send them to all active connections.
			onConnChange := func(conn net.Conn, c *characteristic.Characteristic, new, old interface{}) {
				if c.Events == true {
					t.notifyListener(a, c, conn)
				}
			}
			c.OnConnChange(onConnChange)

			onChange := func(c *characteristic.Characteristic, new, old interface{}) {
				if c.Events == true {
					t.notifyListener(a, c, nil)
				}
			}
			c.OnChange(onChange)
		}
	}
}

func (t *ipTransport) notifyListener(a *accessory.Accessory, c *characteristic.Characteristic, except net.Conn) {
	conns := t.context.ActiveConnections()
	for _, conn := range conns {
		if conn == except {
			continue
		}
		resp, err := netio.New(a, c)
		if err != nil {
			log.Fatal(err)
		}

		// Write response into buffer to replace HTTP protocol
		// specifier with EVENT as required by HAP
		var buffer = new(bytes.Buffer)
		resp.Write(buffer)
		bytes, err := ioutil.ReadAll(buffer)
		bytes = netio.FixProtocolSpecifier(bytes)
		log.Printf("[VERB] %s <- %s", conn.RemoteAddr(), string(bytes))
		conn.Write(bytes)
	}
}

// transportUUIDInStorage returns the uuid stored in storage or
// creates a new random uuid and stores it.
func transportUUIDInStorage(storage util.Storage) string {
	uuid, err := storage.Get("uuid")
	if len(uuid) == 0 || err != nil {
		str := util.RandomHexString()
		uuid = []byte(netio.MAC48Address(str))
		err := storage.Set("uuid", uuid)
		if err != nil {
			log.Fatal(err)
		}
	}
	return string(uuid)
}

// Handles event which are sent when pairing with a device is added or removed
func (t *ipTransport) Handle(ev interface{}) {
	switch ev.(type) {
	case event.DevicePaired:
		log.Printf("[INFO] Event: paired with device")
		t.updateMDNSReachability()
	case event.DeviceUnpaired:
		log.Printf("[INFO] Event: unpaired with device")
		t.updateMDNSReachability()
	default:
		break
	}
}

// GetFirstLocalIPAddress returns the first available IP address of the local machine
// This is a fix for Beaglebone Black where net.LookupIP(hostname) return no IP address.
func getFirstLocalIPAddr() (net.IP, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}

	for _, addr := range addrs {
		var ip net.IP
		switch v := addr.(type) {
		case *net.IPNet:
			ip = v.IP
		case *net.IPAddr:
			ip = v.IP
		}
		if ip == nil || ip.IsLoopback() {
			continue
		}
		ip = ip.To4()
		if ip == nil {
			continue // not an ipv4 address
		}
		return ip, nil
	}

	return nil, errors.New("Could not determine ip address")
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	cmds "github.com/ipfs/go-ipfs/commands"
)

func ExternalBinary() *cmds.Command {
	return &cmds.Command{
		Arguments: []cmds.Argument{
			cmds.StringArg("args", false, true, "Arguments for subcommand."),
		},
		External: true,
		Run: func(req cmds.Request, res cmds.Response) {
			binname := strings.Join(append([]string{"ipfs"}, req.Path()...), "-")
			_, err := exec.LookPath(binname)
			if err != nil {
				// special case for '--help' on uninstalled binaries.
				for _, arg := range req.Arguments() {
					if arg == "--help" || arg == "-h" {
						buf := new(bytes.Buffer)
						fmt.Fprintf(buf, "%s is an 'external' command.\n", binname)
						fmt.Fprintf(buf, "it does not currently appear to be installed.\n")
						fmt.Fprintf(buf, "please refer to the ipfs documentation for instructions\n")
						res.SetOutput(buf)
						return
					}
				}

				res.SetError(fmt.Errorf("%s not installed.", binname), cmds.ErrNormal)
				return
			}

			r, w := io.Pipe()

			cmd := exec.Command(binname, req.Arguments()...)

			// TODO: make commands lib be able to pass stdin through daemon
			//cmd.Stdin = req.Stdin()
			cmd.Stdin = io.LimitReader(nil, 0)
			cmd.Stdout = w
			cmd.Stderr = w

			// setup env of child program
			env := os.Environ()

			nd, err := req.InvocContext().GetNode()
			if err == nil {
				env = append(env, fmt.Sprintf("IPFS_ONLINE=%t", nd.OnlineMode()))
			}

			cmd.Env = env

			err = cmd.Start()
			if err != nil {
				res.SetError(fmt.Errorf("failed to start subcommand: %s", err), cmds.ErrNormal)
				return
			}

			res.SetOutput(r)

			go func() {
				err = cmd.Wait()
				if err != nil {
					res.SetError(err, cmds.ErrNormal)
				}

				w.Close()
			}()
		},
	}
}
<commit_msg>Capitalize print statements, add period to `external`<commit_after>package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	cmds "github.com/ipfs/go-ipfs/commands"
)

func ExternalBinary() *cmds.Command {
	return &cmds.Command{
		Arguments: []cmds.Argument{
			cmds.StringArg("args", false, true, "Arguments for subcommand."),
		},
		External: true,
		Run: func(req cmds.Request, res cmds.Response) {
			binname := strings.Join(append([]string{"ipfs"}, req.Path()...), "-")
			_, err := exec.LookPath(binname)
			if err != nil {
				// special case for '--help' on uninstalled binaries.
				for _, arg := range req.Arguments() {
					if arg == "--help" || arg == "-h" {
						buf := new(bytes.Buffer)
						fmt.Fprintf(buf, "%s is an 'external' command.\n", binname)
						fmt.Fprintf(buf, "It does not currently appear to be installed.\n")
						fmt.Fprintf(buf, "Please refer to the ipfs documentation for instructions.\n")
						res.SetOutput(buf)
						return
					}
				}

				res.SetError(fmt.Errorf("%s not installed.", binname), cmds.ErrNormal)
				return
			}

			r, w := io.Pipe()

			cmd := exec.Command(binname, req.Arguments()...)

			// TODO: make commands lib be able to pass stdin through daemon
			//cmd.Stdin = req.Stdin()
			cmd.Stdin = io.LimitReader(nil, 0)
			cmd.Stdout = w
			cmd.Stderr = w

			// setup env of child program
			env := os.Environ()

			nd, err := req.InvocContext().GetNode()
			if err == nil {
				env = append(env, fmt.Sprintf("IPFS_ONLINE=%t", nd.OnlineMode()))
			}

			cmd.Env = env

			err = cmd.Start()
			if err != nil {
				res.SetError(fmt.Errorf("failed to start subcommand: %s", err), cmds.ErrNormal)
				return
			}

			res.SetOutput(r)

			go func() {
				err = cmd.Wait()
				if err != nil {
					res.SetError(err, cmds.ErrNormal)
				}

				w.Close()
			}()
		},
	}
}
<|endoftext|>


<commit_before>package coreunix

import (
	dag "github.com/jbenet/go-ipfs/merkledag"
	ft "github.com/jbenet/go-ipfs/unixfs"
	u "github.com/jbenet/go-ipfs/util"
)

func AddMetadataTo(key u.Key, m *ft.Metadata, dserv dag.DAGService) (u.Key, error) {
	nd, err := dserv.Get(key)
	if err != nil {
		return "", err
	}

	mdnode := new(dag.Node)
	mdata, err := ft.BytesForMetadata(m)
	if err != nil {
		return "", err
	}

	mdnode.Data = mdata
	err = mdnode.AddNodeLinkClean("file", nd)
	if err != nil {
		return "", err
	}

	return dserv.Add(mdnode)
}

func Metadata(key u.Key, dserv dag.DAGService) (*ft.Metadata, error) {
	nd, err := dserv.Get(key)
	if err != nil {
		return nil, err
	}

	return ft.MetadataFromBytes(nd.Data)
}
<commit_msg>only expose IpfsNode as param<commit_after>package coreunix

import (
	core "github.com/jbenet/go-ipfs/core"
	dag "github.com/jbenet/go-ipfs/merkledag"
	ft "github.com/jbenet/go-ipfs/unixfs"
	u "github.com/jbenet/go-ipfs/util"
)

func AddMetadataTo(key u.Key, m *ft.Metadata, n *core.IpfsNode) (u.Key, error) {
	nd, err := n.DAG.Get(key)
	if err != nil {
		return "", err
	}

	mdnode := new(dag.Node)
	mdata, err := ft.BytesForMetadata(m)
	if err != nil {
		return "", err
	}

	mdnode.Data = mdata
	err = mdnode.AddNodeLinkClean("file", nd)
	if err != nil {
		return "", err
	}

	return n.DAG.Add(mdnode)
}

func Metadata(key u.Key, n *core.IpfsNode) (*ft.Metadata, error) {
	nd, err := n.DAG.Get(key)
	if err != nil {
		return nil, err
	}

	return ft.MetadataFromBytes(nd.Data)
}
<|endoftext|>


<commit_before>package model

import (
	"database/sql"
	"fmt"
	"testing"

	"github.com/af83/edwig/config"
	"github.com/af83/edwig/logger"
	"github.com/rubenv/sql-migrate"
	"gopkg.in/gorp.v1"

	_ "github.com/lib/pq"
)

var Database *gorp.DbMap

func InitDB(config config.DatabaseConfig) *gorp.DbMap {
	dbinfo := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable",
		config.User,
		config.Password,
		config.Name,
	)
	db, err := sql.Open("postgres", dbinfo)
	if err != nil {
		logger.Log.Panicf("Error while connecting to the database:\n%v", err)
	}
	logger.Log.Debugf("Connected to Database %s", config.Name)
	// construct a gorp DbMap
	database := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
	database.AddTableWithName(DatabaseReferential{}, "referentials")
	database.AddTableWithName(DatabasePartner{}, "partners")

	return database
}

func CloseDB(database *gorp.DbMap) {
	database.Db.Close()
}

func InitTestDb(t *testing.T) {
	config.SetEnvironment("test")
	// Load configuration
	err := config.LoadConfig("")
	if err != nil {
		t.Fatal(err)
	}
	config.Config.ApiKey = ""
	// Initialize Database
	Database = InitDB(config.Config.DB)

	_, err = Database.Exec("TRUNCATE referentials, partners, lines, operators, stop_areas, stop_visits;")
	if err != nil {
		t.Fatal(err)
	}

	_, err = Database.Exec("BEGIN;")
	if err != nil {
		t.Fatal(err)
	}
}

func CleanTestDb(t *testing.T) {
	_, err := Database.Exec("ROLLBACK;")
	if err != nil {
		t.Fatal(err)
	}
	CloseDB(Database)
}

func ApplyMigrations(operation, path string, database *sql.DB) error {
	migrations := &migrate.FileMigrationSource{
		Dir: path,
	}

	var n int
	var err error
	switch operation {
	case "up":
		n, err = migrate.Exec(database, "postgres", migrations, migrate.Up)
	case "down":
		n, err = migrate.Exec(database, "postgres", migrations, migrate.Down)
	}
	if err != nil {
		return err
	}
	logger.Log.Debugf("Applied %d migrations\n", n)

	return nil
}
<commit_msg>fix forgotten table when cleaning up test db<commit_after>package model

import (
	"database/sql"
	"fmt"
	"testing"

	"github.com/af83/edwig/config"
	"github.com/af83/edwig/logger"
	"github.com/rubenv/sql-migrate"
	"gopkg.in/gorp.v1"

	_ "github.com/lib/pq"
)

var Database *gorp.DbMap

func InitDB(config config.DatabaseConfig) *gorp.DbMap {
	dbinfo := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable",
		config.User,
		config.Password,
		config.Name,
	)
	db, err := sql.Open("postgres", dbinfo)
	if err != nil {
		logger.Log.Panicf("Error while connecting to the database:\n%v", err)
	}
	logger.Log.Debugf("Connected to Database %s", config.Name)
	// construct a gorp DbMap
	database := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
	database.AddTableWithName(DatabaseReferential{}, "referentials")
	database.AddTableWithName(DatabasePartner{}, "partners")

	return database
}

func CloseDB(database *gorp.DbMap) {
	database.Db.Close()
}

func InitTestDb(t *testing.T) {
	config.SetEnvironment("test")
	// Load configuration
	err := config.LoadConfig("")
	if err != nil {
		t.Fatal(err)
	}
	config.Config.ApiKey = ""
	// Initialize Database
	Database = InitDB(config.Config.DB)

	_, err = Database.Exec("TRUNCATE referentials, partners, lines, operators, stop_areas, stop_visits, vehicle_journeys;")
	if err != nil {
		t.Fatal(err)
	}

	_, err = Database.Exec("BEGIN;")
	if err != nil {
		t.Fatal(err)
	}
}

func CleanTestDb(t *testing.T) {
	_, err := Database.Exec("ROLLBACK;")
	if err != nil {
		t.Fatal(err)
	}
	CloseDB(Database)
}

func ApplyMigrations(operation, path string, database *sql.DB) error {
	migrations := &migrate.FileMigrationSource{
		Dir: path,
	}

	var n int
	var err error
	switch operation {
	case "up":
		n, err = migrate.Exec(database, "postgres", migrations, migrate.Up)
	case "down":
		n, err = migrate.Exec(database, "postgres", migrations, migrate.Down)
	}
	if err != nil {
		return err
	}
	logger.Log.Debugf("Applied %d migrations\n", n)

	return nil
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"io/ioutil"
	"math/big"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/common"
	"github.com/livepeer/go-livepeer/eth"
	lpTypes "github.com/livepeer/go-livepeer/eth/types"
	ffmpeg "github.com/livepeer/lpms/ffmpeg"
)

type StubClaimManager struct {
	verifyCalled         bool
	distributeFeesCalled bool
	receiptAdded         bool
}

func (cm *StubClaimManager) BroadcasterAddr() common.Address { return common.Address{} }
func (cm *StubClaimManager) AddReceipt(seqNo int64, fname string, data []byte, bSig []byte, tData map[ffmpeg.VideoProfile][]byte, tStart time.Time, tEnd time.Time) ([]byte, error) {
	cm.receiptAdded = true
	return []byte{}, nil
}
func (cm *StubClaimManager) SufficientBroadcasterDeposit() (bool, error)   { return true, nil }
func (cm *StubClaimManager) ClaimVerifyAndDistributeFees() error           { return nil }
func (cm *StubClaimManager) CanClaim(*big.Int, *lpTypes.Job) (bool, error) { return true, nil }

type StubTranscoder struct {
	Profiles      []ffmpeg.VideoProfile
	InputData     [][]byte
	FailTranscode bool
}

func (t *StubTranscoder) Transcode(fname string) ([][]byte, error) {
	d, err := ioutil.ReadFile(fname)
	if err != nil || t.FailTranscode {
		return nil, ErrTranscode
	}

	t.InputData = append(t.InputData, d)

	result := make([][]byte, 0)
	for _, p := range t.Profiles {
		result = append(result, []byte(fmt.Sprintf("Transcoded_%v", p.Name)))
	}
	return result, nil
}

func TestTranscodeAndBroadcast(t *testing.T) {
	strmID, _ := MakeStreamID(RandomVideoID(), ffmpeg.P720p30fps4x3.Name)
	jid := big.NewInt(0)
	ffmpeg.InitFFmpeg()
	p := []ffmpeg.VideoProfile{ffmpeg.P720p60fps16x9, ffmpeg.P144p30fps16x9}
	tr := &StubTranscoder{Profiles: p}
	mkid := func(p ffmpeg.VideoProfile) StreamID {
		s, _ := MakeStreamID(strmID.GetVideoID(), p.Name)
		return s
	}
	strmIds := []StreamID{mkid(p[0]), mkid(p[1])}
	cm := StubClaimManager{}
	config := transcodeConfig{StrmID: strmID.String(), Profiles: p, ResultStrmIDs: strmIds, ClaimManager: &cm, JobID: jid, Transcoder: tr}

	n, err := NewLivepeerNode(&eth.StubClient{}, ".", nil) // TODO fix empty work dir
	if err != nil {
		t.Errorf("Error: %v", err)
	}

	ss := StubSegment()
	res := n.transcodeAndCacheSeg(config, ss)
	if res.Err != nil {
		t.Errorf("Error: %v", res.Err)
	}

	if len(res.Urls) != len(p) {
		t.Errorf("Expecting %v profiles, got %v", len(p), len(res.Urls))
	}

	//Should have transcoded the segments into 2 different profiles (right now StubSubscriber emits 1 segment)
	if len(tr.InputData) != 1 {
		t.Errorf("Expecting 1 segment to be transcoded, got %v", tr.InputData)
	}

	// Check playlist was updated
	for _, v := range strmIds {
		pl := n.VideoSource.GetHLSMediaPlaylist(v)
		if pl == nil {
			t.Error("Expected media playlist; got none")
		}
		if len(pl.Segments) != 1 && pl.SeqNo != 100 {
			t.Error("Mismatched segments (expected 1) or seq (expected 100), got ", pl.Segments, pl.SeqNo)
		}
	}

	if !cm.receiptAdded {
		t.Error("Receipt was not added ", cm.receiptAdded)
	}
	// TODO check sig?

	// Test when transcoder fails
	tr.FailTranscode = true
	res = n.transcodeAndCacheSeg(config, ss)
	if res.Err == nil {
		t.Error("Expecting a transcode error")
	}
	tr.FailTranscode = false

	// Test when the number of results mismatchches expectations
	tr.Profiles = []ffmpeg.VideoProfile{p[0]}
	res = n.transcodeAndCacheSeg(config, ss)
	if res.Err == nil || res.Err.Error() != "MismatchedSegments" {
		t.Error("Did not get mismatched segments as expected")
	}
	tr.Profiles = p

	//TODO: Should have done the claiming

}

func TestNodeClaimManager(t *testing.T) {
	n, err := NewLivepeerNode(nil, ".", nil)

	job := &lpTypes.Job{
		JobId:              big.NewInt(15),
		MaxPricePerSegment: big.NewInt(1),
		Profiles:           []ffmpeg.VideoProfile{},
		TotalClaims:        big.NewInt(0),
	}

	// test claimmanager existence via manual insertion
	n.ClaimManagers[15] = &StubClaimManager{}
	cm, err := n.GetClaimManager(job)
	if err != nil || cm == nil {
		t.Errorf("Did not retrieve claimmanager %v %v", cm, err)
		return
	}

	job.JobId = big.NewInt(10)

	// test with a nil eth client
	cm, err = n.GetClaimManager(job)
	if err != nil || cm != nil {
		t.Errorf("Claimmanager unexpected result %v %v", cm, err)
		return
	}

	// test with a nonexisting job
	stubClient := eth.StubClient{}
	n.Eth = &stubClient
	cm, err = n.GetClaimManager(nil)
	if err == nil || err.Error() != "Nil job" {
		t.Error("Expected nil job ", err)
		return
	}

	// test creating a new claimmanager
	cm, err = n.GetClaimManager(job)
	if err != nil || cm == nil {
		t.Error("Expected claimmanager from job ", cm, err)
		return
	}
}
<commit_msg>core: Clean up leftover generated files from tests.<commit_after>package core

import (
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"testing"
	"time"

	"github.com/ethereum/go-ethereum/common"
	"github.com/livepeer/go-livepeer/eth"
	lpTypes "github.com/livepeer/go-livepeer/eth/types"
	ffmpeg "github.com/livepeer/lpms/ffmpeg"
)

type StubClaimManager struct {
	verifyCalled         bool
	distributeFeesCalled bool
	receiptAdded         bool
}

func (cm *StubClaimManager) BroadcasterAddr() common.Address { return common.Address{} }
func (cm *StubClaimManager) AddReceipt(seqNo int64, fname string, data []byte, bSig []byte, tData map[ffmpeg.VideoProfile][]byte, tStart time.Time, tEnd time.Time) ([]byte, error) {
	cm.receiptAdded = true
	return []byte{}, nil
}
func (cm *StubClaimManager) SufficientBroadcasterDeposit() (bool, error)   { return true, nil }
func (cm *StubClaimManager) ClaimVerifyAndDistributeFees() error           { return nil }
func (cm *StubClaimManager) CanClaim(*big.Int, *lpTypes.Job) (bool, error) { return true, nil }

type StubTranscoder struct {
	Profiles      []ffmpeg.VideoProfile
	InputData     [][]byte
	FailTranscode bool
}

func (t *StubTranscoder) Transcode(fname string) ([][]byte, error) {
	d, err := ioutil.ReadFile(fname)
	if err != nil || t.FailTranscode {
		return nil, ErrTranscode
	}

	t.InputData = append(t.InputData, d)

	result := make([][]byte, 0)
	for _, p := range t.Profiles {
		result = append(result, []byte(fmt.Sprintf("Transcoded_%v", p.Name)))
	}
	return result, nil
}

func TestTranscodeAndBroadcast(t *testing.T) {
	strmID, _ := MakeStreamID(RandomVideoID(), ffmpeg.P720p30fps4x3.Name)
	jid := big.NewInt(0)
	ffmpeg.InitFFmpeg()
	p := []ffmpeg.VideoProfile{ffmpeg.P720p60fps16x9, ffmpeg.P144p30fps16x9}
	tr := &StubTranscoder{Profiles: p}
	mkid := func(p ffmpeg.VideoProfile) StreamID {
		s, _ := MakeStreamID(strmID.GetVideoID(), p.Name)
		return s
	}
	strmIds := []StreamID{mkid(p[0]), mkid(p[1])}
	cm := StubClaimManager{}
	config := transcodeConfig{StrmID: strmID.String(), Profiles: p, ResultStrmIDs: strmIds, ClaimManager: &cm, JobID: jid, Transcoder: tr}

	tmpdir, _ := ioutil.TempDir("", "")
	n, err := NewLivepeerNode(&eth.StubClient{}, tmpdir, nil)
	if err != nil {
		t.Errorf("Error: %v", err)
	}
	defer os.RemoveAll(tmpdir)

	ss := StubSegment()
	res := n.transcodeAndCacheSeg(config, ss)
	if res.Err != nil {
		t.Errorf("Error: %v", res.Err)
	}

	if len(res.Urls) != len(p) {
		t.Errorf("Expecting %v profiles, got %v", len(p), len(res.Urls))
	}

	//Should have transcoded the segments into 2 different profiles (right now StubSubscriber emits 1 segment)
	if len(tr.InputData) != 1 {
		t.Errorf("Expecting 1 segment to be transcoded, got %v", tr.InputData)
	}

	// Check playlist was updated
	for _, v := range strmIds {
		pl := n.VideoSource.GetHLSMediaPlaylist(v)
		if pl == nil {
			t.Error("Expected media playlist; got none")
		}
		if len(pl.Segments) != 1 && pl.SeqNo != 100 {
			t.Error("Mismatched segments (expected 1) or seq (expected 100), got ", pl.Segments, pl.SeqNo)
		}
	}

	if !cm.receiptAdded {
		t.Error("Receipt was not added ", cm.receiptAdded)
	}
	// TODO check sig?

	// Test when transcoder fails
	tr.FailTranscode = true
	res = n.transcodeAndCacheSeg(config, ss)
	if res.Err == nil {
		t.Error("Expecting a transcode error")
	}
	tr.FailTranscode = false

	// Test when the number of results mismatchches expectations
	tr.Profiles = []ffmpeg.VideoProfile{p[0]}
	res = n.transcodeAndCacheSeg(config, ss)
	if res.Err == nil || res.Err.Error() != "MismatchedSegments" {
		t.Error("Did not get mismatched segments as expected")
	}
	tr.Profiles = p

	//TODO: Should have done the claiming

}

func TestNodeClaimManager(t *testing.T) {
	tmpdir, _ := ioutil.TempDir("", "")
	n, err := NewLivepeerNode(nil, tmpdir, nil)
	defer os.RemoveAll(tmpdir)

	job := &lpTypes.Job{
		JobId:              big.NewInt(15),
		MaxPricePerSegment: big.NewInt(1),
		Profiles:           []ffmpeg.VideoProfile{},
		TotalClaims:        big.NewInt(0),
	}

	// test claimmanager existence via manual insertion
	n.ClaimManagers[15] = &StubClaimManager{}
	cm, err := n.GetClaimManager(job)
	if err != nil || cm == nil {
		t.Errorf("Did not retrieve claimmanager %v %v", cm, err)
		return
	}

	job.JobId = big.NewInt(10)

	// test with a nil eth client
	cm, err = n.GetClaimManager(job)
	if err != nil || cm != nil {
		t.Errorf("Claimmanager unexpected result %v %v", cm, err)
		return
	}

	// test with a nonexisting job
	stubClient := eth.StubClient{}
	n.Eth = &stubClient
	cm, err = n.GetClaimManager(nil)
	if err == nil || err.Error() != "Nil job" {
		t.Error("Expected nil job ", err)
		return
	}

	// test creating a new claimmanager
	cm, err = n.GetClaimManager(job)
	if err != nil || cm == nil {
		t.Error("Expected claimmanager from job ", cm, err)
		return
	}
}
<|endoftext|>


<commit_before>package daemon

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/daemon/execdriver"
	"github.com/docker/docker/pkg/pubsub"
	"github.com/docker/libcontainer/system"
)

// newStatsCollector returns a new statsCollector that collections
// network and cgroup stats for a registered container at the specified
// interval.  The collector allows non-running containers to be added
// and will start processing stats when they are started.
func newStatsCollector(interval time.Duration) *statsCollector {
	s := &statsCollector{
		interval:   interval,
		publishers: make(map[*Container]*pubsub.Publisher),
		clockTicks: uint64(system.GetClockTicks()),
	}
	go s.run()
	return s
}

// statsCollector manages and provides container resource stats
type statsCollector struct {
	m          sync.Mutex
	interval   time.Duration
	clockTicks uint64
	publishers map[*Container]*pubsub.Publisher
}

// collect registers the container with the collector and adds it to
// the event loop for collection on the specified interval returning
// a channel for the subscriber to receive on.
func (s *statsCollector) collect(c *Container) chan interface{} {
	s.m.Lock()
	defer s.m.Unlock()
	publisher, exists := s.publishers[c]
	if !exists {
		publisher = pubsub.NewPublisher(100*time.Millisecond, 1024)
		s.publishers[c] = publisher
	}
	return publisher.Subscribe()
}

// stopCollection closes the channels for all subscribers and removes
// the container from metrics collection.
func (s *statsCollector) stopCollection(c *Container) {
	s.m.Lock()
	if publisher, exists := s.publishers[c]; exists {
		publisher.Close()
		delete(s.publishers, c)
	}
	s.m.Unlock()
}

// unsubscribe removes a specific subscriber from receiving updates for a container's stats.
func (s *statsCollector) unsubscribe(c *Container, ch chan interface{}) {
	s.m.Lock()
	publisher := s.publishers[c]
	if publisher != nil {
		publisher.Evict(ch)
		if publisher.Len() == 0 {
			delete(s.publishers, c)
		}
	}
	s.m.Unlock()
}

func (s *statsCollector) run() {
	type publishersPair struct {
		container *Container
		publisher *pubsub.Publisher
	}
	// we cannot determine the capacity here.
	// it will grow enough in first iteration
	var pairs []publishersPair

	for range time.Tick(s.interval) {
		systemUsage, err := s.getSystemCpuUsage()
		if err != nil {
			logrus.Errorf("collecting system cpu usage: %v", err)
			continue
		}

		// it does not make sense in the first iteration,
		// but saves allocations in further iterations
		pairs = pairs[:0]

		s.m.Lock()
		for container, publisher := range s.publishers {
			// copy pointers here to release the lock ASAP
			pairs = append(pairs, publishersPair{container, publisher})
		}
		s.m.Unlock()

		for _, pair := range pairs {
			stats, err := pair.container.Stats()
			if err != nil {
				if err != execdriver.ErrNotRunning {
					logrus.Errorf("collecting stats for %s: %v", pair.container.ID, err)
				}
				continue
			}
			stats.SystemUsage = systemUsage
			pair.publisher.Publish(stats)
		}
	}
}

const nanoSeconds = 1e9

// getSystemCpuUSage returns the host system's cpu usage in nanoseconds
// for the system to match the cgroup readings are returned in the same format.
func (s *statsCollector) getSystemCpuUsage() (uint64, error) {
	f, err := os.Open("/proc/stat")
	if err != nil {
		return 0, err
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		parts := strings.Fields(sc.Text())
		switch parts[0] {
		case "cpu":
			if len(parts) < 8 {
				return 0, fmt.Errorf("invalid number of cpu fields")
			}
			var sum uint64
			for _, i := range parts[1:8] {
				v, err := strconv.ParseUint(i, 10, 64)
				if err != nil {
					return 0, fmt.Errorf("Unable to convert value %s to int: %s", i, err)
				}
				sum += v
			}
			return (sum * nanoSeconds) / s.clockTicks, nil
		}
	}
	return 0, fmt.Errorf("invalid stat format")
}
<commit_msg>daemon/stats_collector: refactor getSystemCpuUsage<commit_after>package daemon

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/daemon/execdriver"
	"github.com/docker/docker/pkg/pubsub"
	"github.com/docker/libcontainer/system"
)

// newStatsCollector returns a new statsCollector that collections
// network and cgroup stats for a registered container at the specified
// interval.  The collector allows non-running containers to be added
// and will start processing stats when they are started.
func newStatsCollector(interval time.Duration) *statsCollector {
	s := &statsCollector{
		interval:   interval,
		publishers: make(map[*Container]*pubsub.Publisher),
		clockTicks: uint64(system.GetClockTicks()),
		bufReader:  bufio.NewReaderSize(nil, 128),
	}
	go s.run()
	return s
}

// statsCollector manages and provides container resource stats
type statsCollector struct {
	m          sync.Mutex
	interval   time.Duration
	clockTicks uint64
	publishers map[*Container]*pubsub.Publisher
	bufReader  *bufio.Reader
}

// collect registers the container with the collector and adds it to
// the event loop for collection on the specified interval returning
// a channel for the subscriber to receive on.
func (s *statsCollector) collect(c *Container) chan interface{} {
	s.m.Lock()
	defer s.m.Unlock()
	publisher, exists := s.publishers[c]
	if !exists {
		publisher = pubsub.NewPublisher(100*time.Millisecond, 1024)
		s.publishers[c] = publisher
	}
	return publisher.Subscribe()
}

// stopCollection closes the channels for all subscribers and removes
// the container from metrics collection.
func (s *statsCollector) stopCollection(c *Container) {
	s.m.Lock()
	if publisher, exists := s.publishers[c]; exists {
		publisher.Close()
		delete(s.publishers, c)
	}
	s.m.Unlock()
}

// unsubscribe removes a specific subscriber from receiving updates for a container's stats.
func (s *statsCollector) unsubscribe(c *Container, ch chan interface{}) {
	s.m.Lock()
	publisher := s.publishers[c]
	if publisher != nil {
		publisher.Evict(ch)
		if publisher.Len() == 0 {
			delete(s.publishers, c)
		}
	}
	s.m.Unlock()
}

func (s *statsCollector) run() {
	type publishersPair struct {
		container *Container
		publisher *pubsub.Publisher
	}
	// we cannot determine the capacity here.
	// it will grow enough in first iteration
	var pairs []publishersPair

	for range time.Tick(s.interval) {
		systemUsage, err := s.getSystemCpuUsage()
		if err != nil {
			logrus.Errorf("collecting system cpu usage: %v", err)
			continue
		}

		// it does not make sense in the first iteration,
		// but saves allocations in further iterations
		pairs = pairs[:0]

		s.m.Lock()
		for container, publisher := range s.publishers {
			// copy pointers here to release the lock ASAP
			pairs = append(pairs, publishersPair{container, publisher})
		}
		s.m.Unlock()

		for _, pair := range pairs {
			stats, err := pair.container.Stats()
			if err != nil {
				if err != execdriver.ErrNotRunning {
					logrus.Errorf("collecting stats for %s: %v", pair.container.ID, err)
				}
				continue
			}
			stats.SystemUsage = systemUsage
			pair.publisher.Publish(stats)
		}
	}
}

const nanoSeconds = 1e9

// getSystemCpuUSage returns the host system's cpu usage in nanoseconds
// for the system to match the cgroup readings are returned in the same format.
func (s *statsCollector) getSystemCpuUsage() (uint64, error) {
	var line string
	f, err := os.Open("/proc/stat")
	if err != nil {
		return 0, err
	}
	defer func() {
		s.bufReader.Reset(nil)
		f.Close()
	}()
	s.bufReader.Reset(f)
	err = nil
	for err == nil {
		line, err = s.bufReader.ReadString('\n')
		if err != nil {
			break
		}
		parts := strings.Fields(line)
		switch parts[0] {
		case "cpu":
			if len(parts) < 8 {
				return 0, fmt.Errorf("invalid number of cpu fields")
			}
			var sum uint64
			for _, i := range parts[1:8] {
				v, err := strconv.ParseUint(i, 10, 64)
				if err != nil {
					return 0, fmt.Errorf("Unable to convert value %s to int: %s", i, err)
				}
				sum += v
			}
			return (sum * nanoSeconds) / s.clockTicks, nil
		}
	}
	return 0, fmt.Errorf("invalid stat format")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"html/template"
	"io"
	"log"
	"os"
	"path/filepath"

	matrix "github.com/flynn/flynn/Godeps/_workspace/src/github.com/jvatic/asset-matrix-go"
)

func main() {
	installerSrcDir := os.Getenv("INSTALLER_SRC_DIR")
	if installerSrcDir == "" {
		installerSrcDir = "./lib/installer"
	}
	m := matrix.New(&matrix.Config{
		Paths: []*matrix.AssetRoot{
			{
				GitRepo:   "git://github.com/jvatic/marbles-js.git",
				GitBranch: "master",
				GitRef:    "4d9bf1b2519ac87df630e8f5f3bcf4db52b4a477",
				Path:      "src",
			},
			{
				GitRepo:   "git://github.com/flynn/flynn-dashboard-web-icons.git",
				GitBranch: "master",
				GitRef:    "4d190ece9dd31a3a662cf2dc9534151d7675a338",
				Path:      "assets",
			},
			{
				Path: filepath.Join(installerSrcDir, "images"),
			},
			{
				Path: "./lib/javascripts",
			},
			{
				Path: "./lib/stylesheets",
			},
			{
				Path: "./lib/images",
			},
			{
				Path: "./vendor/javascripts",
			},
			{
				Path: "./vendor/stylesheets",
			},
			{
				Path: "./vendor/fonts",
			},
		},
		Outputs: []string{
			"dashboard.js",
			"dashboard-*.js",
			"dashboard.scss",
			"ansiparse.js",
			"moment.js",
			"es6promise.js",
			"react.js",
			"react.dev.js",
			"*.png",
			"*.eot",
			"*.svg",
			"*.ttf",
			"*.woff",
		},
		OutputDir:      "./build/assets",
		AssetURLPrefix: "/assets/",
	})
	if err := m.Build(); err != nil {
		log.Fatal(err)
	}
	if err := compileTemplate(m.Manifest); err != nil {
		log.Fatal(err)
	}
	m.RemoveOldAssets()
}

func compileTemplate(manifest *matrix.Manifest) error {
	type TemplateData struct {
		Development bool
	}
	tmplHTML, err := readTemplate()
	if err != nil {
		return err
	}
	tmpl, err := template.New("dashboard.html").Funcs(template.FuncMap{
		"assetPath": func(p string) string {
			return "/assets/" + manifest.Assets[p]
		},
	}).Parse(tmplHTML)
	if err != nil {
		return err
	}
	file, err := os.Create("./build/dashboard.html")
	if err != nil {
		return err
	}
	defer file.Close()
	return tmpl.Execute(file, &TemplateData{
		Development: os.Getenv("ENVIRONMENT") == "development",
	})
}

func readTemplate() (string, error) {
	var buf bytes.Buffer
	file, err := os.Open("./lib/dashboard.html.tmpl")
	if err != nil {
		return "", err
	}
	defer file.Close()
	if _, err := io.Copy(&buf, file); err != nil {
		return "", err
	}
	return buf.String(), nil
}
<commit_msg>dashboard: Bump marbles-js<commit_after>package main

import (
	"bytes"
	"html/template"
	"io"
	"log"
	"os"
	"path/filepath"

	matrix "github.com/flynn/flynn/Godeps/_workspace/src/github.com/jvatic/asset-matrix-go"
)

func main() {
	installerSrcDir := os.Getenv("INSTALLER_SRC_DIR")
	if installerSrcDir == "" {
		installerSrcDir = "./lib/installer"
	}
	m := matrix.New(&matrix.Config{
		Paths: []*matrix.AssetRoot{
			{
				GitRepo:   "git://github.com/jvatic/marbles-js.git",
				GitBranch: "master",
				GitRef:    "0a32d09dc73f87482fb12ce963c9385fabb0d036",
				Path:      "src",
			},
			{
				GitRepo:   "git://github.com/flynn/flynn-dashboard-web-icons.git",
				GitBranch: "master",
				GitRef:    "4d190ece9dd31a3a662cf2dc9534151d7675a338",
				Path:      "assets",
			},
			{
				Path: filepath.Join(installerSrcDir, "images"),
			},
			{
				Path: "./lib/javascripts",
			},
			{
				Path: "./lib/stylesheets",
			},
			{
				Path: "./lib/images",
			},
			{
				Path: "./vendor/javascripts",
			},
			{
				Path: "./vendor/stylesheets",
			},
			{
				Path: "./vendor/fonts",
			},
		},
		Outputs: []string{
			"dashboard.js",
			"dashboard-*.js",
			"dashboard.scss",
			"ansiparse.js",
			"moment.js",
			"es6promise.js",
			"react.js",
			"react.dev.js",
			"*.png",
			"*.eot",
			"*.svg",
			"*.ttf",
			"*.woff",
		},
		OutputDir:      "./build/assets",
		AssetURLPrefix: "/assets/",
	})
	if err := m.Build(); err != nil {
		log.Fatal(err)
	}
	if err := compileTemplate(m.Manifest); err != nil {
		log.Fatal(err)
	}
	m.RemoveOldAssets()
}

func compileTemplate(manifest *matrix.Manifest) error {
	type TemplateData struct {
		Development bool
	}
	tmplHTML, err := readTemplate()
	if err != nil {
		return err
	}
	tmpl, err := template.New("dashboard.html").Funcs(template.FuncMap{
		"assetPath": func(p string) string {
			return "/assets/" + manifest.Assets[p]
		},
	}).Parse(tmplHTML)
	if err != nil {
		return err
	}
	file, err := os.Create("./build/dashboard.html")
	if err != nil {
		return err
	}
	defer file.Close()
	return tmpl.Execute(file, &TemplateData{
		Development: os.Getenv("ENVIRONMENT") == "development",
	})
}

func readTemplate() (string, error) {
	var buf bytes.Buffer
	file, err := os.Open("./lib/dashboard.html.tmpl")
	if err != nil {
		return "", err
	}
	defer file.Close()
	if _, err := io.Copy(&buf, file); err != nil {
		return "", err
	}
	return buf.String(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"container/list"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"time"

	"github.com/ovh/cds/sdk"
)

// ProcessActionVariables replaces all placeholders inside action recursively using
// - parent parameters
// - action build arguments
// - Secrets from project, application and environment
//
// This function should be called ONLY from worker
func processActionVariables(a *sdk.Action, parent *sdk.Action, ab sdk.ActionBuild, secrets []sdk.Variable) error {
	// replaces placeholder in parameters with ActionBuild variables
	// replaces placeholder in parameters with Parent params
	for i := range a.Parameters {
		keepReplacing := true
		for keepReplacing {
			t := a.Parameters[i].Value

			if parent != nil {
				for _, p := range parent.Parameters {
					log.Printf("Replacing %s (%s) with %s (%s)\n", a.Parameters[i].Name, a.Parameters[i].Value, p.Name, p.Value)
					a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)
				}
			}

			for _, p := range ab.Args {
				a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)
			}

			for _, p := range secrets {
				a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)

			}

			// If parameters wasn't updated, consider it done
			if a.Parameters[i].Value == t {
				keepReplacing = false
			}
		}
	}

	// replaces placeholder in all children recursively
	for i := range a.Actions {
		err := processActionVariables(&a.Actions[i], a, ab, secrets)
		if err != nil {
			return nil
		}
	}

	return nil
}

func startAction(a *sdk.Action, actionBuild sdk.ActionBuild) sdk.Result {

	// Process action build arguments
	for _, abp := range actionBuild.Args {

		// Process build variable for root action
		for j := range a.Parameters {
			if abp.Name == a.Parameters[j].Name {
				a.Parameters[j].Value = abp.Value
			}
		}
	}

	return runAction(a, actionBuild)
}

func replaceBuildVariablesPlaceholder(a *sdk.Action) {
	for i := range a.Parameters {
		for _, v := range buildVariables {
			a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value,
				"{{.cds.build."+v.Name+"}}", v.Value, -1)

		}
	}
}

func runAction(a *sdk.Action, actionBuild sdk.ActionBuild) sdk.Result {
	r := sdk.Result{
		Status:  sdk.StatusFail,
		BuildID: actionBuild.ID,
	}

	// Replace build variable placeholder that may have been added by last step
	replaceBuildVariablesPlaceholder(a)

	if a.Type == sdk.BuiltinAction {
		return runBuiltin(a, actionBuild)
	}
	if a.Type == sdk.PluginAction {
		return runPlugin(a, actionBuild)
	}

	// Nothing to do, success !
	if len(a.Actions) == 0 {
		r.Status = sdk.StatusSuccess
		return r
	}

	var nbDisabledChildren int

	finalActions := []sdk.Action{}
	var doNotRunChildrenAnymore bool
	for i, child := range a.Actions {
		if !child.Enabled {
			childName := fmt.Sprintf("%s/%s-%d", a.Name, child.Name, i+1)
			sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Step %s is disabled\n", name, childName))
			nbDisabledChildren++
			continue
		}

		if child.Final {
			finalActions = append(finalActions, child)
		} else {
			if !doNotRunChildrenAnymore {
				childName := fmt.Sprintf("%s/%s-%d", a.Name, child.Name, i+1)
				log.Printf("Running %s\n", childName)
				sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Starting step %s...\n", name, childName))
				r = startAction(&child, actionBuild)
				sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Step %s finished (status: %s)\n", name, childName, r.Status))
				if r.Status != sdk.StatusSuccess {
					log.Printf("Stopping %s at step %s", a.Name, childName)
					doNotRunChildrenAnymore = true
				}
			}
		}
	}

	//If all steps are disabled, set action status to disabled
	if nbDisabledChildren >= (len(a.Actions) - len(finalActions)) {
		r.Status = sdk.StatusDisabled
	}

	for i, child := range finalActions {
		childName := fmt.Sprintf("%s/%s-%d", a.Name, child.Name, i+1)
		log.Printf("Running final action : %s\n", childName)
		sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Starting final step %s...\n", name, childName))
		finalActionResult := startAction(&child, actionBuild)
		//If action is success or disabled we consider final action status
		if r.Status == sdk.StatusSuccess || r.Status == sdk.StatusDisabled {
			r = finalActionResult
		}
		if finalActionResult.Status != sdk.StatusSuccess {
			log.Printf("Stoping %s at final step %s", a.Name, childName)
			return r
		}
		sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Final step %s finished (status: %s)\n", name, childName, finalActionResult.Status))
	}

	return r
}

var logsecrets []sdk.Variable

func sendLog(buildid int64, step string, value string) error {
	for i := range logsecrets {
		if len(logsecrets[i].Value) >= 6 {
			value = strings.Replace(value, logsecrets[i].Value, "**"+logsecrets[i].Name+"**", -1)
		}
	}

	l := sdk.NewLog(buildid, step, value)
	logChan <- *l
	return nil
}

func logger(inputChan chan sdk.Log) {
	llist := list.New()

	for {
		select {
		case l, ok := <-inputChan:
			if ok {
				llist.PushBack(l)
			}
			break
		case <-time.After(1 * time.Second):
			var logs []sdk.Log

			// While list is not empty
			for llist.Len() > 0 {
				// get older log line
				l := llist.Front().Value.(sdk.Log)
				llist.Remove(llist.Front())

				// then count how many lines are exactly the same
				count := 1
				for llist.Len() > 0 {
					n := llist.Front().Value.(sdk.Log)
					if n.Value != l.Value {
						break
					}
					count++
					llist.Remove(llist.Front())
				}

				// and if count > 1, then add it at the beginning of the log
				if count > 1 {
					l.Value = fmt.Sprintf("[x%d] %s", count, l.Value)
				}
				// and append to the logs batch
				l.Value = strings.Trim(strings.Replace(l.Value, "\n", " ", -1), " \t\n") + "\n"
				logs = append(logs, l)
			}

			if len(logs) == 0 {
				continue
			}

			// Buffer log list is empty, sending batch to API
			data, err := json.Marshal(logs)
			if err != nil {
				fmt.Printf("Error: cannot marshal logs: %s\n", err)
				continue
			}

			path := fmt.Sprintf("/build/%d/log", logs[0].ActionBuildID)
			_, _, err = sdk.Request("POST", path, data)
			if err != nil {
				fmt.Printf("error: cannot send logs: %s\n", err)
				continue
			}

			break
		}
	}
}

// creates a working directory in $HOME/PROJECT/APP/PIP/BN
func setupBuildDirectory(wd string) error {

	err := os.MkdirAll(wd, 0755)
	if err != nil {
		return err
	}

	err = os.Chdir(wd)
	if err != nil {
		return err
	}

	err = os.Setenv("HOME", wd)
	if err != nil {
		return err
	}

	return nil
}

// remove the buildDirectory created by setupBuildDirectory
func teardownBuildDirectory(wd string) error {

	err := os.RemoveAll(wd)
	if err != nil {
		return err
	}

	return nil
}

func generateWorkingDirectory() (string, error) {
	size := 16
	bs := make([]byte, size)
	_, err := rand.Read(bs)
	if err != nil {
		return "", err
	}
	str := hex.EncodeToString(bs)
	token := []byte(str)[0:size]

	return string(token), nil
}

func workingDirectory(basedir string, ab sdk.ActionBuild) string {

	gen, _ := generateWorkingDirectory()

	dir := path.Join(basedir,
		fmt.Sprintf("%d", ab.PipelineID),
		fmt.Sprintf("%d", ab.PipelineActionID),
		fmt.Sprintf("%d", ab.BuildNumber),
		gen)

	return dir
}

func run(a sdk.Action, ab sdk.ActionBuild, secrets []sdk.Variable) sdk.Result {
	// REPLACE ALL VARIABLE EVEN SECRETS HERE
	err := processActionVariables(&a, nil, ab, secrets)
	if err != nil {
		log.Printf("takeActionBuildHandler> Cannot process action %s parameters: %s\n", a.Name, err)
		return sdk.Result{Status: sdk.StatusFail}
	}

	// Add secrets as string in ActionBuild.Args
	// So they can be used by plugins
	for _, s := range secrets {
		p := sdk.Parameter{
			Type:  sdk.StringParameter,
			Name:  s.Name,
			Value: s.Value,
		}
		ab.Args = append(ab.Args, p)
	}

	// If action is not done within 1 hour, KILL IT WITH FIRE
	doneChan := make(chan bool)
	go func() {
		for {
			select {
			case <-doneChan:
				return
			case <-time.After(12 * time.Hour):
				sendLog(ab.ID, "SYSTEM", fmt.Sprintf("Error: Action %s running for 12 hour on worker %s, aborting", a.Name, name))
				path := fmt.Sprintf("/queue/%d/result", ab.ID)
				body, _ := json.Marshal(sdk.Result{Status: sdk.StatusFail})
				sdk.Request("POST", path, body)
				time.Sleep(5 * time.Second)
				os.Exit(1)
			}
		}
	}()

	// Setup working directory
	wd := workingDirectory(basedir, ab)
	err = setupBuildDirectory(wd)
	if err != nil {
		sendLog(ab.ID, "SYSTEM", fmt.Sprintf("Error: cannot setup working directory (%s)", err))
		time.Sleep(5 * time.Second)
		return sdk.Result{Status: sdk.StatusFail}
	}

	// Setup user ssh keys
	err = setupSSHKey(secrets, path.Join(wd, ".ssh"))
	if err != nil {
		sendLog(ab.ID, "SYSTEM", fmt.Sprintf("Error: cannot setup ssh key (%s)", err))
		time.Sleep(5 * time.Second)
		return sdk.Result{Status: sdk.StatusFail}
	}

	logsecrets = secrets
	res := startAction(&a, ab)
	close(doneChan)
	logsecrets = nil

	err = teardownBuildDirectory(wd)
	if err != nil {
		fmt.Printf("Cannot remove build directory: %s\n", err)
	}

	sendLog(ab.ID, "SYSTEM", "Done.\n")
	fmt.Printf("Run> Done.\n")
	return res
}
<commit_msg>fix (worker): remove verbose logs (#106)<commit_after>package main

import (
	"container/list"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"time"

	"github.com/ovh/cds/sdk"
)

// ProcessActionVariables replaces all placeholders inside action recursively using
// - parent parameters
// - action build arguments
// - Secrets from project, application and environment
//
// This function should be called ONLY from worker
func processActionVariables(a *sdk.Action, parent *sdk.Action, ab sdk.ActionBuild, secrets []sdk.Variable) error {
	// replaces placeholder in parameters with ActionBuild variables
	// replaces placeholder in parameters with Parent params
	for i := range a.Parameters {
		keepReplacing := true
		for keepReplacing {
			t := a.Parameters[i].Value

			if parent != nil {
				for _, p := range parent.Parameters {
					a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)
				}
			}

			for _, p := range ab.Args {
				a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)
			}

			for _, p := range secrets {
				a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value, "{{."+p.Name+"}}", p.Value, -1)

			}

			// If parameters wasn't updated, consider it done
			if a.Parameters[i].Value == t {
				keepReplacing = false
			}
		}
	}

	// replaces placeholder in all children recursively
	for i := range a.Actions {
		err := processActionVariables(&a.Actions[i], a, ab, secrets)
		if err != nil {
			return nil
		}
	}

	return nil
}

func startAction(a *sdk.Action, actionBuild sdk.ActionBuild) sdk.Result {

	// Process action build arguments
	for _, abp := range actionBuild.Args {

		// Process build variable for root action
		for j := range a.Parameters {
			if abp.Name == a.Parameters[j].Name {
				a.Parameters[j].Value = abp.Value
			}
		}
	}

	return runAction(a, actionBuild)
}

func replaceBuildVariablesPlaceholder(a *sdk.Action) {
	for i := range a.Parameters {
		for _, v := range buildVariables {
			a.Parameters[i].Value = strings.Replace(a.Parameters[i].Value,
				"{{.cds.build."+v.Name+"}}", v.Value, -1)

		}
	}
}

func runAction(a *sdk.Action, actionBuild sdk.ActionBuild) sdk.Result {
	r := sdk.Result{
		Status:  sdk.StatusFail,
		BuildID: actionBuild.ID,
	}

	// Replace build variable placeholder that may have been added by last step
	replaceBuildVariablesPlaceholder(a)

	if a.Type == sdk.BuiltinAction {
		return runBuiltin(a, actionBuild)
	}
	if a.Type == sdk.PluginAction {
		return runPlugin(a, actionBuild)
	}

	// Nothing to do, success !
	if len(a.Actions) == 0 {
		r.Status = sdk.StatusSuccess
		return r
	}

	var nbDisabledChildren int

	finalActions := []sdk.Action{}
	var doNotRunChildrenAnymore bool
	for i, child := range a.Actions {
		if !child.Enabled {
			childName := fmt.Sprintf("%s/%s-%d", a.Name, child.Name, i+1)
			sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Step %s is disabled\n", name, childName))
			nbDisabledChildren++
			continue
		}

		if child.Final {
			finalActions = append(finalActions, child)
		} else {
			if !doNotRunChildrenAnymore {
				childName := fmt.Sprintf("%s/%s-%d", a.Name, child.Name, i+1)
				log.Printf("Running %s\n", childName)
				sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Starting step %s...\n", name, childName))
				r = startAction(&child, actionBuild)
				sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Step %s finished (status: %s)\n", name, childName, r.Status))
				if r.Status != sdk.StatusSuccess {
					log.Printf("Stopping %s at step %s", a.Name, childName)
					doNotRunChildrenAnymore = true
				}
			}
		}
	}

	//If all steps are disabled, set action status to disabled
	if nbDisabledChildren >= (len(a.Actions) - len(finalActions)) {
		r.Status = sdk.StatusDisabled
	}

	for i, child := range finalActions {
		childName := fmt.Sprintf("%s/%s-%d", a.Name, child.Name, i+1)
		log.Printf("Running final action : %s\n", childName)
		sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Starting final step %s...\n", name, childName))
		finalActionResult := startAction(&child, actionBuild)
		//If action is success or disabled we consider final action status
		if r.Status == sdk.StatusSuccess || r.Status == sdk.StatusDisabled {
			r = finalActionResult
		}
		if finalActionResult.Status != sdk.StatusSuccess {
			log.Printf("Stoping %s at final step %s", a.Name, childName)
			return r
		}
		sendLog(actionBuild.ID, childName, fmt.Sprintf("%s: Final step %s finished (status: %s)\n", name, childName, finalActionResult.Status))
	}

	return r
}

var logsecrets []sdk.Variable

func sendLog(buildid int64, step string, value string) error {
	for i := range logsecrets {
		if len(logsecrets[i].Value) >= 6 {
			value = strings.Replace(value, logsecrets[i].Value, "**"+logsecrets[i].Name+"**", -1)
		}
	}

	l := sdk.NewLog(buildid, step, value)
	logChan <- *l
	return nil
}

func logger(inputChan chan sdk.Log) {
	llist := list.New()

	for {
		select {
		case l, ok := <-inputChan:
			if ok {
				llist.PushBack(l)
			}
			break
		case <-time.After(1 * time.Second):
			var logs []sdk.Log

			// While list is not empty
			for llist.Len() > 0 {
				// get older log line
				l := llist.Front().Value.(sdk.Log)
				llist.Remove(llist.Front())

				// then count how many lines are exactly the same
				count := 1
				for llist.Len() > 0 {
					n := llist.Front().Value.(sdk.Log)
					if n.Value != l.Value {
						break
					}
					count++
					llist.Remove(llist.Front())
				}

				// and if count > 1, then add it at the beginning of the log
				if count > 1 {
					l.Value = fmt.Sprintf("[x%d] %s", count, l.Value)
				}
				// and append to the logs batch
				l.Value = strings.Trim(strings.Replace(l.Value, "\n", " ", -1), " \t\n") + "\n"
				logs = append(logs, l)
			}

			if len(logs) == 0 {
				continue
			}

			// Buffer log list is empty, sending batch to API
			data, err := json.Marshal(logs)
			if err != nil {
				fmt.Printf("Error: cannot marshal logs: %s\n", err)
				continue
			}

			path := fmt.Sprintf("/build/%d/log", logs[0].ActionBuildID)
			_, _, err = sdk.Request("POST", path, data)
			if err != nil {
				fmt.Printf("error: cannot send logs: %s\n", err)
				continue
			}

			break
		}
	}
}

// creates a working directory in $HOME/PROJECT/APP/PIP/BN
func setupBuildDirectory(wd string) error {

	err := os.MkdirAll(wd, 0755)
	if err != nil {
		return err
	}

	err = os.Chdir(wd)
	if err != nil {
		return err
	}

	err = os.Setenv("HOME", wd)
	if err != nil {
		return err
	}

	return nil
}

// remove the buildDirectory created by setupBuildDirectory
func teardownBuildDirectory(wd string) error {

	err := os.RemoveAll(wd)
	if err != nil {
		return err
	}

	return nil
}

func generateWorkingDirectory() (string, error) {
	size := 16
	bs := make([]byte, size)
	_, err := rand.Read(bs)
	if err != nil {
		return "", err
	}
	str := hex.EncodeToString(bs)
	token := []byte(str)[0:size]

	return string(token), nil
}

func workingDirectory(basedir string, ab sdk.ActionBuild) string {

	gen, _ := generateWorkingDirectory()

	dir := path.Join(basedir,
		fmt.Sprintf("%d", ab.PipelineID),
		fmt.Sprintf("%d", ab.PipelineActionID),
		fmt.Sprintf("%d", ab.BuildNumber),
		gen)

	return dir
}

func run(a sdk.Action, ab sdk.ActionBuild, secrets []sdk.Variable) sdk.Result {
	// REPLACE ALL VARIABLE EVEN SECRETS HERE
	err := processActionVariables(&a, nil, ab, secrets)
	if err != nil {
		log.Printf("takeActionBuildHandler> Cannot process action %s parameters: %s\n", a.Name, err)
		return sdk.Result{Status: sdk.StatusFail}
	}

	// Add secrets as string in ActionBuild.Args
	// So they can be used by plugins
	for _, s := range secrets {
		p := sdk.Parameter{
			Type:  sdk.StringParameter,
			Name:  s.Name,
			Value: s.Value,
		}
		ab.Args = append(ab.Args, p)
	}

	// If action is not done within 1 hour, KILL IT WITH FIRE
	doneChan := make(chan bool)
	go func() {
		for {
			select {
			case <-doneChan:
				return
			case <-time.After(12 * time.Hour):
				sendLog(ab.ID, "SYSTEM", fmt.Sprintf("Error: Action %s running for 12 hour on worker %s, aborting", a.Name, name))
				path := fmt.Sprintf("/queue/%d/result", ab.ID)
				body, _ := json.Marshal(sdk.Result{Status: sdk.StatusFail})
				sdk.Request("POST", path, body)
				time.Sleep(5 * time.Second)
				os.Exit(1)
			}
		}
	}()

	// Setup working directory
	wd := workingDirectory(basedir, ab)
	err = setupBuildDirectory(wd)
	if err != nil {
		sendLog(ab.ID, "SYSTEM", fmt.Sprintf("Error: cannot setup working directory (%s)", err))
		time.Sleep(5 * time.Second)
		return sdk.Result{Status: sdk.StatusFail}
	}

	// Setup user ssh keys
	err = setupSSHKey(secrets, path.Join(wd, ".ssh"))
	if err != nil {
		sendLog(ab.ID, "SYSTEM", fmt.Sprintf("Error: cannot setup ssh key (%s)", err))
		time.Sleep(5 * time.Second)
		return sdk.Result{Status: sdk.StatusFail}
	}

	logsecrets = secrets
	res := startAction(&a, ab)
	close(doneChan)
	logsecrets = nil

	err = teardownBuildDirectory(wd)
	if err != nil {
		fmt.Printf("Cannot remove build directory: %s\n", err)
	}

	sendLog(ab.ID, "SYSTEM", "Done.\n")
	fmt.Printf("Run> Done.\n")
	return res
}
<|endoftext|>


<commit_before>// This file was auto-generated via go generate.
// DO NOT UPDATE MANUALLY

/*
The vdl tool manages veyron VDL source code.  It's similar to the go tool used
for managing Go source code.

Usage:
   vdl [flags] <command>

The vdl commands are:
   generate    Compile packages and dependencies, and generate code
   compile     Compile packages and dependencies, but don't generate code
   audit       Check if any packages are stale and need generation
   list        List package and dependency info in transitive order
   help        Display help for commands or topics
Run "vdl help [command]" for command usage.

The vdl additional help topics are:
   packages    Description of package lists
   vdlpath     Description of VDLPATH environment variable
   vdlroot     Description of VDLROOT environment variable
   vdl.config  Description of vdl.config files
Run "vdl help [topic]" for topic details.

The vdl flags are:
 -exts=.vdl
   Comma-separated list of valid VDL file name extensions.
 -ignore_unknown=false
   Ignore unknown packages provided on the command line.
 -max_errors=-1
   Stop processing after this many errors, or -1 for unlimited.
 -v=false
   Turn on verbose logging.
 -vdl.config=vdl.config
   Basename of the optional per-package config file.

Vdl Generate

Generate compiles packages and their transitive dependencies, and generates code
in the specified languages.

Usage:
   vdl generate [flags] <packages>

<packages> are a list of packages to process, similar to the standard go tool.
For more information, run "vdl help packages".

The vdl generate flags are:
 -go_out_dir=
   Go output directory.  There are three modes:
      ""                     : Generate output in-place in the source tree
      "dir"                  : Generate output rooted at dir
      "src->dst[,s2->d2...]" : Generate output using translation rules
   Assume your source tree is organized as follows:
      VDLPATH=/home/vdl
         /home/vdl/src/veyron/test_base/base1.vdl
         /home/vdl/src/veyron/test_base/base2.vdl
   Here's example output under the different modes:
      --go_out_dir=""
         /home/vdl/src/veyron/test_base/base1.vdl.go
         /home/vdl/src/veyron/test_base/base2.vdl.go
      --go_out_dir="/tmp/foo"
         /tmp/foo/veyron/test_base/base1.vdl.go
         /tmp/foo/veyron/test_base/base2.vdl.go
      --go_out_dir="vdl/src->foo/bar/src"
         /home/foo/bar/src/veyron/test_base/base1.vdl.go
         /home/foo/bar/src/veyron/test_base/base2.vdl.go
   When the src->dst form is used, src must match the suffix of the path just
   before the package path, and dst is the replacement for src.  Use commas to
   separate multiple rules; the first rule matching src is used.  The special
   dst SKIP indicates matching packages are skipped.
 -java_out_dir=go/src->java/src/vdl/java,release/go/src/v.io/core/veyron2/vdl/vdlroot/src->SKIP
   Same semantics as --go_out_dir but applies to java code generation.
 -java_out_pkg=v.io->io/v
   Java output package translation rules.  Must be of the form:
      "src->dst[,s2->d2...]"
   If a VDL package has a prefix src, the prefix will be replaced with dst.  Use
   commas to separate multiple rules; the first rule matching src is used, and
   if there are no matching rules, the package remains unchanged.  The special
   dst SKIP indicates matching packages are skipped.
 -js_out_dir=release/go/src->release/javascript/core/src,roadmap/go/src->release/javascript/core/src,third_party/go/src->SKIP,tools/go/src->SKIP,release/go/src/v.io/core/veyron2/vdl/vdlroot/src->SKIP
   Same semantics as --go_out_dir but applies to js code generation.
 -lang=Go,Java
   Comma-separated list of languages to generate, currently supporting
   Go,Java,Javascript
 -status=true
   Show package names as they are updated

Vdl Compile

Compile compiles packages and their transitive dependencies, but does not
generate code.  This is useful to sanity-check that your VDL files are valid.

Usage:
   vdl compile [flags] <packages>

<packages> are a list of packages to process, similar to the standard go tool.
For more information, run "vdl help packages".

The vdl compile flags are:
 -status=true
   Show package names while we compile

Vdl Audit

Audit runs the same logic as generate, but doesn't write out generated files.
Returns a 0 exit code if all packages are up-to-date, otherwise returns a non-0
exit code indicating some packages need generation.

Usage:
   vdl audit [flags] <packages>

<packages> are a list of packages to process, similar to the standard go tool.
For more information, run "vdl help packages".

The vdl audit flags are:
 -go_out_dir=
   Go output directory.  There are three modes:
      ""                     : Generate output in-place in the source tree
      "dir"                  : Generate output rooted at dir
      "src->dst[,s2->d2...]" : Generate output using translation rules
   Assume your source tree is organized as follows:
      VDLPATH=/home/vdl
         /home/vdl/src/veyron/test_base/base1.vdl
         /home/vdl/src/veyron/test_base/base2.vdl
   Here's example output under the different modes:
      --go_out_dir=""
         /home/vdl/src/veyron/test_base/base1.vdl.go
         /home/vdl/src/veyron/test_base/base2.vdl.go
      --go_out_dir="/tmp/foo"
         /tmp/foo/veyron/test_base/base1.vdl.go
         /tmp/foo/veyron/test_base/base2.vdl.go
      --go_out_dir="vdl/src->foo/bar/src"
         /home/foo/bar/src/veyron/test_base/base1.vdl.go
         /home/foo/bar/src/veyron/test_base/base2.vdl.go
   When the src->dst form is used, src must match the suffix of the path just
   before the package path, and dst is the replacement for src.  Use commas to
   separate multiple rules; the first rule matching src is used.  The special
   dst SKIP indicates matching packages are skipped.
 -java_out_dir=go/src->java/src/vdl/java,release/go/src/v.io/core/veyron2/vdl/vdlroot/src->SKIP
   Same semantics as --go_out_dir but applies to java code generation.
 -java_out_pkg=v.io->io/v
   Java output package translation rules.  Must be of the form:
      "src->dst[,s2->d2...]"
   If a VDL package has a prefix src, the prefix will be replaced with dst.  Use
   commas to separate multiple rules; the first rule matching src is used, and
   if there are no matching rules, the package remains unchanged.  The special
   dst SKIP indicates matching packages are skipped.
 -js_out_dir=release/go/src->release/javascript/core/src,roadmap/go/src->release/javascript/core/src,third_party/go/src->SKIP,tools/go/src->SKIP,release/go/src/v.io/core/veyron2/vdl/vdlroot/src->SKIP
   Same semantics as --go_out_dir but applies to js code generation.
 -lang=Go,Java
   Comma-separated list of languages to generate, currently supporting
   Go,Java,Javascript
 -status=true
   Show package names as they are updated

Vdl List

List returns information about packages and their transitive dependencies, in
transitive order.  This is the same order the generate and compile commands use
for processing.  If "vdl list A" is run and A depends on B, which depends on C,
the returned order will be C, B, A.  If multiple packages are specified the
ordering is over all combined dependencies.

Reminder: cyclic dependencies between packages are not allowed.  Cyclic
dependencies between VDL files within the same package are also not allowed.
This is more strict than regular Go; it makes it easier to generate code for
other languages like C++.

Usage:
   vdl list <packages>

<packages> are a list of packages to process, similar to the standard go tool.
For more information, run "vdl help packages".

Vdl Help

Help with no args displays the usage of the parent command.

Help with args displays the usage of the specified sub-command or help topic.

"help ..." recursively displays help for all commands and topics.

The output is formatted to a target width in runes.  The target width is
determined by checking the environment variable CMDLINE_WIDTH, falling back on
the terminal width from the OS, falling back on 80 chars.  By setting
CMDLINE_WIDTH=x, if x > 0 the width is x, if x < 0 the width is unlimited, and
if x == 0 or is unset one of the fallbacks is used.

Usage:
   vdl help [flags] [command/topic ...]

[command/topic ...] optionally identifies a specific sub-command or help topic.

The vdl help flags are:
 -style=text
   The formatting style for help output, either "text" or "godoc".

Vdl Packages - help topic

Most vdl commands apply to a list of packages:

   vdl command <packages>

<packages> are a list of packages to process, similar to the standard go tool.
In its simplest form each package is an import path; e.g.
   "v.io/core/veyron/lib/vdl"

A package that is an absolute path or that begins with a . or .. element is
interpreted as a file system path, and denotes the package in that directory.

A package is a pattern if it includes one or more "..." wildcards, each of which
can match any string, including the empty string and strings containing slashes.
Such a pattern expands to all packages found in VDLPATH with names matching the
pattern.  As a special-case, x/... matches x as well as x's subdirectories.

The special-case "all" is a synonym for "...", and denotes all packages found in
VDLPATH.

Import path elements and file names are not allowed to begin with "." or "_";
such paths are ignored in wildcard matches, and return errors if specified
explicitly.

 Run "vdl help vdlpath" to see docs on VDLPATH.
 Run "go help packages" to see the standard go package docs.

Vdl Vdlpath - help topic

The VDLPATH environment variable is used to resolve import statements. It must
be set to compile and generate vdl packages.

The format is a colon-separated list of directories, where each directory must
have a "src/" directory that holds vdl source code.  The path below 'src'
determines the import path.  If VDLPATH specifies multiple directories, imports
are resolved by picking the first directory with a matching import name.

An example:

   VDPATH=/home/user/vdlA:/home/user/vdlB

   /home/user/vdlA/
      src/
         foo/                 (import "foo" refers here)
            foo1.vdl
   /home/user/vdlB/
      src/
         foo/                 (this package is ignored)
            foo2.vdl
         bar/
            baz/              (import "bar/baz" refers here)
               baz.vdl

Vdl Vdlroot - help topic

The VDLROOT environment variable is similar to VDLPATH, but instead of pointing
to multiple user source directories, it points at a single source directory
containing the standard vdl packages.

Setting VDLROOT is optional.

If VDLROOT is empty, we try to construct it out of the VANADIUM_ROOT environment
variable.  It is an error if both VDLROOT and VANADIUM_ROOT are empty.

Vdl Vdl.Config - help topic

Each vdl source package P may contain an optional file "vdl.config" within the P
directory.  This file specifies additional configuration for the vdl tool.

The format of this file is described by the vdl.Config type in the "vdl"
standard package, located at VDLROOT/src/vdl/config.go.

If the file does not exist, we use the zero value of vdl.Config.
*/
package main
<commit_msg>veyron: making sure that files create by 'go generate' files are up-to-date<commit_after>// This file was auto-generated via go generate.
// DO NOT UPDATE MANUALLY

/*
The vdl tool manages veyron VDL source code.  It's similar to the go tool used
for managing Go source code.

Usage:
   vdl [flags] <command>

The vdl commands are:
   generate    Compile packages and dependencies, and generate code
   compile     Compile packages and dependencies, but don't generate code
   audit       Check if any packages are stale and need generation
   list        List package and dependency info in transitive order
   help        Display help for commands or topics
Run "vdl help [command]" for command usage.

The vdl additional help topics are:
   packages    Description of package lists
   vdlpath     Description of VDLPATH environment variable
   vdlroot     Description of VDLROOT environment variable
   vdl.config  Description of vdl.config files
Run "vdl help [topic]" for topic details.

The vdl flags are:
 -exts=.vdl
   Comma-separated list of valid VDL file name extensions.
 -ignore_unknown=false
   Ignore unknown packages provided on the command line.
 -max_errors=-1
   Stop processing after this many errors, or -1 for unlimited.
 -v=false
   Turn on verbose logging.
 -vdl.config=vdl.config
   Basename of the optional per-package config file.

Vdl Generate

Generate compiles packages and their transitive dependencies, and generates code
in the specified languages.

Usage:
   vdl generate [flags] <packages>

<packages> are a list of packages to process, similar to the standard go tool.
For more information, run "vdl help packages".

The vdl generate flags are:
 -go_out_dir=
   Go output directory.  There are three modes:
      ""                     : Generate output in-place in the source tree
      "dir"                  : Generate output rooted at dir
      "src->dst[,s2->d2...]" : Generate output using translation rules
   Assume your source tree is organized as follows:
      VDLPATH=/home/vdl
         /home/vdl/src/veyron/test_base/base1.vdl
         /home/vdl/src/veyron/test_base/base2.vdl
   Here's example output under the different modes:
      --go_out_dir=""
         /home/vdl/src/veyron/test_base/base1.vdl.go
         /home/vdl/src/veyron/test_base/base2.vdl.go
      --go_out_dir="/tmp/foo"
         /tmp/foo/veyron/test_base/base1.vdl.go
         /tmp/foo/veyron/test_base/base2.vdl.go
      --go_out_dir="vdl/src->foo/bar/src"
         /home/foo/bar/src/veyron/test_base/base1.vdl.go
         /home/foo/bar/src/veyron/test_base/base2.vdl.go
   When the src->dst form is used, src must match the suffix of the path just
   before the package path, and dst is the replacement for src.  Use commas to
   separate multiple rules; the first rule matching src is used.  The special
   dst SKIP indicates matching packages are skipped.
 -java_out_dir=release/go/src/v.io/core/veyron2/vdl/vdlroot/src->SKIP,go/src->java/src/vdl/java
   Same semantics as --go_out_dir but applies to java code generation.
 -java_out_pkg=v.io->io/v
   Java output package translation rules.  Must be of the form:
      "src->dst[,s2->d2...]"
   If a VDL package has a prefix src, the prefix will be replaced with dst.  Use
   commas to separate multiple rules; the first rule matching src is used, and
   if there are no matching rules, the package remains unchanged.  The special
   dst SKIP indicates matching packages are skipped.
 -js_out_dir=release/go/src->release/javascript/core/src,roadmap/go/src->release/javascript/core/src,third_party/go/src->SKIP,tools/go/src->SKIP,release/go/src/v.io/core/veyron2/vdl/vdlroot/src->SKIP
   Same semantics as --go_out_dir but applies to js code generation.
 -js_relative_path_to_core=
   If set, this is the relative path from js_out_dir to the root of the JS core
 -lang=Go,Java
   Comma-separated list of languages to generate, currently supporting
   Go,Java,Javascript
 -status=true
   Show package names as they are updated

Vdl Compile

Compile compiles packages and their transitive dependencies, but does not
generate code.  This is useful to sanity-check that your VDL files are valid.

Usage:
   vdl compile [flags] <packages>

<packages> are a list of packages to process, similar to the standard go tool.
For more information, run "vdl help packages".

The vdl compile flags are:
 -status=true
   Show package names while we compile

Vdl Audit

Audit runs the same logic as generate, but doesn't write out generated files.
Returns a 0 exit code if all packages are up-to-date, otherwise returns a non-0
exit code indicating some packages need generation.

Usage:
   vdl audit [flags] <packages>

<packages> are a list of packages to process, similar to the standard go tool.
For more information, run "vdl help packages".

The vdl audit flags are:
 -go_out_dir=
   Go output directory.  There are three modes:
      ""                     : Generate output in-place in the source tree
      "dir"                  : Generate output rooted at dir
      "src->dst[,s2->d2...]" : Generate output using translation rules
   Assume your source tree is organized as follows:
      VDLPATH=/home/vdl
         /home/vdl/src/veyron/test_base/base1.vdl
         /home/vdl/src/veyron/test_base/base2.vdl
   Here's example output under the different modes:
      --go_out_dir=""
         /home/vdl/src/veyron/test_base/base1.vdl.go
         /home/vdl/src/veyron/test_base/base2.vdl.go
      --go_out_dir="/tmp/foo"
         /tmp/foo/veyron/test_base/base1.vdl.go
         /tmp/foo/veyron/test_base/base2.vdl.go
      --go_out_dir="vdl/src->foo/bar/src"
         /home/foo/bar/src/veyron/test_base/base1.vdl.go
         /home/foo/bar/src/veyron/test_base/base2.vdl.go
   When the src->dst form is used, src must match the suffix of the path just
   before the package path, and dst is the replacement for src.  Use commas to
   separate multiple rules; the first rule matching src is used.  The special
   dst SKIP indicates matching packages are skipped.
 -java_out_dir=release/go/src/v.io/core/veyron2/vdl/vdlroot/src->SKIP,go/src->java/src/vdl/java
   Same semantics as --go_out_dir but applies to java code generation.
 -java_out_pkg=v.io->io/v
   Java output package translation rules.  Must be of the form:
      "src->dst[,s2->d2...]"
   If a VDL package has a prefix src, the prefix will be replaced with dst.  Use
   commas to separate multiple rules; the first rule matching src is used, and
   if there are no matching rules, the package remains unchanged.  The special
   dst SKIP indicates matching packages are skipped.
 -js_out_dir=release/go/src->release/javascript/core/src,roadmap/go/src->release/javascript/core/src,third_party/go/src->SKIP,tools/go/src->SKIP,release/go/src/v.io/core/veyron2/vdl/vdlroot/src->SKIP
   Same semantics as --go_out_dir but applies to js code generation.
 -js_relative_path_to_core=
   If set, this is the relative path from js_out_dir to the root of the JS core
 -lang=Go,Java
   Comma-separated list of languages to generate, currently supporting
   Go,Java,Javascript
 -status=true
   Show package names as they are updated

Vdl List

List returns information about packages and their transitive dependencies, in
transitive order.  This is the same order the generate and compile commands use
for processing.  If "vdl list A" is run and A depends on B, which depends on C,
the returned order will be C, B, A.  If multiple packages are specified the
ordering is over all combined dependencies.

Reminder: cyclic dependencies between packages are not allowed.  Cyclic
dependencies between VDL files within the same package are also not allowed.
This is more strict than regular Go; it makes it easier to generate code for
other languages like C++.

Usage:
   vdl list <packages>

<packages> are a list of packages to process, similar to the standard go tool.
For more information, run "vdl help packages".

Vdl Help

Help with no args displays the usage of the parent command.

Help with args displays the usage of the specified sub-command or help topic.

"help ..." recursively displays help for all commands and topics.

The output is formatted to a target width in runes.  The target width is
determined by checking the environment variable CMDLINE_WIDTH, falling back on
the terminal width from the OS, falling back on 80 chars.  By setting
CMDLINE_WIDTH=x, if x > 0 the width is x, if x < 0 the width is unlimited, and
if x == 0 or is unset one of the fallbacks is used.

Usage:
   vdl help [flags] [command/topic ...]

[command/topic ...] optionally identifies a specific sub-command or help topic.

The vdl help flags are:
 -style=text
   The formatting style for help output, either "text" or "godoc".

Vdl Packages - help topic

Most vdl commands apply to a list of packages:

   vdl command <packages>

<packages> are a list of packages to process, similar to the standard go tool.
In its simplest form each package is an import path; e.g.
   "v.io/core/veyron/lib/vdl"

A package that is an absolute path or that begins with a . or .. element is
interpreted as a file system path, and denotes the package in that directory.

A package is a pattern if it includes one or more "..." wildcards, each of which
can match any string, including the empty string and strings containing slashes.
Such a pattern expands to all packages found in VDLPATH with names matching the
pattern.  As a special-case, x/... matches x as well as x's subdirectories.

The special-case "all" is a synonym for "...", and denotes all packages found in
VDLPATH.

Import path elements and file names are not allowed to begin with "." or "_";
such paths are ignored in wildcard matches, and return errors if specified
explicitly.

 Run "vdl help vdlpath" to see docs on VDLPATH.
 Run "go help packages" to see the standard go package docs.

Vdl Vdlpath - help topic

The VDLPATH environment variable is used to resolve import statements. It must
be set to compile and generate vdl packages.

The format is a colon-separated list of directories, where each directory must
have a "src/" directory that holds vdl source code.  The path below 'src'
determines the import path.  If VDLPATH specifies multiple directories, imports
are resolved by picking the first directory with a matching import name.

An example:

   VDPATH=/home/user/vdlA:/home/user/vdlB

   /home/user/vdlA/
      src/
         foo/                 (import "foo" refers here)
            foo1.vdl
   /home/user/vdlB/
      src/
         foo/                 (this package is ignored)
            foo2.vdl
         bar/
            baz/              (import "bar/baz" refers here)
               baz.vdl

Vdl Vdlroot - help topic

The VDLROOT environment variable is similar to VDLPATH, but instead of pointing
to multiple user source directories, it points at a single source directory
containing the standard vdl packages.

Setting VDLROOT is optional.

If VDLROOT is empty, we try to construct it out of the VANADIUM_ROOT environment
variable.  It is an error if both VDLROOT and VANADIUM_ROOT are empty.

Vdl Vdl.Config - help topic

Each vdl source package P may contain an optional file "vdl.config" within the P
directory.  This file specifies additional configuration for the vdl tool.

The format of this file is described by the vdl.Config type in the "vdl"
standard package, located at VDLROOT/src/vdl/config.go.

If the file does not exist, we use the zero value of vdl.Config.
*/
package main
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/mozmark/OneCRL-Tools/oneCRL"
)

type revocations struct {
	byIssuerSerialNumber map[string][]string
	bySubjectPubKeyHash map[string][]string
}

func (r *revocations) LoadRecord(record oneCRL.Record) {
	// if there's no issuer name, assume we're revoking by Subject / PubKeyHash
	// otherwise it's issuer / serial
	if 0 == len(record.IssuerName) {
		if nil == r.bySubjectPubKeyHash {
			r.bySubjectPubKeyHash = make(map[string][]string)
		}
		if nil == r.bySubjectPubKeyHash[record.Subject]{
			pubKeyHashes := make([]string, 1)
			pubKeyHashes[0] = record.PubKeyHash
			r.bySubjectPubKeyHash[record.Subject] = pubKeyHashes
		} else {
			r.bySubjectPubKeyHash[record.Subject] = append(r.bySubjectPubKeyHash[record.Subject], record.PubKeyHash)
		}
	} else {
		if nil == r.byIssuerSerialNumber {
			r.byIssuerSerialNumber= make(map[string][]string)
		}
		if nil == r.byIssuerSerialNumber[record.IssuerName]{
			serials := make([]string, 1)
			serials[0] = record.SerialNumber
			r.byIssuerSerialNumber[record.IssuerName] = serials
		} else {
			r.byIssuerSerialNumber[record.IssuerName] = append(r.byIssuerSerialNumber[record.IssuerName], record.SerialNumber)
		}
	}
}

func main() {
	oneCRL.DefineFlags()
	flag.Parse()

	rev := new (revocations)

	config := oneCRL.Config

	url := config.GetRecordURL()

	err := oneCRL.LoadJSONFromURL(url, rev)
	if err != nil {
		panic(err)
	}

	for issuer, serials := range rev.byIssuerSerialNumber {
		fmt.Printf("%s\n", issuer)
		for _, serial := range serials {
			fmt.Printf(" %s\n", serial)
		}
	}
	for subject, pubKeyHashes := range rev.bySubjectPubKeyHash {
		fmt.Printf("%s\n", subject)
		for _, pubKeyHash := range pubKeyHashes {
			fmt.Printf("\t%s\n", pubKeyHash)
		}
	}
}
<commit_msg>Issue #11 - OneCRL2RevocationsTxt does not build<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/mozmark/OneCRL-Tools/oneCRL"
)

type revocations struct {
	byIssuerSerialNumber map[string][]string
	bySubjectPubKeyHash map[string][]string
}

func (r *revocations) LoadRecord(record oneCRL.Record) {
	// if there's no issuer name, assume we're revoking by Subject / PubKeyHash
	// otherwise it's issuer / serial
	if 0 == len(record.IssuerName) {
		if nil == r.bySubjectPubKeyHash {
			r.bySubjectPubKeyHash = make(map[string][]string)
		}
		if nil == r.bySubjectPubKeyHash[record.Subject]{
			pubKeyHashes := make([]string, 1)
			pubKeyHashes[0] = record.PubKeyHash
			r.bySubjectPubKeyHash[record.Subject] = pubKeyHashes
		} else {
			r.bySubjectPubKeyHash[record.Subject] = append(r.bySubjectPubKeyHash[record.Subject], record.PubKeyHash)
		}
	} else {
		if nil == r.byIssuerSerialNumber {
			r.byIssuerSerialNumber= make(map[string][]string)
		}
		if nil == r.byIssuerSerialNumber[record.IssuerName]{
			serials := make([]string, 1)
			serials[0] = record.SerialNumber
			r.byIssuerSerialNumber[record.IssuerName] = serials
		} else {
			r.byIssuerSerialNumber[record.IssuerName] = append(r.byIssuerSerialNumber[record.IssuerName], record.SerialNumber)
		}
	}
}

func main() {
	oneCRL.DefineFlags()
	flag.Parse()

	rev := new (revocations)

	config := oneCRL.GetConfig()

	url := config.GetRecordURL()

	err := oneCRL.LoadJSONFromURL(url, rev)
	if err != nil {
		panic(err)
	}

	for issuer, serials := range rev.byIssuerSerialNumber {
		fmt.Printf("%s\n", issuer)
		for _, serial := range serials {
			fmt.Printf(" %s\n", serial)
		}
	}
	for subject, pubKeyHashes := range rev.bySubjectPubKeyHash {
		fmt.Printf("%s\n", subject)
		for _, pubKeyHash := range pubKeyHashes {
			fmt.Printf("\t%s\n", pubKeyHash)
		}
	}
}
<|endoftext|>