<commit_before>// Package hammer provides a queued work consumer for CCP ESI API
package hammer

import (
	"log"
	"sync"

	"github.com/antihax/evedata/internal/apicache"
	"github.com/antihax/evedata/internal/redisqueue"
	"github.com/antihax/goesi"
	"github.com/garyburd/redigo/redis"
	nsq "github.com/nsqio/go-nsq"
)

// Hammer provides service control.
type Hammer struct {
	stop     chan bool
	hammerWG *sync.WaitGroup
	inQueue  *redisqueue.RedisQueue
	esi      *goesi.APIClient
	redis    *redis.Pool
	nsq      *nsq.Producer
	sem      chan bool
}

// NewHammer Service.
func NewHammer(redis *redis.Pool, nsq *nsq.Producer) *Hammer {
	// Get a caching http client
	cache := apicache.CreateHTTPClientCache(redis)

	// Create our ESI API Client
	esi := goesi.NewAPIClient(cache, "EVEData-API-Hammer")

	// Setup a new hammer
	s := &Hammer{
		stop:     make(chan bool),
		hammerWG: &sync.WaitGroup{},
		inQueue: redisqueue.NewRedisQueue(
			redis,
			"evedata-hammer",
		),
		nsq:   nsq,
		esi:   esi,
		redis: redis,
		sem:   make(chan bool, 100),
	}

	return s
}

// Close the hammer service
func (s *Hammer) Close() {
	close(s.stop)
	s.hammerWG.Wait()
}

// ChangeBasePath for ESI (sisi/mock/tranquility)
func (s *Hammer) ChangeBasePath(path string) {
	s.esi.ChangeBasePath(path)
}

// QueueWork directly
func (s *Hammer) QueueWork(work []redisqueue.Work) error {
	return s.inQueue.QueueWork(work)
}

// Run the hammer service
func (s *Hammer) Run() {
	for {
		select {
		case <-s.stop:
			return
		default:
			err := s.runConsumers()
			if err != nil {
				log.Println(err)
			}
		}
	}
}
<commit_msg>reduce to 50 per node<commit_after>// Package hammer provides a queued work consumer for CCP ESI API
package hammer

import (
	"log"
	"sync"

	"github.com/antihax/evedata/internal/apicache"
	"github.com/antihax/evedata/internal/redisqueue"
	"github.com/antihax/goesi"
	"github.com/garyburd/redigo/redis"
	nsq "github.com/nsqio/go-nsq"
)

// Hammer provides service control.
type Hammer struct {
	stop     chan bool
	hammerWG *sync.WaitGroup
	inQueue  *redisqueue.RedisQueue
	esi      *goesi.APIClient
	redis    *redis.Pool
	nsq      *nsq.Producer
	sem      chan bool
}

// NewHammer Service.
func NewHammer(redis *redis.Pool, nsq *nsq.Producer) *Hammer {
	// Get a caching http client
	cache := apicache.CreateHTTPClientCache(redis)

	// Create our ESI API Client
	esi := goesi.NewAPIClient(cache, "EVEData-API-Hammer")

	// Setup a new hammer
	s := &Hammer{
		stop:     make(chan bool),
		hammerWG: &sync.WaitGroup{},
		inQueue: redisqueue.NewRedisQueue(
			redis,
			"evedata-hammer",
		),
		nsq:   nsq,
		esi:   esi,
		redis: redis,
		sem:   make(chan bool, 50),
	}

	return s
}

// Close the hammer service
func (s *Hammer) Close() {
	close(s.stop)
	s.hammerWG.Wait()
}

// ChangeBasePath for ESI (sisi/mock/tranquility)
func (s *Hammer) ChangeBasePath(path string) {
	s.esi.ChangeBasePath(path)
}

// QueueWork directly
func (s *Hammer) QueueWork(work []redisqueue.Work) error {
	return s.inQueue.QueueWork(work)
}

// Run the hammer service
func (s *Hammer) Run() {
	for {
		select {
		case <-s.stop:
			return
		default:
			err := s.runConsumers()
			if err != nil {
				log.Println(err)
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package input

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/google/git-appraise/repository"
	"io/ioutil"
	"os"
	"os/exec"
)

// LaunchEditor launches the default editor configured for the given repo. This
// method blocks until the editor command has returned.
//
// The specified filename should be a temporary file and provided as a relative path
// from the repo (e.g. "FILENAME" will be converted to ".git/FILENAME"). This file
// will be deleted after the editor is closed and its contents have been read.
//
// This method returns the text that was read from the temporary file, or
// an error if any step in the process failed.
func LaunchEditor(repo repository.Repo, fileName string) (string, error) {
	editor, err := repo.GetCoreEditor()
	if err != nil {
		return "", fmt.Errorf("Unable to detect default git editor: %v\n", err)
	}

	path := fmt.Sprintf("%s/.git/%s", repo.GetPath(), fileName)

	cmd, err := startInlineCommand(editor, path)
	if err != nil {
		// Running the editor directly did not work. This might mean that
		// the editor string is not a path to an executable, but rather
		// a shell command (e.g. "emacsclient --tty"). As such, we'll try
		// to run the command through bash, and if that fails, try with sh
		args := []string{"-c", fmt.Sprintf("%s %q", editor, path)}
		cmd, err = startInlineCommand("bash", args...)
		if err != nil {
			cmd, err = startInlineCommand("sh", args...)
		}
	}
	if err != nil {
		return "", fmt.Errorf("Unable to start editor: %v\n", err)
	}

	if err := cmd.Wait(); err != nil {
		return "", fmt.Errorf("Editing finished with error: %v\n", err)
	}

	output, err := ioutil.ReadFile(path)
	if err != nil {
		os.Remove(path)
		return "", fmt.Errorf("Error reading edited file: %v\n", err)
	}
	os.Remove(path)
	return string(output), err
}

// FromFile loads and returns the contents of a given file. If - is passed
// through, much like git, it will read from stdin. This can be piped data,
// unless there is a tty in which case the user will be prompted to enter a
// message.
func FromFile(fileName string) (string, error) {
	if fileName == "-" {
		stat, _ := os.Stdin.Stat()
		if (stat.Mode() & os.ModeCharDevice) == 0 {
			// There is no tty. This will allow us to read piped data instead.
			output, err := ioutil.ReadAll(os.Stdin)
			if err != nil {
				return "", fmt.Errorf("Error reading from stdin: %v\n", err)
			}
			return string(output), err
		}

		fmt.Printf("(reading log message from standard input)\n")
		var output bytes.Buffer
		s := bufio.NewScanner(os.Stdin)
		for s.Scan() {
			output.Write(s.Bytes())
		}
		return output.String(), nil
	}

	output, err := ioutil.ReadFile(fileName)
	if err != nil {
		return "", fmt.Errorf("Error reading file: %v\n", err)
	}
	return string(output), err
}

func startInlineCommand(command string, args ...string) (*exec.Cmd, error) {
	cmd := exec.Command(command, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Start()
	return cmd, err
}
<commit_msg>Changed stdin prompt message<commit_after>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package input

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/google/git-appraise/repository"
	"io/ioutil"
	"os"
	"os/exec"
)

// LaunchEditor launches the default editor configured for the given repo. This
// method blocks until the editor command has returned.
//
// The specified filename should be a temporary file and provided as a relative path
// from the repo (e.g. "FILENAME" will be converted to ".git/FILENAME"). This file
// will be deleted after the editor is closed and its contents have been read.
//
// This method returns the text that was read from the temporary file, or
// an error if any step in the process failed.
func LaunchEditor(repo repository.Repo, fileName string) (string, error) {
	editor, err := repo.GetCoreEditor()
	if err != nil {
		return "", fmt.Errorf("Unable to detect default git editor: %v\n", err)
	}

	path := fmt.Sprintf("%s/.git/%s", repo.GetPath(), fileName)

	cmd, err := startInlineCommand(editor, path)
	if err != nil {
		// Running the editor directly did not work. This might mean that
		// the editor string is not a path to an executable, but rather
		// a shell command (e.g. "emacsclient --tty"). As such, we'll try
		// to run the command through bash, and if that fails, try with sh
		args := []string{"-c", fmt.Sprintf("%s %q", editor, path)}
		cmd, err = startInlineCommand("bash", args...)
		if err != nil {
			cmd, err = startInlineCommand("sh", args...)
		}
	}
	if err != nil {
		return "", fmt.Errorf("Unable to start editor: %v\n", err)
	}

	if err := cmd.Wait(); err != nil {
		return "", fmt.Errorf("Editing finished with error: %v\n", err)
	}

	output, err := ioutil.ReadFile(path)
	if err != nil {
		os.Remove(path)
		return "", fmt.Errorf("Error reading edited file: %v\n", err)
	}
	os.Remove(path)
	return string(output), err
}

// FromFile loads and returns the contents of a given file. If - is passed
// through, much like git, it will read from stdin. This can be piped data,
// unless there is a tty in which case the user will be prompted to enter a
// message.
func FromFile(fileName string) (string, error) {
	if fileName == "-" {
		stat, _ := os.Stdin.Stat()
		if (stat.Mode() & os.ModeCharDevice) == 0 {
			// There is no tty. This will allow us to read piped data instead.
			output, err := ioutil.ReadAll(os.Stdin)
			if err != nil {
				return "", fmt.Errorf("Error reading from stdin: %v\n", err)
			}
			return string(output), err
		}

		fmt.Printf("(reading comment from standard input)\n")
		var output bytes.Buffer
		s := bufio.NewScanner(os.Stdin)
		for s.Scan() {
			output.Write(s.Bytes())
		}
		return output.String(), nil
	}

	output, err := ioutil.ReadFile(fileName)
	if err != nil {
		return "", fmt.Errorf("Error reading file: %v\n", err)
	}
	return string(output), err
}

func startInlineCommand(command string, args ...string) (*exec.Cmd, error) {
	cmd := exec.Command(command, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Start()
	return cmd, err
}
<|endoftext|>


<commit_before>package rpc

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/streadway/amqp"
)

func (r *RPC) listen() {
	var attempt int

	for {
		select {
		case <-r.done:
			return
		case <-r.connect:
			if r.online == false {
				log.Println("Can not start listening while RPC should be offline")
				return
			}
			if attempt >= 60 {
				log.Println("attempt limit reached: 5")
				return
			}
			attempt++
			log.Printf("RPC: %s connect", r.name)
			go r.dial()

		case <-r.reconnect:
			if r.online == false {
				log.Println("Can not start listening while RPC should be offline")
				return
			}
			log.Printf("RPC: %s reconnect", r.name)
			timer := time.NewTimer(time.Second)
			<-timer.C
			go r.dial()
			timer.Stop()
		}
	}
}

func (r *RPC) dial() {
	log.Println("RPC: DIAL:", r.name)
	var err error

	if r.uri == "" {
		AMQP_USER := os.Getenv("AMQP_USER")
		AMQP_PASS := os.Getenv("AMQP_PASS")
		AMQP_HOST := os.Getenv("AMQP_HOST")
		AMQP_PORT := os.Getenv("AMQP_PORT")

		r.uri = fmt.Sprintf("amqp://%s:%s@%s:%s/", AMQP_USER, AMQP_PASS, AMQP_HOST, AMQP_PORT)
	}

	log.Println("RPC: Dial to:", r.uri)
	r.conn, err = amqp.Dial(r.uri)

	if err != nil {
		log.Println("RPC: Dial error", err)
		r.reconnect <- true
		return
	}

	go func() {
		if r.online == false {
			return
		}
		log.Println("RPC: Closing:", r.name, <-r.conn.NotifyClose(make(chan *amqp.Error)))
		r.connect <- true
	}()

	r.subscribe()
	r.connected <- true
}

func (r *RPC) call(s Sender, d Destination, p Receiver, data []byte) error {
	return r.publish(true, s, d, p, data)
}

func (r *RPC) cast(s Sender, d Destination, p Receiver, data []byte) error {
	return r.publish(false, s, d, p, data)
}

func (r *RPC) publish(call bool, s Sender, d Destination, p Receiver, data []byte) error {

	body, _ := r.encode(s, d, p, data)

	log.Printf("PRC: publish to %s:%s, proxy: %s:%s, send: %dB body (%s)", d.Name, d.UUID, p.Name, p.UUID, len(body), body)

	channel, err := r.conn.Channel()
	if err != nil {
		return fmt.Errorf("Channel: %s", err)
	}

	exchange := fmt.Sprintf("%s:%s", d.Name, "direct")
	if d.All {
		exchange = fmt.Sprintf("%s:%s", d.Name, "topic")
	}

	if p.Name != "" {
		exchange = fmt.Sprintf("%s:%s", p.Name, "direct")
		if d.All {
			exchange = fmt.Sprintf("%s:%s", p.Name, "topic")
		}
	}

	bind := d.UUID
	if bind == "" {
		bind = d.Name
	}

	if p.Name != "" {
		bind = p.Name
		if p.UUID != "" {
			bind = p.UUID
		}
	}

	if call {
		bind += ":call"
	} else {
		bind += ":cast"
	}

	bind = strings.ToLower(bind)
	log.Println("RPC: publish to exchange:", exchange, bind)

	if err := channel.Publish(exchange, bind, false, false, amqp.Publishing{
		ContentType: "application/json",
		Body:        body,
	},
	); err != nil {
		return fmt.Errorf("Exchange Publish: %s", err)
	}

	log.Println("RPC: published")
	return nil
}

func (r *RPC) subscribe() error {
	var err error
	var done = make(chan error)

	r.exchanges.direct = fmt.Sprintf("%s:%s", r.name, "direct")
	r.exchanges.topic = fmt.Sprintf("%s:%s", r.name, "topic")

	r.queues.direct = fmt.Sprintf("%s:%s", r.name, "direct")
	r.queues.topic = fmt.Sprintf("%s:%s", r.name, "topic")

	// Get hostname for register current instance
	log.Printf("RPC: Create new consumer: %s", r.name)

	r.channels.direct, err = r.conn.Channel()
	if err != nil {
		log.Println("Channel:", err)
		return err
	}

	err = r.channels.direct.Qos(r.limit, 0, false)
	if err != nil {
		log.Println("Channel:", err)
		return err
	}

	r.channels.topic, err = r.conn.Channel()
	if err != nil {
		log.Println("Channel:", err)
		return err
	}

	err = r.channels.topic.Qos(r.limit, 0, false)
	if err != nil {
		log.Println("Channel:", err)
		return err
	}

	// create direct exchange for guarantee delivery messages
	if err = r.channels.direct.ExchangeDeclare(r.exchanges.direct, "direct", true, false, false, false, nil); err != nil {
		return fmt.Errorf("Exchange Declare: %s", err)
	}

	// create topic exchange for non guarantee delivery messages
	if err = r.channels.topic.ExchangeDeclare(r.exchanges.topic, "topic", true, false, false, false, nil); err != nil {
		return fmt.Errorf("Exchange Declare: %s", err)
	}

	// create direct queue for guarantee delivery messages
	if _, err := r.channels.direct.QueueDeclare(r.queues.direct, true, false, false, false, nil); err != nil {
		return fmt.Errorf("Queue Declare: %s", err)
	}

	// create topic queue for non guarantee delivery messages
	if _, err := r.channels.topic.QueueDeclare(r.queues.topic, true, true, false, false, nil); err != nil {
		return fmt.Errorf("Queue Declare: %s", err)
	}

	// create bindings for direct messages
	if err = r.channels.direct.QueueBind(r.queues.direct, r.uuid+":call", r.exchanges.direct, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.direct.QueueBind(r.queues.direct, r.name+":call", r.exchanges.direct, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.direct.QueueBind(r.queues.direct, r.uuid+":call", r.exchanges.topic, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.direct.QueueBind(r.queues.direct, r.name+":call", r.exchanges.topic, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	// create bindings for topic messages
	if err = r.channels.topic.QueueBind(r.queues.topic, r.uuid+":cast", r.exchanges.topic, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.topic.QueueBind(r.queues.topic, r.name+":cast", r.exchanges.topic, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.topic.QueueBind(r.queues.topic, r.uuid+":cast", r.exchanges.direct, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.topic.QueueBind(r.queues.topic, r.name+":cast", r.exchanges.direct, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	messages, err := r.channels.direct.Consume(r.queues.direct, r.queues.direct, false, false, false, false, nil)
	if err != nil {
		return fmt.Errorf("Queue Consume: %s", err)
	}

	streams, err := r.channels.topic.Consume(r.queues.topic, r.queues.topic, false, false, false, false, nil)
	if err != nil {
		return fmt.Errorf("Queue Consume: %s", err)
	}

	go r.handle(messages, done)
	go r.handle(streams, done)

	return nil
}

func (r *RPC) handle(msgs <-chan amqp.Delivery, done chan error) {

	concurrent := 0
	last := make(chan bool)

	for d := range msgs {

		log.Println("RPC: message from:", d.DeliveryTag, d.ConsumerTag, string(d.Body))

		s, e, p, data, err := r.decode(d.Body)
		if err != nil {
			log.Println("RPC: message parsing failed: ", err)
			d.Ack(false)
		}

		go func() {
			if p.Name == "" {
				return
			}
			log.Println("PRC: need upstream", d.ConsumerTag)
			_, ok := r.upstreams[p.Handler]

			if !ok {
				log.Println("RPC: upstream not found", p.Handler)
				d.Ack(false)
				return
			}

			concurrent++
			err := r.upstreams[p.Handler](s, e, data)
			if err != nil {
				log.Println("RPC: Proxy error:", err)
			}

			d.Ack(false)
			concurrent--
			if concurrent == 0 {
				last <- true
			}

		}()

		go func() {

			if p.Name != "" {
				return
			}

			log.Println("PRC: send to handler", d.ConsumerTag)
			_, ok := r.handlers[e.Handler]
			if !ok {
				log.Println("RPC: handler not found", e.Handler)
				d.Ack(false)
				return
			}

			concurrent++
			err := r.handlers[e.Handler](s, data)
			if err != nil {
				log.Println("RPC: Proxy error:", err)
			}

			d.Ack(false)
			concurrent--
			if concurrent == 0 {
				last <- true
			}

		}()
	}

	if concurrent > 0 {
		select {
		case <-last:
			break
		}
	}

	fmt.Println("handle: deliveries channel closed")
	r.done <- nil
	return
}

func (r *RPC) cleanup() error {
	var err error
	err = r.channels.direct.ExchangeDelete(r.exchanges.direct, false, true)
	if err != nil {
		log.Println("Exchange remove error", err)
		return err
	}

	_, err = r.channels.direct.QueueDelete(r.queues.direct, false, false, true)
	if err != nil {
		log.Println("Queue remove error", err)
		return err
	}

	err = r.channels.topic.ExchangeDelete(r.exchanges.topic, false, true)
	if err != nil {
		log.Println("Exchange remove error", err)
		return err
	}

	return nil
}

func (r *RPC) shutdown() error {
	// will close() the deliveries channel
	log.Println("RPC: Shutdown broker")
	// close direct channels
	if err := r.channels.direct.Cancel(r.queues.direct, false); err != nil {
		return fmt.Errorf("Consumer cancel failed: %s", err)
	}

	if err := r.channels.topic.Cancel(r.queues.topic, false); err != nil {
		return fmt.Errorf("Consumer cancel failed: %s", err)
	}
	<-r.done
	if err := r.conn.Close(); err != nil {
		return fmt.Errorf("AMQP connection close error: %s", err)
	}

	defer fmt.Printf("AMQP shutdown OK")

	return nil
	// wait for handle() to exit

}
<commit_msg>Update rpc topic declaration<commit_after>package rpc

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/streadway/amqp"
)

func (r *RPC) listen() {
	var attempt int

	for {
		select {
		case <-r.done:
			return
		case <-r.connect:
			if r.online == false {
				log.Println("Can not start listening while RPC should be offline")
				return
			}
			if attempt >= 60 {
				log.Println("attempt limit reached: 5")
				return
			}
			attempt++
			log.Printf("RPC: %s connect", r.name)
			go r.dial()

		case <-r.reconnect:
			if r.online == false {
				log.Println("Can not start listening while RPC should be offline")
				return
			}
			log.Printf("RPC: %s reconnect", r.name)
			timer := time.NewTimer(time.Second)
			<-timer.C
			go r.dial()
			timer.Stop()
		}
	}
}

func (r *RPC) dial() {
	log.Println("RPC: DIAL:", r.name)
	var err error

	if r.uri == "" {
		AMQP_USER := os.Getenv("AMQP_USER")
		AMQP_PASS := os.Getenv("AMQP_PASS")
		AMQP_HOST := os.Getenv("AMQP_HOST")
		AMQP_PORT := os.Getenv("AMQP_PORT")

		r.uri = fmt.Sprintf("amqp://%s:%s@%s:%s/", AMQP_USER, AMQP_PASS, AMQP_HOST, AMQP_PORT)
	}

	log.Println("RPC: Dial to:", r.uri)
	r.conn, err = amqp.Dial(r.uri)

	if err != nil {
		log.Println("RPC: Dial error", err)
		r.reconnect <- true
		return
	}

	go func() {
		if r.online == false {
			return
		}
		log.Println("RPC: Closing:", r.name, <-r.conn.NotifyClose(make(chan *amqp.Error)))
		r.connect <- true
	}()

	r.subscribe()
	r.connected <- true
}

func (r *RPC) call(s Sender, d Destination, p Receiver, data []byte) error {
	return r.publish(true, s, d, p, data)
}

func (r *RPC) cast(s Sender, d Destination, p Receiver, data []byte) error {
	return r.publish(false, s, d, p, data)
}

func (r *RPC) publish(call bool, s Sender, d Destination, p Receiver, data []byte) error {

	body, _ := r.encode(s, d, p, data)

	log.Printf("PRC: publish to %s:%s, proxy: %s:%s, send: %dB body (%s)", d.Name, d.UUID, p.Name, p.UUID, len(body), body)

	channel, err := r.conn.Channel()
	if err != nil {
		return fmt.Errorf("Channel: %s", err)
	}

	exchange := fmt.Sprintf("%s:%s", d.Name, "direct")
	if d.All {
		exchange = fmt.Sprintf("%s:%s", d.Name, "topic")
	}

	if p.Name != "" {
		exchange = fmt.Sprintf("%s:%s", p.Name, "direct")
		if d.All {
			exchange = fmt.Sprintf("%s:%s", p.Name, "topic")
		}
	}

	bind := d.UUID
	if bind == "" {
		bind = d.Name
	}

	if p.Name != "" {
		bind = p.Name
		if p.UUID != "" {
			bind = p.UUID
		}
	}

	if call {
		bind += ":call"
	} else {
		bind += ":cast"
	}

	bind = strings.ToLower(bind)
	log.Println("RPC: publish to exchange:", exchange, bind)

	if err := channel.Publish(exchange, bind, false, false, amqp.Publishing{
		ContentType: "application/json",
		Body:        body,
	},
	); err != nil {
		return fmt.Errorf("Exchange Publish: %s", err)
	}

	log.Println("RPC: published")
	return nil
}

func (r *RPC) subscribe() error {
	var err error
	var done = make(chan error)

	r.exchanges.direct = fmt.Sprintf("%s:%s", r.name, "direct")
	r.exchanges.topic = fmt.Sprintf("%s:%s", r.name, "topic")

	r.queues.direct = fmt.Sprintf("%s:%s", r.name, "direct")
	r.queues.topic = fmt.Sprintf("%s:%s", r.name, "topic")

	// Get hostname for register current instance
	log.Printf("RPC: Create new consumer: %s", r.name)

	r.channels.direct, err = r.conn.Channel()
	if err != nil {
		log.Println("Channel:", err)
		return err
	}

	err = r.channels.direct.Qos(r.limit, 0, false)
	if err != nil {
		log.Println("Channel:", err)
		return err
	}

	r.channels.topic, err = r.conn.Channel()
	if err != nil {
		log.Println("Channel:", err)
		return err
	}

	err = r.channels.topic.Qos(r.limit, 0, false)
	if err != nil {
		log.Println("Channel:", err)
		return err
	}

	// create direct exchange for guarantee delivery messages
	if err = r.channels.direct.ExchangeDeclare(r.exchanges.direct, "direct", true, false, false, false, nil); err != nil {
		return fmt.Errorf("Exchange Declare: %s", err)
	}

	// create topic exchange for non guarantee delivery messages
	if err = r.channels.topic.ExchangeDeclare(r.exchanges.topic, "topic", true, false, false, false, nil); err != nil {
		return fmt.Errorf("Exchange Declare: %s", err)
	}

	// create direct queue for guarantee delivery messages
	if _, err := r.channels.direct.QueueDeclare(r.queues.direct, true, false, false, false, nil); err != nil {
		return fmt.Errorf("Queue Declare: %s", err)
	}

	// create topic queue for non guarantee delivery messages
	if _, err := r.channels.topic.QueueDeclare(r.queues.topic, true, true, false, false, nil); err != nil {
		return fmt.Errorf("Queue Declare: %s", err)
	}

	// create bindings for direct messages
	if err = r.channels.direct.QueueBind(r.queues.direct, r.uuid+":call", r.exchanges.direct, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.direct.QueueBind(r.queues.direct, r.name+":call", r.exchanges.direct, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.direct.QueueBind(r.queues.direct, r.uuid+":call", r.exchanges.topic, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.direct.QueueBind(r.queues.direct, r.name+":call", r.exchanges.topic, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	// create bindings for topic messages
	if err = r.channels.topic.QueueBind(r.queues.topic, r.uuid+":cast", r.exchanges.topic, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.topic.QueueBind(r.queues.topic, r.uuid+":cast", r.exchanges.direct, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = r.channels.topic.QueueBind(r.queues.topic, r.name+":cast", r.exchanges.direct, false, nil); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	messages, err := r.channels.direct.Consume(r.queues.direct, r.queues.direct, false, false, false, false, nil)
	if err != nil {
		return fmt.Errorf("Queue Consume: %s", err)
	}

	streams, err := r.channels.topic.Consume(r.queues.topic, r.queues.topic, false, false, false, false, nil)
	if err != nil {
		return fmt.Errorf("Queue Consume: %s", err)
	}

	go r.handle(messages, done)
	go r.handle(streams, done)

	return nil
}

func (r *RPC) handle(msgs <-chan amqp.Delivery, done chan error) {

	concurrent := 0
	last := make(chan bool)

	for d := range msgs {

		log.Println("RPC: message from:", d.DeliveryTag, d.ConsumerTag, string(d.Body))

		s, e, p, data, err := r.decode(d.Body)
		if err != nil {
			log.Println("RPC: message parsing failed: ", err)
			d.Ack(false)
		}

		go func() {
			if p.Name == "" {
				return
			}
			log.Println("PRC: need upstream", d.ConsumerTag)
			_, ok := r.upstreams[p.Handler]

			if !ok {
				log.Println("RPC: upstream not found", p.Handler)
				d.Ack(false)
				return
			}

			concurrent++
			err := r.upstreams[p.Handler](s, e, data)
			if err != nil {
				log.Println("RPC: Proxy error:", err)
			}

			d.Ack(false)
			concurrent--
			if concurrent == 0 {
				last <- true
			}

		}()

		go func() {

			if p.Name != "" {
				return
			}

			log.Println("PRC: send to handler", d.ConsumerTag)
			_, ok := r.handlers[e.Handler]
			if !ok {
				log.Println("RPC: handler not found", e.Handler)
				d.Ack(false)
				return
			}

			concurrent++
			err := r.handlers[e.Handler](s, data)
			if err != nil {
				log.Println("RPC: Proxy error:", err)
			}

			d.Ack(false)
			concurrent--
			if concurrent == 0 {
				last <- true
			}

		}()
	}

	if concurrent > 0 {
		select {
		case <-last:
			break
		}
	}

	fmt.Println("handle: deliveries channel closed")
	r.done <- nil
	return
}

func (r *RPC) cleanup() error {
	var err error
	err = r.channels.direct.ExchangeDelete(r.exchanges.direct, false, true)
	if err != nil {
		log.Println("Exchange remove error", err)
		return err
	}

	_, err = r.channels.direct.QueueDelete(r.queues.direct, false, false, true)
	if err != nil {
		log.Println("Queue remove error", err)
		return err
	}

	err = r.channels.topic.ExchangeDelete(r.exchanges.topic, false, true)
	if err != nil {
		log.Println("Exchange remove error", err)
		return err
	}

	return nil
}

func (r *RPC) shutdown() error {
	// will close() the deliveries channel
	log.Println("RPC: Shutdown broker")
	// close direct channels
	if err := r.channels.direct.Cancel(r.queues.direct, false); err != nil {
		return fmt.Errorf("Consumer cancel failed: %s", err)
	}

	if err := r.channels.topic.Cancel(r.queues.topic, false); err != nil {
		return fmt.Errorf("Consumer cancel failed: %s", err)
	}
	<-r.done
	if err := r.conn.Close(); err != nil {
		return fmt.Errorf("AMQP connection close error: %s", err)
	}

	defer fmt.Printf("AMQP shutdown OK")

	return nil
	// wait for handle() to exit

}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"github.com/codegangsta/cli"
	"github.com/headmade/backuper/config"
)

func providerAction(c *cli.Context) {
	var providerConfig config.Provider
	switch c.Command.Name {
	case "AWS":
		validateArgs(c, 2)
		providerConfig = config.Provider{"AWS_ACCESS_KEY_ID": c.Args()[0], "AWS_SECRET_ACCESS_KEY": c.Args()[1]}
	case "encryption":
		validateArgs(c, 1)
		providerConfig = config.Provider{"pass": c.Args()[0]}
	}
	providerCommand(c.Command.Name, providerConfig)
}

func validateArgs(c *cli.Context, length int) {
	if len(c.Args()) != length {
		log.Fatal("Bad arguments")
	}
}

func providerCommand(name string, providerConfig config.Provider) {
	conf, err := config.New()
	if err != nil {
		log.Fatal(err)
	}
	if conf.Secret == nil {
		conf.Secret = config.Providers{}
	}
	// conf.Secret[name] = providerConfig
	if conf.Secret[name] == nil {
		conf.Secret[name] = config.Provider{}
	}
	for k, v := range providerConfig {
		conf.Secret[name][k] = v
	}
	conf.Write(conf.Secret)
}
<commit_msg>Add bucket to p command<commit_after>package main

import (
	"log"

	"github.com/codegangsta/cli"
	"github.com/headmade/backuper/config"
)

func providerAction(c *cli.Context) {
	var providerConfig config.Provider
	switch c.Command.Name {
	case "AWS":
		validateArgs(c, 3)
		providerConfig = config.Provider{"bucket": c.Args()[0], "AWS_ACCESS_KEY_ID": c.Args()[1], "AWS_SECRET_ACCESS_KEY": c.Args()[2]}
	case "encryption":
		validateArgs(c, 1)
		providerConfig = config.Provider{"pass": c.Args()[0]}
	}
	providerCommand(c.Command.Name, providerConfig)
}

func validateArgs(c *cli.Context, length int) {
	if len(c.Args()) != length {
		log.Fatal("Bad arguments")
	}
}

func providerCommand(name string, providerConfig config.Provider) {
	conf, err := config.New()
	if err != nil {
		log.Fatal(err)
	}
	if conf.Secret == nil {
		conf.Secret = config.Providers{}
	}
	// conf.Secret[name] = providerConfig
	if conf.Secret[name] == nil {
		conf.Secret[name] = config.Provider{}
	}
	for k, v := range providerConfig {
		conf.Secret[name][k] = v
	}
	conf.Write(conf.Secret)
}
<|endoftext|>


<commit_before>package handler

import (
	"fmt"
	"fullerite/metric"
	"net"
	"sort"
	"time"

	l "github.com/Sirupsen/logrus"
)

func init() {
	RegisterHandler("Graphite", newGraphite)
}

// Graphite type
type Graphite struct {
	BaseHandler
	server string
	port   string
}

// newGraphite returns a new Graphite handler.
func newGraphite(
	channel chan metric.Metric,
	initialInterval int,
	initialBufferSize int,
	initialTimeout time.Duration,
	log *l.Entry) Handler {

	inst := new(Graphite)
	inst.name = "Graphite"

	inst.interval = initialInterval
	inst.maxBufferSize = initialBufferSize
	inst.timeout = initialTimeout
	inst.log = log
	inst.channel = channel

	return inst
}

// Server returns the Graphite server's name or IP
func (g Graphite) Server() string {
	return g.server
}

// Port returns the Graphite server's port number
func (g Graphite) Port() string {
	return g.port
}

// Configure accepts the different configuration options for the Graphite handler
func (g *Graphite) Configure(configMap map[string]interface{}) {
	if server, exists := configMap["server"]; exists {
		g.server = server.(string)
	} else {
		g.log.Error("There was no server specified for the Graphite Handler, there won't be any emissions")
	}

	if port, exists := configMap["port"]; exists {
		g.port = fmt.Sprint(port)
	} else {
		g.log.Error("There was no port specified for the Graphite Handler, there won't be any emissions")
	}
	g.configureCommonParams(configMap)
}

// Run runs the handler main loop
func (g *Graphite) Run() {
	g.run(g.emitMetrics)
}

func (g Graphite) convertToGraphite(incomingMetric metric.Metric) (datapoint string) {
	//orders dimensions so datapoint keeps consistent name
	var keys []string
	dimensions := incomingMetric.GetDimensions(g.DefaultDimensions())
	for k := range dimensions {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	datapoint = g.Prefix() + incomingMetric.Name
	for _, key := range keys {
		datapoint = fmt.Sprintf("%s.%s.%s", datapoint, key, dimensions[key])
	}
	datapoint = fmt.Sprintf("%s %f %s\n", datapoint, incomingMetric.Value, incomingMetric.MetricTime)
	return datapoint
}

func (g *Graphite) emitMetrics(metrics []metric.Metric) bool {
	g.log.Info("Starting to emit ", len(metrics), " metrics")

	if len(metrics) == 0 {
		g.log.Warn("Skipping send because of an empty payload")
		return false
	}

	addr := fmt.Sprintf("%s:%s", g.server, g.port)
	conn, err := net.DialTimeout("tcp", addr, g.timeout)
	if err != nil {
		g.log.Error("Failed to connect ", addr)
		return false
	}

	for _, m := range metrics {
		fmt.Fprintf(conn, g.convertToGraphite(m))
	}
	return true
}
<commit_msg>use Unix timestamp<commit_after>package handler

import (
	"fmt"
	"fullerite/metric"
	"net"
	"sort"
	"time"

	l "github.com/Sirupsen/logrus"
)

func init() {
	RegisterHandler("Graphite", newGraphite)
}

// Graphite type
type Graphite struct {
	BaseHandler
	server string
	port   string
}

// newGraphite returns a new Graphite handler.
func newGraphite(
	channel chan metric.Metric,
	initialInterval int,
	initialBufferSize int,
	initialTimeout time.Duration,
	log *l.Entry) Handler {

	inst := new(Graphite)
	inst.name = "Graphite"

	inst.interval = initialInterval
	inst.maxBufferSize = initialBufferSize
	inst.timeout = initialTimeout
	inst.log = log
	inst.channel = channel

	return inst
}

// Server returns the Graphite server's name or IP
func (g Graphite) Server() string {
	return g.server
}

// Port returns the Graphite server's port number
func (g Graphite) Port() string {
	return g.port
}

// Configure accepts the different configuration options for the Graphite handler
func (g *Graphite) Configure(configMap map[string]interface{}) {
	if server, exists := configMap["server"]; exists {
		g.server = server.(string)
	} else {
		g.log.Error("There was no server specified for the Graphite Handler, there won't be any emissions")
	}

	if port, exists := configMap["port"]; exists {
		g.port = fmt.Sprint(port)
	} else {
		g.log.Error("There was no port specified for the Graphite Handler, there won't be any emissions")
	}
	g.configureCommonParams(configMap)
}

// Run runs the handler main loop
func (g *Graphite) Run() {
	g.run(g.emitMetrics)
}

func (g Graphite) convertToGraphite(incomingMetric metric.Metric) (datapoint string) {
	//orders dimensions so datapoint keeps consistent name
	var keys []string
	dimensions := incomingMetric.GetDimensions(g.DefaultDimensions())
	for k := range dimensions {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	datapoint = g.Prefix() + incomingMetric.Name
	for _, key := range keys {
		datapoint = fmt.Sprintf("%s.%s.%s", datapoint, key, dimensions[key])
	}
	datapoint = fmt.Sprintf("%s %f %d\n", datapoint, incomingMetric.Value, incomingMetric.MetricTime.Unix())
	return datapoint
}

func (g *Graphite) emitMetrics(metrics []metric.Metric) bool {
	g.log.Info("Starting to emit ", len(metrics), " metrics")

	if len(metrics) == 0 {
		g.log.Warn("Skipping send because of an empty payload")
		return false
	}

	addr := fmt.Sprintf("%s:%s", g.server, g.port)
	conn, err := net.DialTimeout("tcp", addr, g.timeout)
	if err != nil {
		g.log.Error("Failed to connect ", addr)
		return false
	}

	for _, m := range metrics {
		fmt.Fprintf(conn, g.convertToGraphite(m))
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package formats

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"golang.org/x/net/context"

	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/data/text/indented"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/resultdb/pbutil"
	pb "go.chromium.org/luci/resultdb/proto/rpc/v1"
)

const testNamePrefixKey = "test_name_prefix"

var testRunSubdirRe = regexp.MustCompile("/retry_([0-9]+)/")

// JSONTestResults represents the structure in
// https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md
//
// Deprecated fields and fields not used by Test Results are omitted.
type JSONTestResults struct {
	Interrupted bool `json:"interrupted"`

	PathDelimiter string `json:"path_delimiter"`

	TestsRaw json.RawMessage `json:"tests"`
	Tests    map[string]*TestFields

	Version int32 `json:"version"`

	ArtifactTypes map[string]string `json:"artifact_types"`

	BuildNumber string `json:"build_number"`
	BuilderName string `json:"builder_name"`

	// Metadata associated with results, which may include a list of expectation_files, or
	// test_name_prefix e.g. in GPU tests (distinct from test_path_prefix passed in the recorder API
	// request).
	Metadata map[string]json.RawMessage `json:"metadata"`
}

// TestFields represents the test fields structure in
// https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md
//
// Deprecated fields and fields not used by Test Results are omitted.
type TestFields struct {
	Actual   string `json:"actual"`
	Expected string `json:"expected"`

	Artifacts map[string][]string `json:"artifacts"`

	Time  float64   `json:"time"`
	Times []float64 `json:"times"`
}

// ConvertFromJSON converts a JSON of test results in the JSON Test Results
// format to the internal struct format.
//
// The receiver is cleared and its fields overwritten.
func (r *JSONTestResults) ConvertFromJSON(ctx context.Context, reader io.Reader) error {
	*r = JSONTestResults{}
	if err := json.NewDecoder(reader).Decode(r); err != nil {
		return err
	}

	// Convert Tests and return.
	if err := r.convertTests("", r.TestsRaw); err != nil {
		return err
	}
	return nil
}

// ToProtos converts test results in r []*pb.TestResult and updates inv
// in-place accordingly.
// If an error is returned, inv is left unchanged.
//
// Takes outputsToProcess, the isolated outputs associated with the task, to use to populate
// artifacts, and deletes any that are successfully processed.
// Does not populate TestResult.Name; that happens server-side on RPC response.
func (r *JSONTestResults) ToProtos(ctx context.Context, testPathPrefix string, inv *pb.Invocation, outputsToProcess map[string]*pb.Artifact) ([]*pb.TestResult, error) {
	if r.Version != 3 {
		return nil, errors.Reason("unknown JSON Test Results version %d", r.Version).Err()
	}

	// Sort the test name to make the output deterministic.
	testNames := make([]string, 0, len(r.Tests))
	for name := range r.Tests {
		testNames = append(testNames, name)
	}
	sort.Strings(testNames)

	ret := make([]*pb.TestResult, 0, len(r.Tests))
	for _, name := range testNames {
		testPath := testPathPrefix + name

		// Populate protos.
		unresolvedOutputs, err := r.Tests[name].toProtos(ctx, &ret, testPath, outputsToProcess)
		if err != nil {
			return nil, errors.Annotate(err, "test %q failed to convert run fields", name).Err()
		}

		// If any outputs cannot be processed, don't cause the rest of processing to fail, but do log.
		if len(unresolvedOutputs) > 0 {
			logging.Errorf(ctx,
				"Test %s could not generate artifact protos for the following:\n%s",
				testPath,
				artifactsToString(unresolvedOutputs))
		}
	}

	// Get tags from metadata if any.
	tags, err := r.extractTags()
	if err != nil {
		return nil, err
	}

	// The code below does not return errors, so it is safe to make in-place
	// modifications of inv.

	if r.Interrupted {
		inv.State = pb.Invocation_INTERRUPTED
	}

	inv.Tags = append(inv.Tags, pbutil.StringPair(OriginalFormatTagKey, FormatJTR))
	for _, tag := range tags {
		inv.Tags = append(inv.Tags, pbutil.StringPair("json_format_tag", tag))
	}
	if r.BuildNumber != "" {
		inv.Tags = append(inv.Tags, pbutil.StringPair("build_number", r.BuildNumber))
	}

	pbutil.NormalizeInvocation(inv)
	return ret, nil
}

// convertTests converts the trie of tests.
func (r *JSONTestResults) convertTests(curPath string, curNode json.RawMessage) error {
	// curNode should certainly be a map.
	var maybeNode map[string]json.RawMessage
	if err := json.Unmarshal(curNode, &maybeNode); err != nil {
		return errors.Annotate(err, "%q not map[string]json.RawMessage", curNode).Err()
	}

	// Convert the tree.
	for key, value := range maybeNode {
		// Set up test path.
		delim := "/"
		testPath := key
		if r.PathDelimiter != "" {
			delim = r.PathDelimiter
		}

		if curPath != "" {
			testPath = fmt.Sprintf("%s%s%s", curPath, delim, key)
		} else {
			if prefixJSON, ok := r.Metadata[testNamePrefixKey]; ok {
				var prefix string
				if err := json.Unmarshal(prefixJSON, &prefix); err != nil {
					return errors.Annotate(err, "%s not string, got %q", testNamePrefixKey, prefixJSON).Err()
				}
				testPath = prefix + key
			}
		}

		// Try to unmarshal value to TestFields. We check success by checking fields we expect to
		// be populated.
		maybeFields := &TestFields{}
		json.Unmarshal(value, maybeFields)
		if maybeFields.Actual != "" && maybeFields.Expected != "" {
			if r.Tests == nil {
				r.Tests = make(map[string]*TestFields)
			}
			r.Tests[testPath] = maybeFields
			continue
		}

		// Otherwise, try to process it as an intermediate node.
		if err := r.convertTests(testPath, value); err != nil {
			return errors.Annotate(err, "error attempting conversion of %q as intermediated node", value).Err()
		}
	}
	return nil
}

// extractTags tries to read the optional "tags" field in "metadata" as a slice of strings.
func (r *JSONTestResults) extractTags() ([]string, error) {
	maybeTags, ok := r.Metadata["tags"]
	if !ok {
		return nil, nil
	}

	var tags []string
	if err := json.Unmarshal(maybeTags, &tags); err != nil {
		return nil, errors.Annotate(err, "tags not []string, got %q", maybeTags).Err()
	}

	return tags, nil
}

func fromJSONStatus(s string) (pb.TestStatus, error) {
	switch s {
	case "CRASH":
		return pb.TestStatus_CRASH, nil
	case "FAIL":
		return pb.TestStatus_FAIL, nil
	case "PASS":
		return pb.TestStatus_PASS, nil
	case "SKIP":
		return pb.TestStatus_SKIP, nil
	case "TIMEOUT":
		return pb.TestStatus_ABORT, nil

	// The below are web test-specific statuses. They are officially deprecated, but in practice
	// still generated by the tests and should be converted.
	case "IMAGE", "TEXT", "IMAGE+TEXT", "AUDIO", "LEAK", "MISSING":
		return pb.TestStatus_FAIL, nil

	default:
		// There are a number of web test-specific statuses not handled here as they are deprecated.
		return 0, errors.Reason("unknown or unexpected JSON Test Format status %s", s).Err()
	}
}

// testArtifactsPerRun maps a run index to a map of run index to slice of
// associated *pb.Artifacts.
type testArtifactsPerRun map[int][]*pb.Artifact

// toProtos converts the TestFields into zero or more pb.TestResult and
// appends them to dest.
//
// Any artifacts that could not be processed are returned.
// TODO(jchinlee): once we've curated the artifacts to process, make unprocessed artifacts error.
func (f *TestFields) toProtos(ctx context.Context, dest *[]*pb.TestResult, testPath string, outputsToProcess map[string]*pb.Artifact) (map[string][]string, error) {
	// Process statuses.
	actualStatuses := strings.Split(f.Actual, " ")
	expectedSet := stringset.NewFromSlice(strings.Split(f.Expected, " ")...)

	// Process times.
	// Time and Times are both optional, but if Times is present, its length should match the number
	// of runs. Otherwise we have only Time as the duration of the first run.
	if len(f.Times) > 0 && len(f.Times) != len(actualStatuses) {
		return nil, errors.Reason(
			"%d durations populated but has %d test statuses; should match",
			len(f.Times), len(actualStatuses)).Err()
	}

	var durations []float64
	if len(f.Times) > 0 {
		durations = f.Times
	} else if f.Time != 0 { // Do not set duration if it is unknown.
		durations = []float64{f.Time}
	}

	// Get artifacts.
	// We expect that if we have any artifacts, the number of runs from deriving the artifacts
	// should match the number of actual runs. Because the arts are a map from run index to
	// *pb.Artifacts slice, we will not error if artifacts are missing for a run, but log a warning
	// in case the number of runs do not match each other for further investigation.
	arts, unresolved := f.getArtifacts(outputsToProcess)
	if len(arts) > 0 && len(actualStatuses) != len(arts) {
		logging.Warningf(ctx,
			"Number of runs of test %s (%d) does not match number of runs generated from artifacts (%d)",
			len(actualStatuses), len(arts), testPath)
	}

	// Populate protos.
	for i, runStatus := range actualStatuses {
		status, err := fromJSONStatus(runStatus)
		if err != nil {
			return nil, err
		}

		tr := &pb.TestResult{
			TestPath:        testPath,
			Expected:        expectedSet.Has(runStatus),
			Status:          status,
			Tags:            pbutil.StringPairs("json_format_status", runStatus),
			OutputArtifacts: arts[i],
		}

		if i < len(durations) {
			tr.Duration = secondsToDuration(durations[i])
		}

		pbutil.NormalizeTestResult(tr)
		*dest = append(*dest, tr)
	}

	return unresolved, nil
}

// getArtifacts gets pb.Artifacts corresponding to the TestField's artifacts.
//
// It tries to derive the pb.Artifacts in the following order:
//   - look for them in the isolated outputs represented as pb.Artifacts
//   - check if they're a known special case
//   - fail to process and mark them as `unresolvedArtifacts`
func (f *TestFields) getArtifacts(outputsToProcess map[string]*pb.Artifact) (artifacts testArtifactsPerRun, unresolvedArtifacts map[string][]string) {
	artifacts = testArtifactsPerRun{}
	unresolvedArtifacts = map[string][]string{}

	for name, paths := range f.Artifacts {
		for i, path := range paths {
			// Get the run ID of the artifact. Defaults to 0 (i.e. assumes there is only one run).
			runID, err := artifactRunID(path)
			if err != nil {
				unresolvedArtifacts[name] = append(unresolvedArtifacts[name], path)
				continue
			}

			// Look for the path in isolated outputs.
			if art, ok := outputsToProcess[path]; ok {
				artifacts[runID] = append(artifacts[runID], art)
				delete(outputsToProcess, path)
				continue
			}

			// If the name is otherwise understood by ResultDB, process it.
			// So far, that's only gold_triage_links.
			if name == "gold_triage_link" {
				// We don't expect more than one triage link per test run, but if there is more than one,
				// suffix the name with index to ensure we retain it too.
				if i > 0 {
					name = fmt.Sprintf("%s_%d", name, i)
				}

				artifacts[runID] = append(artifacts[runID], &pb.Artifact{Name: name, ViewUrl: path})
				continue
			}

			// Otherwise, could not populate artifact, so mark it as unresolved.
			unresolvedArtifacts[name] = append(unresolvedArtifacts[name], path)
		}
	}

	return
}

// artifactRunID extracts a run ID, defaulting to 0, or error if it doesn't recognize the format.
func artifactRunID(path string) (int, error) {
	if m := testRunSubdirRe.FindStringSubmatch(path); m != nil {
		return strconv.Atoi(m[1])
	}

	// No retry_<i> subdirectory, so assume it's the first/0th run.
	return 0, nil
}

// artifactsToString converts the given name->paths artifacts map to a string for logging.
func artifactsToString(arts map[string][]string) string {
	names := make([]string, 0, len(arts))
	for name := range arts {
		names = append(names, name)
	}
	sort.Strings(names)

	var msg bytes.Buffer
	w := &indented.Writer{Writer: &msg}
	for _, name := range names {
		fmt.Fprintln(w, name)
		w.Level++
		for _, p := range arts[name] {
			fmt.Fprintln(w, p)
		}
		w.Level--
	}
	return msg.String()
}
<commit_msg>[resultdb] Support "triage_link_for_entire_cl" Gold link as artifact.<commit_after>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package formats

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"golang.org/x/net/context"

	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/data/text/indented"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/resultdb/pbutil"
	pb "go.chromium.org/luci/resultdb/proto/rpc/v1"
)

const testNamePrefixKey = "test_name_prefix"

var testRunSubdirRe = regexp.MustCompile("/retry_([0-9]+)/")

// JSONTestResults represents the structure in
// https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md
//
// Deprecated fields and fields not used by Test Results are omitted.
type JSONTestResults struct {
	Interrupted bool `json:"interrupted"`

	PathDelimiter string `json:"path_delimiter"`

	TestsRaw json.RawMessage `json:"tests"`
	Tests    map[string]*TestFields

	Version int32 `json:"version"`

	ArtifactTypes map[string]string `json:"artifact_types"`

	BuildNumber string `json:"build_number"`
	BuilderName string `json:"builder_name"`

	// Metadata associated with results, which may include a list of expectation_files, or
	// test_name_prefix e.g. in GPU tests (distinct from test_path_prefix passed in the recorder API
	// request).
	Metadata map[string]json.RawMessage `json:"metadata"`
}

// TestFields represents the test fields structure in
// https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md
//
// Deprecated fields and fields not used by Test Results are omitted.
type TestFields struct {
	Actual   string `json:"actual"`
	Expected string `json:"expected"`

	Artifacts map[string][]string `json:"artifacts"`

	Time  float64   `json:"time"`
	Times []float64 `json:"times"`
}

// ConvertFromJSON converts a JSON of test results in the JSON Test Results
// format to the internal struct format.
//
// The receiver is cleared and its fields overwritten.
func (r *JSONTestResults) ConvertFromJSON(ctx context.Context, reader io.Reader) error {
	*r = JSONTestResults{}
	if err := json.NewDecoder(reader).Decode(r); err != nil {
		return err
	}

	// Convert Tests and return.
	if err := r.convertTests("", r.TestsRaw); err != nil {
		return err
	}
	return nil
}

// ToProtos converts test results in r []*pb.TestResult and updates inv
// in-place accordingly.
// If an error is returned, inv is left unchanged.
//
// Takes outputsToProcess, the isolated outputs associated with the task, to use to populate
// artifacts, and deletes any that are successfully processed.
// Does not populate TestResult.Name; that happens server-side on RPC response.
func (r *JSONTestResults) ToProtos(ctx context.Context, testPathPrefix string, inv *pb.Invocation, outputsToProcess map[string]*pb.Artifact) ([]*pb.TestResult, error) {
	if r.Version != 3 {
		return nil, errors.Reason("unknown JSON Test Results version %d", r.Version).Err()
	}

	// Sort the test name to make the output deterministic.
	testNames := make([]string, 0, len(r.Tests))
	for name := range r.Tests {
		testNames = append(testNames, name)
	}
	sort.Strings(testNames)

	ret := make([]*pb.TestResult, 0, len(r.Tests))
	for _, name := range testNames {
		testPath := testPathPrefix + name

		// Populate protos.
		unresolvedOutputs, err := r.Tests[name].toProtos(ctx, &ret, testPath, outputsToProcess)
		if err != nil {
			return nil, errors.Annotate(err, "test %q failed to convert run fields", name).Err()
		}

		// If any outputs cannot be processed, don't cause the rest of processing to fail, but do log.
		if len(unresolvedOutputs) > 0 {
			logging.Errorf(ctx,
				"Test %s could not generate artifact protos for the following:\n%s",
				testPath,
				artifactsToString(unresolvedOutputs))
		}
	}

	// Get tags from metadata if any.
	tags, err := r.extractTags()
	if err != nil {
		return nil, err
	}

	// The code below does not return errors, so it is safe to make in-place
	// modifications of inv.

	if r.Interrupted {
		inv.State = pb.Invocation_INTERRUPTED
	}

	inv.Tags = append(inv.Tags, pbutil.StringPair(OriginalFormatTagKey, FormatJTR))
	for _, tag := range tags {
		inv.Tags = append(inv.Tags, pbutil.StringPair("json_format_tag", tag))
	}
	if r.BuildNumber != "" {
		inv.Tags = append(inv.Tags, pbutil.StringPair("build_number", r.BuildNumber))
	}

	pbutil.NormalizeInvocation(inv)
	return ret, nil
}

// convertTests converts the trie of tests.
func (r *JSONTestResults) convertTests(curPath string, curNode json.RawMessage) error {
	// curNode should certainly be a map.
	var maybeNode map[string]json.RawMessage
	if err := json.Unmarshal(curNode, &maybeNode); err != nil {
		return errors.Annotate(err, "%q not map[string]json.RawMessage", curNode).Err()
	}

	// Convert the tree.
	for key, value := range maybeNode {
		// Set up test path.
		delim := "/"
		testPath := key
		if r.PathDelimiter != "" {
			delim = r.PathDelimiter
		}

		if curPath != "" {
			testPath = fmt.Sprintf("%s%s%s", curPath, delim, key)
		} else {
			if prefixJSON, ok := r.Metadata[testNamePrefixKey]; ok {
				var prefix string
				if err := json.Unmarshal(prefixJSON, &prefix); err != nil {
					return errors.Annotate(err, "%s not string, got %q", testNamePrefixKey, prefixJSON).Err()
				}
				testPath = prefix + key
			}
		}

		// Try to unmarshal value to TestFields. We check success by checking fields we expect to
		// be populated.
		maybeFields := &TestFields{}
		json.Unmarshal(value, maybeFields)
		if maybeFields.Actual != "" && maybeFields.Expected != "" {
			if r.Tests == nil {
				r.Tests = make(map[string]*TestFields)
			}
			r.Tests[testPath] = maybeFields
			continue
		}

		// Otherwise, try to process it as an intermediate node.
		if err := r.convertTests(testPath, value); err != nil {
			return errors.Annotate(err, "error attempting conversion of %q as intermediated node", value).Err()
		}
	}
	return nil
}

// extractTags tries to read the optional "tags" field in "metadata" as a slice of strings.
func (r *JSONTestResults) extractTags() ([]string, error) {
	maybeTags, ok := r.Metadata["tags"]
	if !ok {
		return nil, nil
	}

	var tags []string
	if err := json.Unmarshal(maybeTags, &tags); err != nil {
		return nil, errors.Annotate(err, "tags not []string, got %q", maybeTags).Err()
	}

	return tags, nil
}

func fromJSONStatus(s string) (pb.TestStatus, error) {
	switch s {
	case "CRASH":
		return pb.TestStatus_CRASH, nil
	case "FAIL":
		return pb.TestStatus_FAIL, nil
	case "PASS":
		return pb.TestStatus_PASS, nil
	case "SKIP":
		return pb.TestStatus_SKIP, nil
	case "TIMEOUT":
		return pb.TestStatus_ABORT, nil

	// The below are web test-specific statuses. They are officially deprecated, but in practice
	// still generated by the tests and should be converted.
	case "IMAGE", "TEXT", "IMAGE+TEXT", "AUDIO", "LEAK", "MISSING":
		return pb.TestStatus_FAIL, nil

	default:
		// There are a number of web test-specific statuses not handled here as they are deprecated.
		return 0, errors.Reason("unknown or unexpected JSON Test Format status %s", s).Err()
	}
}

// testArtifactsPerRun maps a run index to a map of run index to slice of
// associated *pb.Artifacts.
type testArtifactsPerRun map[int][]*pb.Artifact

// toProtos converts the TestFields into zero or more pb.TestResult and
// appends them to dest.
//
// Any artifacts that could not be processed are returned.
// TODO(jchinlee): once we've curated the artifacts to process, make unprocessed artifacts error.
func (f *TestFields) toProtos(ctx context.Context, dest *[]*pb.TestResult, testPath string, outputsToProcess map[string]*pb.Artifact) (map[string][]string, error) {
	// Process statuses.
	actualStatuses := strings.Split(f.Actual, " ")
	expectedSet := stringset.NewFromSlice(strings.Split(f.Expected, " ")...)

	// Process times.
	// Time and Times are both optional, but if Times is present, its length should match the number
	// of runs. Otherwise we have only Time as the duration of the first run.
	if len(f.Times) > 0 && len(f.Times) != len(actualStatuses) {
		return nil, errors.Reason(
			"%d durations populated but has %d test statuses; should match",
			len(f.Times), len(actualStatuses)).Err()
	}

	var durations []float64
	if len(f.Times) > 0 {
		durations = f.Times
	} else if f.Time != 0 { // Do not set duration if it is unknown.
		durations = []float64{f.Time}
	}

	// Get artifacts.
	// We expect that if we have any artifacts, the number of runs from deriving the artifacts
	// should match the number of actual runs. Because the arts are a map from run index to
	// *pb.Artifacts slice, we will not error if artifacts are missing for a run, but log a warning
	// in case the number of runs do not match each other for further investigation.
	arts, unresolved := f.getArtifacts(outputsToProcess)
	if len(arts) > 0 && len(actualStatuses) != len(arts) {
		logging.Warningf(ctx,
			"Number of runs of test %s (%d) does not match number of runs generated from artifacts (%d)",
			len(actualStatuses), len(arts), testPath)
	}

	// Populate protos.
	for i, runStatus := range actualStatuses {
		status, err := fromJSONStatus(runStatus)
		if err != nil {
			return nil, err
		}

		tr := &pb.TestResult{
			TestPath:        testPath,
			Expected:        expectedSet.Has(runStatus),
			Status:          status,
			Tags:            pbutil.StringPairs("json_format_status", runStatus),
			OutputArtifacts: arts[i],
		}

		if i < len(durations) {
			tr.Duration = secondsToDuration(durations[i])
		}

		pbutil.NormalizeTestResult(tr)
		*dest = append(*dest, tr)
	}

	return unresolved, nil
}

// getArtifacts gets pb.Artifacts corresponding to the TestField's artifacts.
//
// It tries to derive the pb.Artifacts in the following order:
//   - look for them in the isolated outputs represented as pb.Artifacts
//   - check if they're a known special case
//   - fail to process and mark them as `unresolvedArtifacts`
func (f *TestFields) getArtifacts(outputsToProcess map[string]*pb.Artifact) (artifacts testArtifactsPerRun, unresolvedArtifacts map[string][]string) {
	artifacts = testArtifactsPerRun{}
	unresolvedArtifacts = map[string][]string{}

	for name, paths := range f.Artifacts {
		for i, path := range paths {
			// Get the run ID of the artifact. Defaults to 0 (i.e. assumes there is only one run).
			runID, err := artifactRunID(path)
			if err != nil {
				unresolvedArtifacts[name] = append(unresolvedArtifacts[name], path)
				continue
			}

			// Look for the path in isolated outputs.
			if art, ok := outputsToProcess[path]; ok {
				artifacts[runID] = append(artifacts[runID], art)
				delete(outputsToProcess, path)
				continue
			}

			// If the name is otherwise understood by ResultDB, process it.
			// So far, that's only gold_triage_links.
			if name == "gold_triage_link" || name == "triage_link_for_entire_cl" {
				// We don't expect more than one triage link per test run, but if there is more than one,
				// suffix the name with index to ensure we retain it too.
				if i > 0 {
					name = fmt.Sprintf("%s_%d", name, i)
				}

				artifacts[runID] = append(artifacts[runID], &pb.Artifact{Name: name, ViewUrl: path})
				continue
			}

			// Otherwise, could not populate artifact, so mark it as unresolved.
			unresolvedArtifacts[name] = append(unresolvedArtifacts[name], path)
		}
	}

	return
}

// artifactRunID extracts a run ID, defaulting to 0, or error if it doesn't recognize the format.
func artifactRunID(path string) (int, error) {
	if m := testRunSubdirRe.FindStringSubmatch(path); m != nil {
		return strconv.Atoi(m[1])
	}

	// No retry_<i> subdirectory, so assume it's the first/0th run.
	return 0, nil
}

// artifactsToString converts the given name->paths artifacts map to a string for logging.
func artifactsToString(arts map[string][]string) string {
	names := make([]string, 0, len(arts))
	for name := range arts {
		names = append(names, name)
	}
	sort.Strings(names)

	var msg bytes.Buffer
	w := &indented.Writer{Writer: &msg}
	for _, name := range names {
		fmt.Fprintln(w, name)
		w.Level++
		for _, p := range arts[name] {
			fmt.Fprintln(w, p)
		}
		w.Level--
	}
	return msg.String()
}
<|endoftext|>


<commit_before>package uiprogress

import (
	"fmt"
	"io"
	"os"
	"sync"
	"time"

	"github.com/gosuri/uilive"
)

// Out is the default writer to render progress bars to
var Out = os.Stdout

// RefreshInterval in the default time duration to wait for refreshing the output
var RefreshInterval = time.Millisecond * 10

// defaultProgress is the default progress
var defaultProgress = New()

// Progress represents the container that renders progress bars
type Progress struct {
	// Out is the writer to render progress bars to
	Out io.Writer

	// Width is the width of the progress bars
	Width int

	// Bars is the collection of progress bars
	Bars []*Bar

	// RefreshInterval in the time duration to wait for refreshing the output
	RefreshInterval time.Duration

	lw     *uilive.Writer
	ticker *time.Ticker
	tdone  chan bool
	mtx    *sync.RWMutex
}

// New returns a new progress bar with defaults
func New() *Progress {
	return &Progress{
		Width:           Width,
		Out:             Out,
		Bars:            make([]*Bar, 0),
		RefreshInterval: RefreshInterval,

		lw:  uilive.New(),
		mtx: &sync.RWMutex{},
	}
}

// AddBar creates a new progress bar and adds it to the default progress container
func AddBar(total int) *Bar {
	return defaultProgress.AddBar(total)
}

// Start starts the rendering the progress of progress bars using the DefaultProgress. It listens for updates using `bar.Set(n)` and new bars when added using `AddBar`
func Start() {
	defaultProgress.Start()
}

// Stop stops listening
func Stop() {
	defaultProgress.Stop()
}

// Listen listens for updates and renders the progress bars
func Listen() {
	defaultProgress.Listen()
}

// AddBar creates a new progress bar and adds to the container
func (p *Progress) AddBar(total int) *Bar {
	p.mtx.Lock()
	defer p.mtx.Unlock()

	bar := NewBar(total)
	bar.Width = p.Width
	p.Bars = append(p.Bars, bar)
	return bar
}

// Listen listens for updates and renders the progress bars
func (p *Progress) Listen() {
	p.lw.Out = p.Out

	for {
		select {
		case <-p.ticker.C:
			p.mtx.RLock()

			if p.ticker != nil {
				p.print()
				p.lw.Flush()
			}

			p.mtx.RUnlock()
		case <-p.tdone:
			if p.ticker != nil {
				p.ticker.Stop()
				p.ticker = nil
				return
			}
		}
	}
}

func (p *Progress) print() {
	for _, bar := range p.Bars {
		fmt.Fprintln(p.lw, bar.String())
	}
}

// Start starts the rendering the progress of progress bars. It listens for updates using `bar.Set(n)` and new bars when added using `AddBar`
func (p *Progress) Start() {
	p.mtx.Lock()
	if p.ticker == nil {
		p.ticker = time.NewTicker(RefreshInterval)
		p.tdone = make(chan bool, 1)
	}
	p.mtx.Unlock()

	go p.Listen()
}

// Stop stops listening
func (p *Progress) Stop() {
	p.mtx.Lock()
	close(p.tdone)
	p.print()
	p.lw.Flush()
	p.mtx.Unlock()
}

// Bypass returns a writer which allows non-buffered data to be written to the underlying output
func (p *Progress) Bypass() io.Writer {
	return p.lw.Bypass()
}
<commit_msg>Adding tickChain variable to avoid nil dereference error.<commit_after>package uiprogress

import (
	"fmt"
	"io"
	"os"
	"sync"
	"time"

	"github.com/gosuri/uilive"
)

// Out is the default writer to render progress bars to
var Out = os.Stdout

// RefreshInterval in the default time duration to wait for refreshing the output
var RefreshInterval = time.Millisecond * 10

// defaultProgress is the default progress
var defaultProgress = New()

// Progress represents the container that renders progress bars
type Progress struct {
	// Out is the writer to render progress bars to
	Out io.Writer

	// Width is the width of the progress bars
	Width int

	// Bars is the collection of progress bars
	Bars []*Bar

	// RefreshInterval in the time duration to wait for refreshing the output
	RefreshInterval time.Duration

	lw     *uilive.Writer
	ticker *time.Ticker
	tdone  chan bool
	mtx    *sync.RWMutex
}

// New returns a new progress bar with defaults
func New() *Progress {
	return &Progress{
		Width:           Width,
		Out:             Out,
		Bars:            make([]*Bar, 0),
		RefreshInterval: RefreshInterval,

		lw:  uilive.New(),
		mtx: &sync.RWMutex{},
	}
}

// AddBar creates a new progress bar and adds it to the default progress container
func AddBar(total int) *Bar {
	return defaultProgress.AddBar(total)
}

// Start starts the rendering the progress of progress bars using the DefaultProgress. It listens for updates using `bar.Set(n)` and new bars when added using `AddBar`
func Start() {
	defaultProgress.Start()
}

// Stop stops listening
func Stop() {
	defaultProgress.Stop()
}

// Listen listens for updates and renders the progress bars
func Listen() {
	defaultProgress.Listen()
}

// AddBar creates a new progress bar and adds to the container
func (p *Progress) AddBar(total int) *Bar {
	p.mtx.Lock()
	defer p.mtx.Unlock()

	bar := NewBar(total)
	bar.Width = p.Width
	p.Bars = append(p.Bars, bar)
	return bar
}

// Listen listens for updates and renders the progress bars
func (p *Progress) Listen() {
	var tickChan = p.ticker.C
	p.lw.Out = p.Out

	for {
		select {
		case <-tickChan:
			p.mtx.RLock()

			if p.ticker != nil {
				p.print()
				p.lw.Flush()
			}

			p.mtx.RUnlock()
		case <-p.tdone:
			if p.ticker != nil {
				p.ticker.Stop()
				p.ticker = nil
				return
			}
		}
	}
}

func (p *Progress) print() {
	for _, bar := range p.Bars {
		fmt.Fprintln(p.lw, bar.String())
	}
}

// Start starts the rendering the progress of progress bars. It listens for updates using `bar.Set(n)` and new bars when added using `AddBar`
func (p *Progress) Start() {
	p.mtx.Lock()
	if p.ticker == nil {
		p.ticker = time.NewTicker(RefreshInterval)
		p.tdone = make(chan bool, 1)
	}
	p.mtx.Unlock()

	go p.Listen()
}

// Stop stops listening
func (p *Progress) Stop() {
	p.mtx.Lock()
	close(p.tdone)
	p.print()
	p.lw.Flush()
	p.mtx.Unlock()
}

// Bypass returns a writer which allows non-buffered data to be written to the underlying output
func (p *Progress) Bypass() io.Writer {
	return p.lw.Bypass()
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"fmt"
	"socialapi/config"
	"time"

	"github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/jinzhu/gorm"
	"github.com/koding/bongo"
)

var mentionRegex = verbalexpressions.New().
	Find("@").
	BeginCapture().
	Word().
	EndCapture().
	Regex()

type ChannelMessage struct {
	// unique identifier of the channel message
	Id int64 `json:"id,string"`

	// Body of the mesage
	Body string `json:"body"`

	// Generated Slug for body
	Slug string `json:"slug"                               sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// type of the message
	TypeConstant string `json:"typeConstant"               sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// Creator of the channel message
	AccountId int64 `json:"accountId,string"               sql:"NOT NULL"`

	// in which channel this message is created
	InitialChannelId int64 `json:"initialChannelId,string" sql:"NOT NULL"`

	// Creation date of the message
	CreatedAt time.Time `json:"createdAt"                  sql:"DEFAULT:CURRENT_TIMESTAMP"`

	// Modification date of the message
	UpdatedAt time.Time `json:"updatedAt"                  sql:"DEFAULT:CURRENT_TIMESTAMP"`

	// Deletion date of the channel message
	DeletedAt time.Time `json:"deletedAt"`

	// Extra data storage
	Payload gorm.Hstore `json:"payload,omitempty"`
}

func (c *ChannelMessage) BeforeCreate() {
	c.DeletedAt = ZeroDate()
}

func (c *ChannelMessage) AfterCreate() {
	bongo.B.AfterCreate(c)
}

func (c *ChannelMessage) AfterUpdate() {
	bongo.B.AfterUpdate(c)
}

func (c ChannelMessage) AfterDelete() {
	bongo.B.AfterDelete(c)
}

func (c ChannelMessage) GetId() int64 {
	return c.Id
}

func (c ChannelMessage) TableName() string {
	return "api.channel_message"
}

const (
	ChannelMessage_TYPE_POST            = "post"
	ChannelMessage_TYPE_REPLY           = "reply"
	ChannelMessage_TYPE_JOIN            = "join"
	ChannelMessage_TYPE_LEAVE           = "leave"
	ChannelMessage_TYPE_CHAT            = "chat"
	ChannelMessage_TYPE_PRIVATE_MESSAGE = "privatemessage"
)

func NewChannelMessage() *ChannelMessage {
	return &ChannelMessage{}
}

func (c *ChannelMessage) ById(id int64) error {
	return bongo.B.ById(c, id)
}

func (c *ChannelMessage) One(q *bongo.Query) error {
	return bongo.B.One(c, c, q)
}

func (c *ChannelMessage) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(c, data, q)
}

func bodyLenCheck(body string) error {
	if len(body) < config.Get().Limits.MessageBodyMinLen {
		return fmt.Errorf("Message Body Length should be greater than %d, yours is %d ", config.Get().Limits.MessageBodyMinLen, len(body))
	}

	return nil
}

// todo create a new message while updating the channel_message and delete other
// cases, since deletion is a soft delete, old instances will still be there
func (c *ChannelMessage) Update() error {
	if err := bodyLenCheck(c.Body); err != nil {
		return err
	}
	// only update body
	err := bongo.B.UpdatePartial(c,
		map[string]interface{}{
			"body": c.Body,
		},
	)
	return err
}

func (c *ChannelMessage) Create() error {
	if err := bodyLenCheck(c.Body); err != nil {
		return err
	}

	var err error
	c, err = Slugify(c)
	if err != nil {
		return err
	}

	return bongo.B.Create(c)
}

// CreateRaw creates a new channel message without effected by auto generated createdAt
// and updatedAt values
func (c *ChannelMessage) CreateRaw() error {
	insertSql := "INSERT INTO " +
		c.TableName() +
		` ("body","slug","type_constant","account_id","initial_channel_id",` +
		`"created_at","updated_at","deleted_at","payload") ` +
		"VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) " +
		"RETURNING ID"

	return bongo.B.DB.CommonDB().QueryRow(insertSql, c.Body, c.Slug, c.TypeConstant, c.AccountId, c.InitialChannelId,
		c.CreatedAt, c.UpdatedAt, c.DeletedAt, c.Payload).Scan(&c.Id)
}

func (c *ChannelMessage) Delete() error {
	return bongo.B.Delete(c)
}

func (c *ChannelMessage) FetchByIds(ids []int64) ([]ChannelMessage, error) {
	var messages []ChannelMessage

	if len(ids) == 0 {
		return messages, nil
	}

	if err := bongo.B.FetchByIds(c, &messages, ids); err != nil {
		return nil, err
	}

	return messages, nil
}

func (c *ChannelMessage) BuildMessages(query *Query, messages []ChannelMessage) ([]*ChannelMessageContainer, error) {
	containers := make([]*ChannelMessageContainer, len(messages))
	if len(containers) == 0 {
		return containers, nil
	}

	for i, message := range messages {
		d := NewChannelMessage()
		*d = message
		data, err := d.BuildMessage(query)
		if err != nil {
			return containers, err
		}
		containers[i] = data
	}

	return containers, nil
}

func (c *ChannelMessage) BuildMessage(query *Query) (*ChannelMessageContainer, error) {
	cmc, err := c.FetchRelatives(query)
	if err != nil {
		return nil, err
	}

	mr := NewMessageReply()
	mr.MessageId = c.Id
	q := query
	q.Limit = 3
	replies, err := mr.List(query)
	if err != nil {
		return nil, err
	}

	repliesCount, err := mr.Count()
	if err != nil {
		return nil, err
	}
	cmc.RepliesCount = repliesCount

	cmc.IsFollowed, err = c.CheckIsMessageFollowed(query)
	if err != nil {
		return nil, err
	}

	populatedChannelMessagesReplies := make([]*ChannelMessageContainer, len(replies))
	for rl := 0; rl < len(replies); rl++ {
		cmrc, err := replies[rl].FetchRelatives(query)
		if err != nil {
			return nil, err
		}
		populatedChannelMessagesReplies[rl] = cmrc
	}

	cmc.Replies = populatedChannelMessagesReplies
	return cmc, nil
}

func (c *ChannelMessage) CheckIsMessageFollowed(query *Query) (bool, error) {
	channel := NewChannel()
	if err := channel.FetchPinnedActivityChannel(query.AccountId, query.GroupName); err != nil {
		if err == gorm.RecordNotFound {
			return false, nil
		}
		return false, err
	}

	cml := NewChannelMessageList()
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"channel_id": channel.Id,
			"message_id": c.Id,
		},
	}
	if err := cml.One(q); err != nil {
		if err == gorm.RecordNotFound {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

func (c *ChannelMessage) BuildEmptyMessageContainer() (*ChannelMessageContainer, error) {
	if c.Id == 0 {
		return nil, errors.New("Channel message id is not set")
	}
	container := NewChannelMessageContainer()
	container.Message = c

	oldId, err := AccountOldIdById(c.AccountId)
	if err != nil {
		return nil, err
	}

	container.AccountOldId = oldId

	interactionContainer := NewInteractionContainer()
	interactionContainer.ActorsPreview = make([]string, 0)
	interactionContainer.IsInteracted = false
	interactionContainer.ActorsCount = 0

	container.Interactions = make(map[string]*InteractionContainer)
	container.Interactions["like"] = interactionContainer

	return container, nil
}

func (c *ChannelMessage) FetchRelatives(query *Query) (*ChannelMessageContainer, error) {
	container, err := c.BuildEmptyMessageContainer()
	if err != nil {
		return nil, err
	}

	i := NewInteraction()
	i.MessageId = c.Id

	// get preview
	query.Type = "like"
	query.Limit = 3
	interactorIds, err := i.List(query)
	if err != nil {
		return nil, err
	}

	oldIds, err := FetchOldIdsByAccountIds(interactorIds)
	if err != nil {
		return nil, err
	}

	interactionContainer := NewInteractionContainer()
	interactionContainer.ActorsPreview = oldIds

	// check if the current user is interacted in this thread
	isInteracted, err := i.IsInteracted(query.AccountId)
	if err != nil {
		return nil, err
	}

	interactionContainer.IsInteracted = isInteracted

	// fetch interaction count
	count, err := i.Count(query.Type)
	if err != nil {
		return nil, err
	}

	interactionContainer.ActorsCount = count

	container.Interactions["like"] = interactionContainer
	return container, nil
}

func generateMessageListQuery(channelId int64, q *Query) *bongo.Query {
	messageType := q.Type
	if messageType == "" {
		messageType = ChannelMessage_TYPE_POST
	}

	return &bongo.Query{
		Selector: map[string]interface{}{
			"account_id":         q.AccountId,
			"initial_channel_id": channelId,
			"type_constant":      messageType,
		},
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
		Sort: map[string]string{
			"created_at": "DESC",
		},
	}
}

func (c *ChannelMessage) FetchMessagesByChannelId(channelId int64, q *Query) ([]ChannelMessage, error) {
	query := generateMessageListQuery(channelId, q)

	var messages []ChannelMessage
	if err := c.Some(&messages, query); err != nil {
		return nil, err
	}

	if messages == nil {
		return make([]ChannelMessage, 0), nil
	}
	return messages, nil
}

func (c *ChannelMessage) GetMentionedUsernames() []string {
	flattened := make([]string, 0)

	res := mentionRegex.FindAllStringSubmatch(c.Body, -1)
	if len(res) == 0 {
		return flattened
	}

	participants := map[string]struct{}{}
	// remove duplicate mentions
	for _, ele := range res {
		participants[ele[1]] = struct{}{}
	}

	for participant := range participants {
		flattened = append(flattened, participant)
	}

	return flattened
}
<commit_msg>Social: implement BySlug method for channelmessage<commit_after>package models

import (
	"errors"
	"fmt"
	"socialapi/config"
	"time"

	"github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/jinzhu/gorm"
	"github.com/koding/bongo"
)

var mentionRegex = verbalexpressions.New().
	Find("@").
	BeginCapture().
	Word().
	EndCapture().
	Regex()

type ChannelMessage struct {
	// unique identifier of the channel message
	Id int64 `json:"id,string"`

	// Body of the mesage
	Body string `json:"body"`

	// Generated Slug for body
	Slug string `json:"slug"                               sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// type of the message
	TypeConstant string `json:"typeConstant"               sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// Creator of the channel message
	AccountId int64 `json:"accountId,string"               sql:"NOT NULL"`

	// in which channel this message is created
	InitialChannelId int64 `json:"initialChannelId,string" sql:"NOT NULL"`

	// Creation date of the message
	CreatedAt time.Time `json:"createdAt"                  sql:"DEFAULT:CURRENT_TIMESTAMP"`

	// Modification date of the message
	UpdatedAt time.Time `json:"updatedAt"                  sql:"DEFAULT:CURRENT_TIMESTAMP"`

	// Deletion date of the channel message
	DeletedAt time.Time `json:"deletedAt"`

	// Extra data storage
	Payload gorm.Hstore `json:"payload,omitempty"`
}

func (c *ChannelMessage) BeforeCreate() {
	c.DeletedAt = ZeroDate()
}

func (c *ChannelMessage) AfterCreate() {
	bongo.B.AfterCreate(c)
}

func (c *ChannelMessage) AfterUpdate() {
	bongo.B.AfterUpdate(c)
}

func (c ChannelMessage) AfterDelete() {
	bongo.B.AfterDelete(c)
}

func (c ChannelMessage) GetId() int64 {
	return c.Id
}

func (c ChannelMessage) TableName() string {
	return "api.channel_message"
}

const (
	ChannelMessage_TYPE_POST            = "post"
	ChannelMessage_TYPE_REPLY           = "reply"
	ChannelMessage_TYPE_JOIN            = "join"
	ChannelMessage_TYPE_LEAVE           = "leave"
	ChannelMessage_TYPE_CHAT            = "chat"
	ChannelMessage_TYPE_PRIVATE_MESSAGE = "privatemessage"
)

func NewChannelMessage() *ChannelMessage {
	return &ChannelMessage{}
}

func (c *ChannelMessage) ById(id int64) error {
	return bongo.B.ById(c, id)
}

func (c *ChannelMessage) One(q *bongo.Query) error {
	return bongo.B.One(c, c, q)
}

func (c *ChannelMessage) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(c, data, q)
}

func bodyLenCheck(body string) error {
	if len(body) < config.Get().Limits.MessageBodyMinLen {
		return fmt.Errorf("Message Body Length should be greater than %d, yours is %d ", config.Get().Limits.MessageBodyMinLen, len(body))
	}

	return nil
}

// todo create a new message while updating the channel_message and delete other
// cases, since deletion is a soft delete, old instances will still be there
func (c *ChannelMessage) Update() error {
	if err := bodyLenCheck(c.Body); err != nil {
		return err
	}
	// only update body
	err := bongo.B.UpdatePartial(c,
		map[string]interface{}{
			"body": c.Body,
		},
	)
	return err
}

func (c *ChannelMessage) Create() error {
	if err := bodyLenCheck(c.Body); err != nil {
		return err
	}

	var err error
	c, err = Slugify(c)
	if err != nil {
		return err
	}

	return bongo.B.Create(c)
}

// CreateRaw creates a new channel message without effected by auto generated createdAt
// and updatedAt values
func (c *ChannelMessage) CreateRaw() error {
	insertSql := "INSERT INTO " +
		c.TableName() +
		` ("body","slug","type_constant","account_id","initial_channel_id",` +
		`"created_at","updated_at","deleted_at","payload") ` +
		"VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) " +
		"RETURNING ID"

	return bongo.B.DB.CommonDB().QueryRow(insertSql, c.Body, c.Slug, c.TypeConstant, c.AccountId, c.InitialChannelId,
		c.CreatedAt, c.UpdatedAt, c.DeletedAt, c.Payload).Scan(&c.Id)
}

func (c *ChannelMessage) Delete() error {
	return bongo.B.Delete(c)
}

func (c *ChannelMessage) FetchByIds(ids []int64) ([]ChannelMessage, error) {
	var messages []ChannelMessage

	if len(ids) == 0 {
		return messages, nil
	}

	if err := bongo.B.FetchByIds(c, &messages, ids); err != nil {
		return nil, err
	}

	return messages, nil
}

func (c *ChannelMessage) BuildMessages(query *Query, messages []ChannelMessage) ([]*ChannelMessageContainer, error) {
	containers := make([]*ChannelMessageContainer, len(messages))
	if len(containers) == 0 {
		return containers, nil
	}

	for i, message := range messages {
		d := NewChannelMessage()
		*d = message
		data, err := d.BuildMessage(query)
		if err != nil {
			return containers, err
		}
		containers[i] = data
	}

	return containers, nil
}

func (c *ChannelMessage) BuildMessage(query *Query) (*ChannelMessageContainer, error) {
	cmc, err := c.FetchRelatives(query)
	if err != nil {
		return nil, err
	}

	mr := NewMessageReply()
	mr.MessageId = c.Id
	q := query
	q.Limit = 3
	replies, err := mr.List(query)
	if err != nil {
		return nil, err
	}

	repliesCount, err := mr.Count()
	if err != nil {
		return nil, err
	}
	cmc.RepliesCount = repliesCount

	cmc.IsFollowed, err = c.CheckIsMessageFollowed(query)
	if err != nil {
		return nil, err
	}

	populatedChannelMessagesReplies := make([]*ChannelMessageContainer, len(replies))
	for rl := 0; rl < len(replies); rl++ {
		cmrc, err := replies[rl].FetchRelatives(query)
		if err != nil {
			return nil, err
		}
		populatedChannelMessagesReplies[rl] = cmrc
	}

	cmc.Replies = populatedChannelMessagesReplies
	return cmc, nil
}

func (c *ChannelMessage) CheckIsMessageFollowed(query *Query) (bool, error) {
	channel := NewChannel()
	if err := channel.FetchPinnedActivityChannel(query.AccountId, query.GroupName); err != nil {
		if err == gorm.RecordNotFound {
			return false, nil
		}
		return false, err
	}

	cml := NewChannelMessageList()
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"channel_id": channel.Id,
			"message_id": c.Id,
		},
	}
	if err := cml.One(q); err != nil {
		if err == gorm.RecordNotFound {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

func (c *ChannelMessage) BuildEmptyMessageContainer() (*ChannelMessageContainer, error) {
	if c.Id == 0 {
		return nil, errors.New("Channel message id is not set")
	}
	container := NewChannelMessageContainer()
	container.Message = c

	oldId, err := AccountOldIdById(c.AccountId)
	if err != nil {
		return nil, err
	}

	container.AccountOldId = oldId

	interactionContainer := NewInteractionContainer()
	interactionContainer.ActorsPreview = make([]string, 0)
	interactionContainer.IsInteracted = false
	interactionContainer.ActorsCount = 0

	container.Interactions = make(map[string]*InteractionContainer)
	container.Interactions["like"] = interactionContainer

	return container, nil
}

func (c *ChannelMessage) FetchRelatives(query *Query) (*ChannelMessageContainer, error) {
	container, err := c.BuildEmptyMessageContainer()
	if err != nil {
		return nil, err
	}

	i := NewInteraction()
	i.MessageId = c.Id

	// get preview
	query.Type = "like"
	query.Limit = 3
	interactorIds, err := i.List(query)
	if err != nil {
		return nil, err
	}

	oldIds, err := FetchOldIdsByAccountIds(interactorIds)
	if err != nil {
		return nil, err
	}

	interactionContainer := NewInteractionContainer()
	interactionContainer.ActorsPreview = oldIds

	// check if the current user is interacted in this thread
	isInteracted, err := i.IsInteracted(query.AccountId)
	if err != nil {
		return nil, err
	}

	interactionContainer.IsInteracted = isInteracted

	// fetch interaction count
	count, err := i.Count(query.Type)
	if err != nil {
		return nil, err
	}

	interactionContainer.ActorsCount = count

	container.Interactions["like"] = interactionContainer
	return container, nil
}

func generateMessageListQuery(channelId int64, q *Query) *bongo.Query {
	messageType := q.Type
	if messageType == "" {
		messageType = ChannelMessage_TYPE_POST
	}

	return &bongo.Query{
		Selector: map[string]interface{}{
			"account_id":         q.AccountId,
			"initial_channel_id": channelId,
			"type_constant":      messageType,
		},
		Pagination: *bongo.NewPagination(q.Limit, q.Skip),
		Sort: map[string]string{
			"created_at": "DESC",
		},
	}
}

func (c *ChannelMessage) FetchMessagesByChannelId(channelId int64, q *Query) ([]ChannelMessage, error) {
	query := generateMessageListQuery(channelId, q)

	var messages []ChannelMessage
	if err := c.Some(&messages, query); err != nil {
		return nil, err
	}

	if messages == nil {
		return make([]ChannelMessage, 0), nil
	}
	return messages, nil
}

func (c *ChannelMessage) GetMentionedUsernames() []string {
	flattened := make([]string, 0)

	res := mentionRegex.FindAllStringSubmatch(c.Body, -1)
	if len(res) == 0 {
		return flattened
	}

	participants := map[string]struct{}{}
	// remove duplicate mentions
	for _, ele := range res {
		participants[ele[1]] = struct{}{}
	}

	for participant := range participants {
		flattened = append(flattened, participant)
	}

	return flattened
}

// BySlug fetchs channel message by its slug
// checks if message is in the channel or not
func (c *ChannelMessage) BySlug(query *Query) error {
	if query.Slug == "" {
		return errors.New("slug is not set")
	}

	// fetch message itself
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"slug": query.Slug,
		},
	}

	if err := c.One(q); err != nil {
		return nil
	}

	// fetch channel by group name
	query.Name = query.GroupName
	query.Type = Channel_TYPE_GROUP
	ch := NewChannel()
	channel, err := ch.ByName(query)
	if err != nil {
		return err
	}

	if channel.Id == 0 {
		return errors.New("channel is not found")
	}

	// check if message is in the channel
	cml := NewChannelMessageList()
	res, err := cml.IsInChannel(c.Id, channel.Id)
	if err != nil {
		return err
	}

	// if message is not in the channel
	if !res {
		return gorm.RecordNotFound
	}

	return nil
}
<|endoftext|>


<commit_before>package MinimapStitcher

import (
	"fmt"
	"runtime"
	"os"
	"log"
	"sync"
	"io/ioutil"
	"strings"
	"strconv"
	"image/draw"
	"image"
	"image/png"
	"encoding/json"
)

const (
	TILE_SIZE = 256
)

type reportCallbackMessage map[string]string
type reportCallback func(reportCallbackMessage)
type reportCallbackWrapper func(string, reportCallbackMessage)

func Stitch(sourceDirectory string, destinationDirectory string) {
	var callback = func(message reportCallbackMessage) {
		json, _ := json.Marshal(message)
		line := append(json, "\r\n"...)
		os.Stdout.Write(line)
	}

	var wg sync.WaitGroup;
	tasks := make(chan [5]string);

	setupWaitGroup(wg, tasks, callback);
	listMapsFound(callback, sourceDirectory);
	addMapsToWaitGroup(tasks, sourceDirectory, destinationDirectory);

	wg.Wait()
}

func listMapsFound(callback reportCallback, sourceDirectory string) {
	files, _ := ioutil.ReadDir(sourceDirectory)
	for _, f := range files {
		fd, err := os.Open(sourceDirectory + f.Name())
		if err != nil {
			fmt.Println(err)
			return
		}
		fi, err := fd.Stat()
		if err != nil {
			fmt.Println(err)
			return
		}
		mode := fi.Mode()
		if mode.IsDir() {
			if f.Name() != "WMO" {
				message := make(map [string]string)
				message["minimap"] = f.Name()
				message["type"] = "found"
				callback(message)
			}
		}
		defer fd.Close()
	}
}

func setupWaitGroup(wg sync.WaitGroup, tasks chan [5]string, callback reportCallback, ) {
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go func() {
			for arguments := range tasks {
				message := make(map [string]string)
				message["minimap"] = arguments[4]
				message["tile"] = "0"
				message["tiles"] = "0"
				var callbackWrapper = func(messageText string, extras reportCallbackMessage) {
					message["type"] = messageText

					if _, ok := extras["tile"]; ok {
						message["tile"] = extras["tile"]
					}
					if _, ok := extras["tiles"]; ok {
						message["tiles"] = extras["tiles"]
					}

					callback(message)
				}
				callbackWrapper("start_compile", make(map [string]string))
				compileMinimap(callbackWrapper, tasks, arguments[0], arguments[1], arguments[2], arguments[3])
				callbackWrapper("complete_compile", make(map [string]string))
			}
			wg.Done()
			return
		}()
	}
}

func addMapsToWaitGroup(tasks chan [5]string, sourceDirectory string, destinationDirectory string) {
	files, _ := ioutil.ReadDir(sourceDirectory)
	for _, f := range files {
		fd, err := os.Open(sourceDirectory + f.Name())
		if err != nil {
			fmt.Println(err)
			return
		}
		fi, err := fd.Stat()
		if err != nil {
			fmt.Println(err)
			return
		}
		mode := fi.Mode()
		if mode.IsDir() {
			if f.Name() != "WMO" {
				var array [5]string
				array[0] = destinationDirectory + f.Name()
				array[1] = sourceDirectory + f.Name()
				array[2] = f.Name()
				array[3] = "false"
				array[4] = f.Name()
				tasks <- array
			}
		}
		defer fd.Close()
	}
}

func compileMinimap(callback reportCallbackWrapper, tasks chan [5]string, resultFileName string, sourceDirectory string, minimapName string, noLiquidString string) {
	var falseString = "false"
	var foundNoLiquid = false
	var tiles = make(map[string]string);
	files, _ := ioutil.ReadDir(sourceDirectory)
	for _, f := range files {
		var fullFileName = sourceDirectory + "/" + f.Name()
		if strings.Contains(f.Name(), ".png") {
			var fName = strings.TrimRight(f.Name(), ".png")
			var fNameNoLiquid = strings.Contains(fName, "noLiquid")
			if fNameNoLiquid && noLiquidString == falseString {
				foundNoLiquid = true
			} else if !fNameNoLiquid && noLiquidString == falseString {
				tiles[strings.TrimLeft(fName, "map")] = fullFileName
			} else if fNameNoLiquid && noLiquidString != falseString {
				tiles[strings.TrimLeft(fName, "noLiquid_map")] = fullFileName
			} else if !fNameNoLiquid && noLiquidString != falseString {
				var trimmerFName = strings.TrimLeft(fName, "map")
				if _, ok := tiles[trimmerFName]; !ok {
					tiles[trimmerFName] = fullFileName
				}
			}
		}
	}

	if foundNoLiquid && noLiquidString == falseString {
		var array [5]string
		array[0] = resultFileName + "NoLiquid"
		array[1] = sourceDirectory
		array[2] = minimapName
		array[3] = "true"
		array[4] = minimapName + "NoLiquid"
		go func() { tasks <- array }()
	}

	callback("start_build", make(map [string]string))
	buildMinimap(callback, resultFileName, tiles)
	callback("start_build", make(map [string]string))
}

func buildMinimap(callback reportCallbackWrapper, resultFileName string, tiles map[string]string)  {
	callback("calculate_minimap_size", make(map [string]string))
	var hc, lc, hr, lr = calculateMinimapSize(tiles)

	callback("calculate_minimap_tileplacement", make(map [string]string))
	var files, width, height = calculateMinimapTilePlacement(tiles, hc, lc, hr, lr)

	createMinimapImage(callback, resultFileName, width, height, files)
}

func calculateMinimapSize(tiles map[string]string) (hc, lc, hr, lr int) {
	hc = 1000
	lc = 0
	hr = 1000
	lr = 0
	for tile, _ := range tiles {
		var tileParts = strings.Split(tile, "_")
		var col, _ = strconv.Atoi(tileParts[0])
		var row, _ = strconv.Atoi(tileParts[1])
		if hc > col {
			hc = col
		}
		if lc < col {
			lc = col
		}
		if hr > row {
			hr = row
		}
		if lr < row {
			lr = row
		}
	}
	return
}

func calculateMinimapTilePlacement(tiles map[string]string, hc int, lc int, hr int, lr int) (map[string]string, int, int) {
	var files = make(map[string]string)

	var width = 0;
	var height = 0;

	for i := hc; i < lc; i++ {
		width += TILE_SIZE
		for j := hr; j < lr; j++ {
			if i == hc {
				height += TILE_SIZE
			}

			var si = strconv.Itoa(i)
			var sj = strconv.Itoa(j)
			var oi = strconv.Itoa((i - hc) * TILE_SIZE)
			var oj = strconv.Itoa((j - hr) * TILE_SIZE)
			if _, ok := tiles[si + "_" + sj]; ok {
				files[oi + "_" + oj] = tiles[si + "_" + sj]
			}
		}
	}

	return files, width, height
}

func createMinimapImage(callback reportCallbackWrapper, resultFileName string, width int, height int, files map[string]string) {
	extras := make(map [string]string)
	extras["tiles"] = strconv.Itoa(len(files))
	callback("start_stitch", extras)

	m := image.NewRGBA(image.Rect(0, 0, width, height))
	for coords, fileName := range files {
		extras := make(map [string]string)
		extras["tile"] = coords
		callback("stitch_tile", extras)

		var coordParts = strings.Split(coords, "_")
		var x, _ = strconv.Atoi(coordParts[0])
		var y, _ = strconv.Atoi(coordParts[1])

		applyTileToImage(m, fileName, x, y)
	}

	toimg, _ := os.Create(resultFileName + ".png")
	png.Encode(toimg, m)

	defer toimg.Close()

	callback("finish_stitch", make(map [string]string))
}

func applyTileToImage(m *image.RGBA, fileName string, x int, y int) {
	tile, err := os.Open(fileName)
	if err != nil {
		log.Fatal(err)
	}
	defer tile.Close()

	tileImage, err := png.Decode(tile)

	if err != nil {
		fmt.Printf("%v", fileName)
		log.Fatal(err)
	}

	draw.Draw(m, image.Rect(x, y, x + TILE_SIZE, y + TILE_SIZE), tileImage, image.Point{0,0}, draw.Src)
}
<commit_msg>Tile size has been increased to 512<commit_after>package MinimapStitcher

import (
	"fmt"
	"runtime"
	"os"
	"log"
	"sync"
	"io/ioutil"
	"strings"
	"strconv"
	"image/draw"
	"image"
	"image/png"
	"encoding/json"
)

const (
	TILE_SIZE = 512
)

type reportCallbackMessage map[string]string
type reportCallback func(reportCallbackMessage)
type reportCallbackWrapper func(string, reportCallbackMessage)

func Stitch(sourceDirectory string, destinationDirectory string) {
	var callback = func(message reportCallbackMessage) {
		json, _ := json.Marshal(message)
		line := append(json, "\r\n"...)
		os.Stdout.Write(line)
	}

	var wg sync.WaitGroup;
	tasks := make(chan [5]string);

	setupWaitGroup(wg, tasks, callback);
	listMapsFound(callback, sourceDirectory);
	addMapsToWaitGroup(tasks, sourceDirectory, destinationDirectory);

	wg.Wait()
}

func listMapsFound(callback reportCallback, sourceDirectory string) {
	files, _ := ioutil.ReadDir(sourceDirectory)
	for _, f := range files {
		fd, err := os.Open(sourceDirectory + f.Name())
		if err != nil {
			fmt.Println(err)
			return
		}
		fi, err := fd.Stat()
		if err != nil {
			fmt.Println(err)
			return
		}
		mode := fi.Mode()
		if mode.IsDir() {
			if f.Name() != "WMO" {
				message := make(map [string]string)
				message["minimap"] = f.Name()
				message["type"] = "found"
				callback(message)
			}
		}
		defer fd.Close()
	}
}

func setupWaitGroup(wg sync.WaitGroup, tasks chan [5]string, callback reportCallback, ) {
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go func() {
			for arguments := range tasks {
				message := make(map [string]string)
				message["minimap"] = arguments[4]
				message["tile"] = "0"
				message["tiles"] = "0"
				var callbackWrapper = func(messageText string, extras reportCallbackMessage) {
					message["type"] = messageText

					if _, ok := extras["tile"]; ok {
						message["tile"] = extras["tile"]
					}
					if _, ok := extras["tiles"]; ok {
						message["tiles"] = extras["tiles"]
					}

					callback(message)
				}
				callbackWrapper("start_compile", make(map [string]string))
				compileMinimap(callbackWrapper, tasks, arguments[0], arguments[1], arguments[2], arguments[3])
				callbackWrapper("complete_compile", make(map [string]string))
			}
			wg.Done()
			return
		}()
	}
}

func addMapsToWaitGroup(tasks chan [5]string, sourceDirectory string, destinationDirectory string) {
	files, _ := ioutil.ReadDir(sourceDirectory)
	for _, f := range files {
		fd, err := os.Open(sourceDirectory + f.Name())
		if err != nil {
			fmt.Println(err)
			return
		}
		fi, err := fd.Stat()
		if err != nil {
			fmt.Println(err)
			return
		}
		mode := fi.Mode()
		if mode.IsDir() {
			if f.Name() != "WMO" {
				var array [5]string
				array[0] = destinationDirectory + f.Name()
				array[1] = sourceDirectory + f.Name()
				array[2] = f.Name()
				array[3] = "false"
				array[4] = f.Name()
				tasks <- array
			}
		}
		defer fd.Close()
	}
}

func compileMinimap(callback reportCallbackWrapper, tasks chan [5]string, resultFileName string, sourceDirectory string, minimapName string, noLiquidString string) {
	var falseString = "false"
	var foundNoLiquid = false
	var tiles = make(map[string]string);
	files, _ := ioutil.ReadDir(sourceDirectory)
	for _, f := range files {
		var fullFileName = sourceDirectory + "/" + f.Name()
		if strings.Contains(f.Name(), ".png") {
			var fName = strings.TrimRight(f.Name(), ".png")
			var fNameNoLiquid = strings.Contains(fName, "noLiquid")
			if fNameNoLiquid && noLiquidString == falseString {
				foundNoLiquid = true
			} else if !fNameNoLiquid && noLiquidString == falseString {
				tiles[strings.TrimLeft(fName, "map")] = fullFileName
			} else if fNameNoLiquid && noLiquidString != falseString {
				tiles[strings.TrimLeft(fName, "noLiquid_map")] = fullFileName
			} else if !fNameNoLiquid && noLiquidString != falseString {
				var trimmerFName = strings.TrimLeft(fName, "map")
				if _, ok := tiles[trimmerFName]; !ok {
					tiles[trimmerFName] = fullFileName
				}
			}
		}
	}

	if foundNoLiquid && noLiquidString == falseString {
		var array [5]string
		array[0] = resultFileName + "NoLiquid"
		array[1] = sourceDirectory
		array[2] = minimapName
		array[3] = "true"
		array[4] = minimapName + "NoLiquid"
		go func() { tasks <- array }()
	}

	callback("start_build", make(map [string]string))
	buildMinimap(callback, resultFileName, tiles)
	callback("start_build", make(map [string]string))
}

func buildMinimap(callback reportCallbackWrapper, resultFileName string, tiles map[string]string)  {
	callback("calculate_minimap_size", make(map [string]string))
	var hc, lc, hr, lr = calculateMinimapSize(tiles)

	callback("calculate_minimap_tileplacement", make(map [string]string))
	var files, width, height = calculateMinimapTilePlacement(tiles, hc, lc, hr, lr)

	createMinimapImage(callback, resultFileName, width, height, files)
}

func calculateMinimapSize(tiles map[string]string) (hc, lc, hr, lr int) {
	hc = 1000
	lc = 0
	hr = 1000
	lr = 0
	for tile, _ := range tiles {
		var tileParts = strings.Split(tile, "_")
		var col, _ = strconv.Atoi(tileParts[0])
		var row, _ = strconv.Atoi(tileParts[1])
		if hc > col {
			hc = col
		}
		if lc < col {
			lc = col
		}
		if hr > row {
			hr = row
		}
		if lr < row {
			lr = row
		}
	}
	return
}

func calculateMinimapTilePlacement(tiles map[string]string, hc int, lc int, hr int, lr int) (map[string]string, int, int) {
	var files = make(map[string]string)

	var width = 0;
	var height = 0;

	for i := hc; i < lc; i++ {
		width += TILE_SIZE
		for j := hr; j < lr; j++ {
			if i == hc {
				height += TILE_SIZE
			}

			var si = strconv.Itoa(i)
			var sj = strconv.Itoa(j)
			var oi = strconv.Itoa((i - hc) * TILE_SIZE)
			var oj = strconv.Itoa((j - hr) * TILE_SIZE)
			if _, ok := tiles[si + "_" + sj]; ok {
				files[oi + "_" + oj] = tiles[si + "_" + sj]
			}
		}
	}

	return files, width, height
}

func createMinimapImage(callback reportCallbackWrapper, resultFileName string, width int, height int, files map[string]string) {
	extras := make(map [string]string)
	extras["tiles"] = strconv.Itoa(len(files))
	callback("start_stitch", extras)

	m := image.NewRGBA(image.Rect(0, 0, width, height))
	for coords, fileName := range files {
		extras := make(map [string]string)
		extras["tile"] = coords
		callback("stitch_tile", extras)

		var coordParts = strings.Split(coords, "_")
		var x, _ = strconv.Atoi(coordParts[0])
		var y, _ = strconv.Atoi(coordParts[1])

		applyTileToImage(m, fileName, x, y)
	}

	toimg, _ := os.Create(resultFileName + ".png")
	png.Encode(toimg, m)

	defer toimg.Close()

	callback("finish_stitch", make(map [string]string))
}

func applyTileToImage(m *image.RGBA, fileName string, x int, y int) {
	tile, err := os.Open(fileName)
	if err != nil {
		log.Fatal(err)
	}
	defer tile.Close()

	tileImage, err := png.Decode(tile)

	if err != nil {
		fmt.Printf("%v", fileName)
		log.Fatal(err)
	}

	draw.Draw(m, image.Rect(x, y, x + TILE_SIZE, y + TILE_SIZE), tileImage, image.Point{0,0}, draw.Src)
}
<|endoftext|>