<commit_before>package main

import (
	"fmt"
	"log"
	"net/url"
	"os"
	"os/exec"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
	"github.com/flynn/flynn/pkg/cliutil"
)

var defaultRegistry = "registry.hub.docker.com"

func run(cmd *exec.Cmd) {
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

func upload(args *docopt.Args) {
	tag := args.String["<tag>"]
	if tag == "" {
		tag = "latest"
	}

	var manifest map[string]string
	if err := cliutil.DecodeJSONArg(args.String["<manifest>"], &manifest); err != nil {
		log.Fatal(err)
	}

	for image, id := range manifest {
		u, err := url.Parse(image)
		if err != nil {
			log.Fatal(err)
		}

		var tagged string
		if u.Host == defaultRegistry {
			tagged = u.Path[1:] + ":" + tag
		} else {
			tagged = u.Host + u.Path + ":" + tag
		}

		run(exec.Command("docker", "tag", id, tagged))
		fmt.Println("Tagged", tagged)

		fmt.Printf("Uploading %s...\n", tagged)
		run(exec.Command("docker", "push", tagged))
	}
}
<commit_msg>util/release: Adding --force to docker tag command<commit_after>package main

import (
	"fmt"
	"log"
	"net/url"
	"os"
	"os/exec"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
	"github.com/flynn/flynn/pkg/cliutil"
)

var defaultRegistry = "registry.hub.docker.com"

func run(cmd *exec.Cmd) {
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

func upload(args *docopt.Args) {
	tag := args.String["<tag>"]
	if tag == "" {
		tag = "latest"
	}

	var manifest map[string]string
	if err := cliutil.DecodeJSONArg(args.String["<manifest>"], &manifest); err != nil {
		log.Fatal(err)
	}

	for image, id := range manifest {
		u, err := url.Parse(image)
		if err != nil {
			log.Fatal(err)
		}

		var tagged string
		if u.Host == defaultRegistry {
			tagged = u.Path[1:] + ":" + tag
		} else {
			tagged = u.Host + u.Path + ":" + tag
		}

		run(exec.Command("docker", "tag", "--force", id, tagged))
		fmt.Println("Tagged", tagged)

		fmt.Printf("Uploading %s...\n", tagged)
		run(exec.Command("docker", "push", tagged))
	}
}
<|endoftext|>


<commit_before>package device

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/gorilla/websocket"
	"github.com/jochenvg/go-udev"
	"github.com/tarm/serial"
)

var modemCommands = struct {
	IMEI, IMSI string
}{
	"AT+GSN", "AT+CIMI",
}
var upgrader = websocket.Upgrader{}

type Message struct {
	Type string                 `json:"type"`
	Data map[string]interface{} `json:"data"`
}

// Manager manages devices that are plugged into the system. It supports auto
// detection of devices.
//
// Serial ports are opened each for a device, and a clean API for communicating
// is provided via Read, Write and Flush methods.
//
// The devices are monitored via udev, and any changes that requires reloading
// of the  ports are handled by reloading the ports to the devices.
//
// This is safe to use concurrently in multiple goroutines
type Manager struct {
	devices map[string]serial.Config
	modems  map[string]*Modem
	mu      sync.RWMutex
	monitor *udev.Monitor
	done    chan struct{}
	stop    chan struct{}
	events  chan *Message
}

// New returns a new Manager instance
func New() *Manager {
	return &Manager{
		devices: make(map[string]serial.Config),
		modems:  make(map[string]*Modem),
		done:    make(chan struct{}),
		stop:    make(chan struct{}),
		events:  make(chan *Message),
	}
}

// Init initializes the manager. This involves creating a new goroutine to watch
// over the changes detected by udev for any device interaction with the system.
//
// The only interesting device actions are add and reomove for adding and
// removing devices respctively.
func (m *Manager) Init() {
	u := udev.Udev{}
	monitor := u.NewMonitorFromNetlink("udev")
	monitor.FilterAddMatchTag("systemd")
	devCh, err := monitor.DeviceChan(m.done)
	if err != nil {
		panic(err)
	}
	m.monitor = monitor
	go func() {
	stop:
		for {
			select {
			case d := <-devCh:
				dpath := filepath.Join("/dev", filepath.Base(d.Devpath()))
				switch d.Action() {
				case "add":
					fmt.Println(" add ", dpath)
					m.AddDevice(d)
					fmt.Println(" done adding ", dpath)
				case "remove":
					fmt.Println(" removed device " + dpath)
					m.RemoveDevice(dpath)
				}
			case quit := <-m.stop:
				m.done <- quit
				break stop
			}
		}
	}()

}

// AddDevice adds device name to the manager
//
// WARNING: The way modems are picked is a hack. It asserts that the modem with
// the lowest tty number is the control modem( Which I'm not so sure is always
// correct).
//
// TODO: comeup with a proper way to identify modems
func (m *Manager) AddDevice(d *udev.Device) error {
	err := m.addDevice(d)
	if err != nil {
		return err
	}
	return m.Symlink()
}
func (m *Manager) addDevice(d *udev.Device) error {
	name := filepath.Join("/dev", filepath.Base(d.Devpath()))
	cfg := serial.Config{Name: name, Baud: 9600, ReadTimeout: 10 * time.Second}
	conn := &Conn{device: cfg}
	if strings.Contains(name, "ttyUSB") {
		fmt.Println("checking modem")
		modem, err := newModem(conn)
		if err != nil {
			return err
		}
		fmt.Println("found ", *modem)
		if mm, ok := m.getModem(modem.IMEI); ok {
			n1, err := getttyNum(mm.Path)
			if err != nil {
				return err
			}
			n2, err := getttyNum(modem.Path)
			if err != nil {
				return err
			}
			if n1 > n2 {
				m.setModem(modem)
			}
			return nil
		}
		m.setModem(modem)
	}
	return nil
}

func (m *Manager) Symlink() error {
	fmt.Println("SYMLINKING")
	for _, v := range m.modems {
		err := v.Symlink()
		if err != nil {
			fmt.Printf(" ERROR %v\n", err)
			continue
		}
	}
	return nil
}

func (m *Manager) setModem(mod *Modem) {
	fmt.Println("SETTING UP " + mod.IMEI)
	m.mu.Lock()
	m.modems[mod.IMEI] = mod
	m.mu.Unlock()
}

func (m *Manager) getModem(imei string) (*Modem, bool) {
	m.mu.RLock()
	mod, ok := m.modems[imei]
	m.mu.RUnlock()
	return mod, ok
}

func getttyNum(tty string) (int, error) {
	b := filepath.Base(tty)
	b = strings.TrimPrefix(b, "ttyUSB")
	return strconv.Atoi(b)
}

// List serves the list of current devices. The list wont cover all devices ,
// only the significant ones( modems for now)
func (m *Manager) List(w http.ResponseWriter, r *http.Request) {
	m.mu.RLock()
	json.NewEncoder(w).Encode(m.modems)
	m.mu.RUnlock()
	w.Header().Set("Content-Type", "application/json")
}

// RunComand runs commands to the exposed devices over serial ports
func (m *Manager) RunCommand(w http.ResponseWriter, r *http.Request) {
}

// RemoveDevice removes device name from the manager
func (m *Manager) RemoveDevice(name string) error {
	return nil
}

type Modem struct {
	IMEI         string `json:"imei"`
	IMSI         string `json:"imsi"`
	Manufacturer string `json:"manufacturer"`
	Path         string `json:"tty"`
	conn         *Conn
}

// Symlink adds symlink to the  modem. The symlink links the tty to the IMEI
// number of the modem.
func (m *Modem) Symlink() error {
	newLink := (fmt.Sprintf("/dev/%s", m.IMEI))
	err := syscall.Unlink(newLink)
	if err != nil {
		fmt.Printf(" ERROR %v \n", err)
	}
	return os.Symlink(m.Path, newLink)
}

func newModem(c *Conn) (*Modem, error) {
	m := &Modem{}
	ich := time.After(10 * time.Second)
STOP:
	for {
		select {
		case <-ich:
			break STOP
		default:
			imei, err := c.Run(modemCommands.IMEI)
			if err != nil {
				continue
			}
			i, err := cleanResult(imei)
			if err != nil {
				continue
			}
			im := string(i)
			if !isNumber(im) {
				continue
			}
			m.IMEI = im
			break STOP
		}
	}
	if m.IMEI == "" {
		return nil, errors.New("no imei")
	}
	// we make sure we obtain the sim card information.
	var wg sync.WaitGroup
	done := time.After(20 * time.Second)
	wg.Add(1)
	go func() {
		defer wg.Done()
	END:
		for {
			select {
			case <-done:
				fmt.Println("Timed out")
				break END
			default:
				imsi, err := c.Run("AT+CIMI")
				if err != nil {
					continue
				}
				s, err := cleanResult(imsi)
				if err != nil {
					continue
				}
				ss := string(s)
				if !isNumber(ss) {
					continue
				}
				if m.IMEI == ss {
					continue
				}
				fmt.Println(string(s))
				m.IMSI = ss
				break END
			}
		}
	}()
	wg.Wait()
	if m.IMSI == "" {
		return nil, errors.New(" can't find IMSI")
	}
	m.conn = c
	m.Path = c.device.Name
	return m, nil
}

func isNumber(src string) bool {
	for _, v := range src {
		if !unicode.IsDigit(v) {
			return false
		}
	}
	return true
}

func (m *Manager) reload() {
	for _, v := range m.devices {
		conn := &Conn{device: v}
		modem, err := newModem(conn)
		if err != nil {
			continue
		}
		fmt.Println(*modem)
		m.modems[modem.IMEI] = modem
	}
}

func cleanResult(src []byte) ([]byte, error) {
	i := bytes.Index(src, []byte("OK"))
	if i == -1 {
		return nil, errors.New("not okay")
	}
	ns := bytes.TrimSpace(src[:i])
	ch, _ := utf8.DecodeRune(ns)
	if unicode.IsLetter(ch) {
		at := bytes.Index(ns, []byte("AT"))
		if at != -1 {
			i := bytes.IndexRune(ns[at:], '\r')
			if i > 0 {
				return bytes.TrimSpace(ns[at+i:]), nil
			}
		}
	}
	return ns, nil
}

//Close shuts down the device manager. This makes sure the udev monitor is
//closed and all goroutines are properly exited.
func (m *Manager) Close() {
	m.stop <- struct{}{}
}

func reader(ws *websocket.Conn) {
	defer ws.Close()
	for {
		_, _, err := ws.ReadMessage()
		if err != nil {
			log.Println(err)
		}
	}
}

// Updates is websocket http handler for sending updates about the devices
// plugged into the system in real time.
func (m *Manager) Updates(w http.ResponseWriter, r *http.Request) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		if _, ok := err.(websocket.HandshakeError); !ok {
			log.Println(err)
		}
		return
	}
	go func() {
		for {
			select {
			case ev := <-m.events:
				_ = ws.WriteJSON(ev)
			}
		}
	}()
	reader(ws)
}

// Conn is a device serial connection
type Conn struct {
	device serial.Config
	imei   string
	port   *serial.Port
	isOpen bool
}

// Open opens a serial port to the undelying device
func (c *Conn) Open() error {
	p, err := serial.OpenPort(&c.device)
	if err != nil {
		return err
	}
	c.port = p
	c.isOpen = true
	return nil
}

// Close closes the port helt by *Conn.
func (c *Conn) Close() error {
	if c.isOpen {
		return c.port.Close()
	}
	return nil
}

// Write wites b to the serieal port
func (c *Conn) Write(b []byte) (int, error) {
	return c.port.Write(b)
}

// Read reads from serial port
func (c *Conn) Read(b []byte) (int, error) {
	return c.port.Read(b)
}

// Exec sends the command over serial port and rrturns the response. If the port
// is closed it is opened  before sending the command.
func (c *Conn) Exec(cmd string) ([]byte, error) {
	if !c.isOpen {
		err := c.Open()
		if err != nil {
			return nil, err
		}
	}
	_, err := c.Write([]byte(cmd))
	if err != nil {
		return nil, err
	}
	buf := make([]byte, 128)
	_, err = c.Read(buf)
	if err != nil {
		return nil, err
	}
	if !bytes.Contains(buf, []byte("OK")) {
		return nil, errors.New("command " + string(cmd) + " xeite without OK" + " got " + string(buf))
	}
	_ = c.port.Flush()
	_ = c.port.Close()
	c.isOpen = false
	return buf, nil
}

// Run helper for Exec that adds \r to the command
func (c *Conn) Run(cmd string) ([]byte, error) {
	return c.Exec(fmt.Sprintf("%s \r", cmd))
}
<commit_msg>Symlink already initialized modems<commit_after>package device

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/gorilla/websocket"
	"github.com/jochenvg/go-udev"
	"github.com/tarm/serial"
)

var modemCommands = struct {
	IMEI, IMSI string
}{
	"AT+GSN", "AT+CIMI",
}
var upgrader = websocket.Upgrader{}

type Message struct {
	Type string                 `json:"type"`
	Data map[string]interface{} `json:"data"`
}

// Manager manages devices that are plugged into the system. It supports auto
// detection of devices.
//
// Serial ports are opened each for a device, and a clean API for communicating
// is provided via Read, Write and Flush methods.
//
// The devices are monitored via udev, and any changes that requires reloading
// of the  ports are handled by reloading the ports to the devices.
//
// This is safe to use concurrently in multiple goroutines
type Manager struct {
	devices map[string]serial.Config
	modems  map[string]*Modem
	mu      sync.RWMutex
	monitor *udev.Monitor
	done    chan struct{}
	stop    chan struct{}
	events  chan *Message
}

// New returns a new Manager instance
func New() *Manager {
	return &Manager{
		devices: make(map[string]serial.Config),
		modems:  make(map[string]*Modem),
		done:    make(chan struct{}),
		stop:    make(chan struct{}),
		events:  make(chan *Message),
	}
}

// Init initializes the manager. This involves creating a new goroutine to watch
// over the changes detected by udev for any device interaction with the system.
//
// The only interesting device actions are add and reomove for adding and
// removing devices respctively.
func (m *Manager) Init() {
	m.startup()
	u := udev.Udev{}
	monitor := u.NewMonitorFromNetlink("udev")
	monitor.FilterAddMatchTag("systemd")
	devCh, err := monitor.DeviceChan(m.done)
	if err != nil {
		panic(err)
	}
	m.monitor = monitor
	go func() {
	stop:
		for {
			select {
			case d := <-devCh:
				dpath := filepath.Join("/dev", filepath.Base(d.Devpath()))
				switch d.Action() {
				case "add":
					fmt.Println(" add ", dpath)
					m.AddDevice(d)
					fmt.Println(" done adding ", dpath)
				case "remove":
					fmt.Println(" removed device " + dpath)
					m.RemoveDevice(dpath)
				}
			case quit := <-m.stop:
				m.done <- quit
				break stop
			}
		}
	}()

}

func (m *Manager) startup() {
	u := udev.Udev{}
	e := u.NewEnumerate()
	e.AddMatchIsInitialized()
	e.AddMatchTag("systemd")
	devices, _ := e.Devices()
	for i := 0; i < len(devices); i++ {
		device := devices[i]
		m.AddDevice(device)
	}
}

// AddDevice adds device name to the manager
//
// WARNING: The way modems are picked is a hack. It asserts that the modem with
// the lowest tty number is the control modem( Which I'm not so sure is always
// correct).
//
// TODO: comeup with a proper way to identify modems
func (m *Manager) AddDevice(d *udev.Device) error {
	err := m.addDevice(d)
	if err != nil {
		return err
	}
	return m.Symlink()
}
func (m *Manager) addDevice(d *udev.Device) error {
	name := filepath.Join("/dev", filepath.Base(d.Devpath()))
	cfg := serial.Config{Name: name, Baud: 9600, ReadTimeout: 10 * time.Second}
	conn := &Conn{device: cfg}
	if strings.Contains(name, "ttyUSB") {
		fmt.Println("checking modem")
		modem, err := newModem(conn)
		if err != nil {
			return err
		}
		fmt.Println("found ", *modem)
		if mm, ok := m.getModem(modem.IMEI); ok {
			n1, err := getttyNum(mm.Path)
			if err != nil {
				return err
			}
			n2, err := getttyNum(modem.Path)
			if err != nil {
				return err
			}
			if n1 > n2 {
				m.setModem(modem)
			}
			return nil
		}
		m.setModem(modem)
	}
	return nil
}

func (m *Manager) Symlink() error {
	fmt.Println("SYMLINKING")
	for _, v := range m.modems {
		err := v.Symlink()
		if err != nil {
			fmt.Printf(" ERROR %v\n", err)
			continue
		}
		fmt.Println("SYMLINKING SUCCESS " + v.IMEI)
	}
	return nil
}

func (m *Manager) setModem(mod *Modem) {
	fmt.Println("SETTING UP " + mod.IMEI)
	m.mu.Lock()
	m.modems[mod.IMEI] = mod
	m.mu.Unlock()
}

func (m *Manager) getModem(imei string) (*Modem, bool) {
	m.mu.RLock()
	mod, ok := m.modems[imei]
	m.mu.RUnlock()
	return mod, ok
}

func getttyNum(tty string) (int, error) {
	b := filepath.Base(tty)
	b = strings.TrimPrefix(b, "ttyUSB")
	return strconv.Atoi(b)
}

// List serves the list of current devices. The list wont cover all devices ,
// only the significant ones( modems for now)
func (m *Manager) List(w http.ResponseWriter, r *http.Request) {
	m.mu.RLock()
	json.NewEncoder(w).Encode(m.modems)
	m.mu.RUnlock()
	w.Header().Set("Content-Type", "application/json")
}

// RunComand runs commands to the exposed devices over serial ports
func (m *Manager) RunCommand(w http.ResponseWriter, r *http.Request) {
}

// RemoveDevice removes device name from the manager
func (m *Manager) RemoveDevice(name string) error {
	return nil
}

type Modem struct {
	IMEI         string `json:"imei"`
	IMSI         string `json:"imsi"`
	Manufacturer string `json:"manufacturer"`
	Path         string `json:"tty"`
	conn         *Conn
}

// Symlink adds symlink to the  modem. The symlink links the tty to the IMEI
// number of the modem.
func (m *Modem) Symlink() error {
	newLink := (fmt.Sprintf("/dev/%s", m.IMEI))
	err := syscall.Unlink(newLink)
	if err != nil {
		fmt.Printf(" ERROR %v \n", err)
	}
	return os.Symlink(m.Path, newLink)
}

func newModem(c *Conn) (*Modem, error) {
	m := &Modem{}
	ich := time.After(10 * time.Second)
STOP:
	for {
		select {
		case <-ich:
			break STOP
		default:
			imei, err := c.Run(modemCommands.IMEI)
			if err != nil {
				continue
			}
			i, err := cleanResult(imei)
			if err != nil {
				continue
			}
			im := string(i)
			if !isNumber(im) {
				continue
			}
			m.IMEI = im
			break STOP
		}
	}
	if m.IMEI == "" {
		return nil, errors.New("no imei")
	}
	// we make sure we obtain the sim card information.
	var wg sync.WaitGroup
	done := time.After(20 * time.Second)
	wg.Add(1)
	go func() {
		defer wg.Done()
	END:
		for {
			select {
			case <-done:
				fmt.Println("Timed out")
				break END
			default:
				imsi, err := c.Run("AT+CIMI")
				if err != nil {
					continue
				}
				s, err := cleanResult(imsi)
				if err != nil {
					continue
				}
				ss := string(s)
				if !isNumber(ss) {
					continue
				}
				if m.IMEI == ss {
					continue
				}
				fmt.Println(string(s))
				m.IMSI = ss
				break END
			}
		}
	}()
	wg.Wait()
	if m.IMSI == "" {
		return nil, errors.New(" can't find IMSI")
	}
	m.conn = c
	m.Path = c.device.Name
	return m, nil
}

func isNumber(src string) bool {
	for _, v := range src {
		if !unicode.IsDigit(v) {
			return false
		}
	}
	return true
}

func (m *Manager) reload() {
	for _, v := range m.devices {
		conn := &Conn{device: v}
		modem, err := newModem(conn)
		if err != nil {
			continue
		}
		fmt.Println(*modem)
		m.modems[modem.IMEI] = modem
	}
}

func cleanResult(src []byte) ([]byte, error) {
	i := bytes.Index(src, []byte("OK"))
	if i == -1 {
		return nil, errors.New("not okay")
	}
	ns := bytes.TrimSpace(src[:i])
	ch, _ := utf8.DecodeRune(ns)
	if unicode.IsLetter(ch) {
		at := bytes.Index(ns, []byte("AT"))
		if at != -1 {
			i := bytes.IndexRune(ns[at:], '\r')
			if i > 0 {
				return bytes.TrimSpace(ns[at+i:]), nil
			}
		}
	}
	return ns, nil
}

//Close shuts down the device manager. This makes sure the udev monitor is
//closed and all goroutines are properly exited.
func (m *Manager) Close() {
	m.stop <- struct{}{}
}

func reader(ws *websocket.Conn) {
	defer ws.Close()
	for {
		_, _, err := ws.ReadMessage()
		if err != nil {
			log.Println(err)
		}
	}
}

// Updates is websocket http handler for sending updates about the devices
// plugged into the system in real time.
func (m *Manager) Updates(w http.ResponseWriter, r *http.Request) {
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		if _, ok := err.(websocket.HandshakeError); !ok {
			log.Println(err)
		}
		return
	}
	go func() {
		for {
			select {
			case ev := <-m.events:
				_ = ws.WriteJSON(ev)
			}
		}
	}()
	reader(ws)
}

// Conn is a device serial connection
type Conn struct {
	device serial.Config
	imei   string
	port   *serial.Port
	isOpen bool
}

// Open opens a serial port to the undelying device
func (c *Conn) Open() error {
	p, err := serial.OpenPort(&c.device)
	if err != nil {
		return err
	}
	c.port = p
	c.isOpen = true
	return nil
}

// Close closes the port helt by *Conn.
func (c *Conn) Close() error {
	if c.isOpen {
		return c.port.Close()
	}
	return nil
}

// Write wites b to the serieal port
func (c *Conn) Write(b []byte) (int, error) {
	return c.port.Write(b)
}

// Read reads from serial port
func (c *Conn) Read(b []byte) (int, error) {
	return c.port.Read(b)
}

// Exec sends the command over serial port and rrturns the response. If the port
// is closed it is opened  before sending the command.
func (c *Conn) Exec(cmd string) ([]byte, error) {
	if !c.isOpen {
		err := c.Open()
		if err != nil {
			return nil, err
		}
	}
	_, err := c.Write([]byte(cmd))
	if err != nil {
		return nil, err
	}
	buf := make([]byte, 128)
	_, err = c.Read(buf)
	if err != nil {
		return nil, err
	}
	if !bytes.Contains(buf, []byte("OK")) {
		return nil, errors.New("command " + string(cmd) + " xeite without OK" + " got " + string(buf))
	}
	_ = c.port.Flush()
	_ = c.port.Close()
	c.isOpen = false
	return buf, nil
}

// Run helper for Exec that adds \r to the command
func (c *Conn) Run(cmd string) ([]byte, error) {
	return c.Exec(fmt.Sprintf("%s \r", cmd))
}
<|endoftext|>


<commit_before>package device

import (
	"fmt"
)

// deviceError exposes the basic error metadata
type deviceError struct {
	id  ID
	key Key
}

func (e *deviceError) ID() ID {
	return e.id
}

func (e *deviceError) Key() Key {
	return e.key
}

// ClosedError indicates that an operation was attempted on
// a closed device that is not allowed, e.g. sending a message
type ClosedError struct {
	deviceError
}

func (e *ClosedError) Error() string {
	return fmt.Sprintf("Device [%s] with key [%s] is closed", e.id, e.key)
}

func NewClosedError(id ID, key Key) *ClosedError {
	return &ClosedError{
		deviceError{
			id:  id,
			key: key,
		},
	}
}

// BusyError indicates that a device's message queue is full
type BusyError struct {
	deviceError
}

func (e *BusyError) Error() string {
	return fmt.Sprintf("Device [%s] with key [%s] is busy", e.id, e.key)
}

func NewBusyError(id ID, key Key) *BusyError {
	return &BusyError{
		deviceError{
			id:  id,
			key: key,
		},
	}
}

type MissingIDError struct {
	deviceError
}

func (e *MissingIDError) Error() string {
	return fmt.Sprintf("No device exists with id [%s]", e.id)
}

func NewMissingIDError(id ID) *MissingIDError {
	return &MissingIDError{
		deviceError{
			id: id,
		},
	}
}

type MissingKeyError struct {
	deviceError
}

func (e *MissingKeyError) Error() string {
	return fmt.Sprintf("No device exists with key [%s]", e.key)
}

func NewMissingKeyError(key Key) *MissingKeyError {
	return &MissingKeyError{
		deviceError{
			key: key,
		},
	}
}

type DuplicateKeyError struct {
	deviceError
}

func (e *DuplicateKeyError) Error() string {
	return fmt.Sprintf("Duplicate key [%s]", e.key)
}

func NewDuplicateKeyError(key Key) *DuplicateKeyError {
	return &DuplicateKeyError{
		deviceError{
			key: key,
		},
	}
}
<commit_msg>Normalized the errors into fewer types<commit_after>package device

import (
	"fmt"
)

// DeviceError is the common interface implemented by all error objects
// which carry device-related metadata
type DeviceError interface {
	error
	ID() ID
	Key() Key
}

// deviceError exposes the basic error metadata
type deviceError struct {
	id   ID
	key  Key
	text string
}

func (e *deviceError) ID() ID {
	return e.id
}

func (e *deviceError) Key() Key {
	return e.key
}

func (e *deviceError) Error() string {
	return e.text
}

func newDeviceError(id ID, key Key, message string) DeviceError {
	return &deviceError{
		id:   id,
		key:  key,
		text: fmt.Sprintf("Device [id=%s, key=%s]: %s", id, key, message),
	}
}

func NewClosedError(id ID, key Key) DeviceError {
	return newDeviceError(id, key, "closed")
}

func NewBusyError(id ID, key Key) DeviceError {
	return newDeviceError(id, key, "busy")
}

func NewMissingIDError(id ID) DeviceError {
	return newDeviceError(id, invalidKey, "ID does not exist")
}

func NewMissingKeyError(key Key) DeviceError {
	return newDeviceError(invalidID, key, "Key does not exist")
}

func NewDuplicateKeyError(key Key) DeviceError {
	return newDeviceError(invalidID, key, "duplicate key")
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 Chadev. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "testing"

func TestGetMeetupEvents(t *testing.T) {
	_, err := getTalkDetails()
	if err != nil {
		t.Error(err)
	}
}
<commit_msg>Fix tests on Travis CI<commit_after>// Copyright 2014-2015 Chadev. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"os"
	"testing"
)

func TestGetMeetupEvents(t *testing.T) {
	if os.Getenv("CHADEV_MEETUP") == "" {
		t.Skip("no meetup API key set, skipping test")
	}

	_, err := getTalkDetails()
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package btelegram

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type Btelegram struct {
	c *tgbotapi.BotAPI
	*config.BridgeConfig
	avatarMap map[string]string // keep cache of userid and avatar sha
}

func New(cfg *config.BridgeConfig) bridge.Bridger {
	return &Btelegram{BridgeConfig: cfg, avatarMap: make(map[string]string)}
}

func (b *Btelegram) Connect() error {
	var err error
	b.Log.Info("Connecting")
	b.c, err = tgbotapi.NewBotAPI(b.Config.Token)
	if err != nil {
		b.Log.Debugf("%#v", err)
		return err
	}
	u := tgbotapi.NewUpdate(0)
	u.Timeout = 60
	updates, err := b.c.GetUpdatesChan(u)
	if err != nil {
		b.Log.Debugf("%#v", err)
		return err
	}
	b.Log.Info("Connection succeeded")
	go b.handleRecv(updates)
	return nil
}

func (b *Btelegram) Disconnect() error {
	return nil
}

func (b *Btelegram) JoinChannel(channel config.ChannelInfo) error {
	return nil
}

func (b *Btelegram) Send(msg config.Message) (string, error) {
	b.Log.Debugf("=> Receiving %#v", msg)

	// get the chatid
	chatid, err := strconv.ParseInt(msg.Channel, 10, 64)
	if err != nil {
		return "", err
	}

	// map the file SHA to our user (caches the avatar)
	if msg.Event == config.EVENT_AVATAR_DOWNLOAD {
		return b.cacheAvatar(&msg)
	}

	if b.Config.MessageFormat == "HTML" {
		msg.Text = makeHTML(msg.Text)
	}

	// Delete message
	if msg.Event == config.EVENT_MSG_DELETE {
		if msg.ID == "" {
			return "", nil
		}
		msgid, err := strconv.Atoi(msg.ID)
		if err != nil {
			return "", err
		}
		_, err = b.c.DeleteMessage(tgbotapi.DeleteMessageConfig{ChatID: chatid, MessageID: msgid})
		return "", err
	}

	// Upload a file if it exists
	if msg.Extra != nil {
		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
			b.sendMessage(chatid, rmsg.Username+rmsg.Text)
		}
		// check if we have files to upload (from slack, telegram or mattermost)
		if len(msg.Extra["file"]) > 0 {
			b.handleUploadFile(&msg, chatid)
		}
	}

	// edit the message if we have a msg ID
	if msg.ID != "" {
		msgid, err := strconv.Atoi(msg.ID)
		if err != nil {
			return "", err
		}
		m := tgbotapi.NewEditMessageText(chatid, msgid, msg.Username+msg.Text)
		if b.Config.MessageFormat == "HTML" {
			b.Log.Debug("Using mode HTML")
			m.ParseMode = tgbotapi.ModeHTML
		}
		if b.Config.MessageFormat == "Markdown" {
			b.Log.Debug("Using mode markdown")
			m.ParseMode = tgbotapi.ModeMarkdown
		}
		_, err = b.c.Send(m)
		if err != nil {
			return "", err
		}
		return "", nil
	}

	// Post normal message
	return b.sendMessage(chatid, msg.Username+msg.Text)
}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		b.Log.Debugf("== Receiving event: %#v", update.Message)

		if update.Message == nil && update.ChannelPost == nil {
			b.Log.Error("Getting nil messages, this shouldn't happen.")
			continue
		}

		var message *tgbotapi.Message

		rmsg := config.Message{Account: b.Account, Extra: make(map[string][]interface{})}

		// handle channels
		if update.ChannelPost != nil {
			message = update.ChannelPost
		}

		// edited channel message
		if update.EditedChannelPost != nil && !b.Config.EditDisable {
			message = update.EditedChannelPost
			rmsg.Text = rmsg.Text + message.Text + b.Config.EditSuffix
		}

		// handle groups
		if update.Message != nil {
			message = update.Message
		}

		// edited group message
		if update.EditedMessage != nil && !b.Config.EditDisable {
			message = update.EditedMessage
			rmsg.Text = rmsg.Text + message.Text + b.Config.EditSuffix
		}

		// set the ID's from the channel or group message
		rmsg.ID = strconv.Itoa(message.MessageID)
		rmsg.UserID = strconv.Itoa(message.From.ID)
		rmsg.Channel = strconv.FormatInt(message.Chat.ID, 10)

		// handle username
		if message.From != nil {
			if b.Config.UseFirstName {
				rmsg.Username = message.From.FirstName
			}
			if rmsg.Username == "" {
				rmsg.Username = message.From.UserName
				if rmsg.Username == "" {
					rmsg.Username = message.From.FirstName
				}
			}
			// only download avatars if we have a place to upload them (configured mediaserver)
			if b.General.MediaServerUpload != "" {
				b.handleDownloadAvatar(message.From.ID, rmsg.Channel)
			}
		}

		// if we really didn't find a username, set it to unknown
		if rmsg.Username == "" {
			rmsg.Username = "unknown"
		}

		// handle any downloads
		err := b.handleDownload(message, &rmsg)
		if err != nil {
			b.Log.Errorf("download failed: %s", err)
		}

		// handle forwarded messages
		if message.ForwardFrom != nil {
			usernameForward := ""
			if b.Config.UseFirstName {
				usernameForward = message.ForwardFrom.FirstName
			}
			if usernameForward == "" {
				usernameForward = message.ForwardFrom.UserName
				if usernameForward == "" {
					usernameForward = message.ForwardFrom.FirstName
				}
			}
			if usernameForward == "" {
				usernameForward = "unknown"
			}
			rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
		}

		// quote the previous message
		if message.ReplyToMessage != nil {
			usernameReply := ""
			if message.ReplyToMessage.From != nil {
				if b.Config.UseFirstName {
					usernameReply = message.ReplyToMessage.From.FirstName
				}
				if usernameReply == "" {
					usernameReply = message.ReplyToMessage.From.UserName
					if usernameReply == "" {
						usernameReply = message.ReplyToMessage.From.FirstName
					}
				}
			}
			if usernameReply == "" {
				usernameReply = "unknown"
			}
			rmsg.Text = rmsg.Text + " (re @" + usernameReply + ":" + message.ReplyToMessage.Text + ")"
		}

		if rmsg.Text != "" || len(rmsg.Extra) > 0 {
			rmsg.Avatar = helper.GetAvatar(b.avatarMap, strconv.Itoa(message.From.ID), b.General)

			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
			b.Log.Debugf("<= Message is %#v", rmsg)
			b.Remote <- rmsg
		}
	}
}

func (b *Btelegram) getFileDirectURL(id string) string {
	res, err := b.c.GetFileDirectURL(id)
	if err != nil {
		return ""
	}
	return res
}

// handleDownloadAvatar downloads the avatar of userid from channel
// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
// logs an error message if it fails
func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
	rmsg := config.Message{Username: "system", Text: "avatar", Channel: channel, Account: b.Account, UserID: strconv.Itoa(userid), Event: config.EVENT_AVATAR_DOWNLOAD, Extra: make(map[string][]interface{})}
	if _, ok := b.avatarMap[strconv.Itoa(userid)]; !ok {
		photos, err := b.c.GetUserProfilePhotos(tgbotapi.UserProfilePhotosConfig{UserID: userid, Limit: 1})
		if err != nil {
			b.Log.Errorf("Userprofile download failed for %#v %s", userid, err)
		}

		if len(photos.Photos) > 0 {
			photo := photos.Photos[0][0]
			url := b.getFileDirectURL(photo.FileID)
			name := strconv.Itoa(userid) + ".png"
			b.Log.Debugf("trying to download %#v fileid %#v with size %#v", name, photo.FileID, photo.FileSize)

			err := helper.HandleDownloadSize(b.Log, &rmsg, name, int64(photo.FileSize), b.General)
			if err != nil {
				b.Log.Error(err)
				return
			}
			data, err := helper.DownloadFile(url)
			if err != nil {
				b.Log.Errorf("download %s failed %#v", url, err)
				return
			}
			helper.HandleDownloadData(b.Log, &rmsg, name, rmsg.Text, "", data, b.General)
			b.Remote <- rmsg
		}
	}
}

// handleDownloadFile handles file download
func (b *Btelegram) handleDownload(message *tgbotapi.Message, rmsg *config.Message) error {
	size := 0
	var url, name, text string

	if message.Sticker != nil {
		v := message.Sticker
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		if !strings.HasSuffix(name, ".webp") {
			name = name + ".webp"
		}
		text = " " + url
	}
	if message.Video != nil {
		v := message.Video
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		text = " " + url
	}
	if message.Photo != nil {
		photos := *message.Photo
		size = photos[len(photos)-1].FileSize
		url = b.getFileDirectURL(photos[len(photos)-1].FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		text = " " + url
	}
	if message.Document != nil {
		v := message.Document
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		name = v.FileName
		text = " " + v.FileName + " : " + url
	}
	if message.Voice != nil {
		v := message.Voice
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		text = " " + url
		if !strings.HasSuffix(name, ".ogg") {
			name = name + ".ogg"
		}
	}
	if message.Audio != nil {
		v := message.Audio
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		text = " " + url
	}
	// if name is empty we didn't match a thing to download
	if name == "" {
		return nil
	}
	// use the URL instead of native upload
	if b.Config.UseInsecureURL {
		b.Log.Debugf("Setting message text to :%s", text)
		rmsg.Text = rmsg.Text + text
		return nil
	}
	// if we have a file attached, download it (in memory) and put a pointer to it in msg.Extra
	err := helper.HandleDownloadSize(b.Log, rmsg, name, int64(size), b.General)
	if err != nil {
		return err
	}
	data, err := helper.DownloadFile(url)
	if err != nil {
		return err
	}
	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
	return nil
}

// handleUploadFile handles native upload of files
func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64) (string, error) {
	var c tgbotapi.Chattable
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		file := tgbotapi.FileBytes{fi.Name, *fi.Data}
		re := regexp.MustCompile(".(jpg|png)$")
		if re.MatchString(fi.Name) {
			c = tgbotapi.NewPhotoUpload(chatid, file)
		} else {
			c = tgbotapi.NewDocumentUpload(chatid, file)
		}
		_, err := b.c.Send(c)
		if err != nil {
			b.Log.Errorf("file upload failed: %#v", err)
		}
		if fi.Comment != "" {
			b.sendMessage(chatid, msg.Username+fi.Comment)
		}
	}
	return "", nil
}

func (b *Btelegram) sendMessage(chatid int64, text string) (string, error) {
	m := tgbotapi.NewMessage(chatid, text)
	if b.Config.MessageFormat == "HTML" {
		b.Log.Debug("Using mode HTML")
		m.ParseMode = tgbotapi.ModeHTML
	}
	if b.Config.MessageFormat == "Markdown" {
		b.Log.Debug("Using mode markdown")
		m.ParseMode = tgbotapi.ModeMarkdown
	}
	res, err := b.c.Send(m)
	if err != nil {
		return "", err
	}
	return strconv.Itoa(res.MessageID), nil
}

func (b *Btelegram) cacheAvatar(msg *config.Message) (string, error) {
	fi := msg.Extra["file"][0].(config.FileInfo)
	/* if we have a sha we have successfully uploaded the file to the media server,
	so we can now cache the sha */
	if fi.SHA != "" {
		b.Log.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID)
		b.avatarMap[msg.UserID] = fi.SHA
	}
	return "", nil
}
<commit_msg>Escape html on username (telegram). Closes #378<commit_after>package btelegram

import (
	"html"
	"regexp"
	"strconv"
	"strings"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type Btelegram struct {
	c *tgbotapi.BotAPI
	*config.BridgeConfig
	avatarMap map[string]string // keep cache of userid and avatar sha
}

func New(cfg *config.BridgeConfig) bridge.Bridger {
	return &Btelegram{BridgeConfig: cfg, avatarMap: make(map[string]string)}
}

func (b *Btelegram) Connect() error {
	var err error
	b.Log.Info("Connecting")
	b.c, err = tgbotapi.NewBotAPI(b.Config.Token)
	if err != nil {
		b.Log.Debugf("%#v", err)
		return err
	}
	u := tgbotapi.NewUpdate(0)
	u.Timeout = 60
	updates, err := b.c.GetUpdatesChan(u)
	if err != nil {
		b.Log.Debugf("%#v", err)
		return err
	}
	b.Log.Info("Connection succeeded")
	go b.handleRecv(updates)
	return nil
}

func (b *Btelegram) Disconnect() error {
	return nil
}

func (b *Btelegram) JoinChannel(channel config.ChannelInfo) error {
	return nil
}

func (b *Btelegram) Send(msg config.Message) (string, error) {
	b.Log.Debugf("=> Receiving %#v", msg)

	// get the chatid
	chatid, err := strconv.ParseInt(msg.Channel, 10, 64)
	if err != nil {
		return "", err
	}

	// map the file SHA to our user (caches the avatar)
	if msg.Event == config.EVENT_AVATAR_DOWNLOAD {
		return b.cacheAvatar(&msg)
	}

	if b.Config.MessageFormat == "HTML" {
		msg.Text = makeHTML(msg.Text)
	}

	// Delete message
	if msg.Event == config.EVENT_MSG_DELETE {
		if msg.ID == "" {
			return "", nil
		}
		msgid, err := strconv.Atoi(msg.ID)
		if err != nil {
			return "", err
		}
		_, err = b.c.DeleteMessage(tgbotapi.DeleteMessageConfig{ChatID: chatid, MessageID: msgid})
		return "", err
	}

	// Upload a file if it exists
	if msg.Extra != nil {
		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
			b.sendMessage(chatid, rmsg.Username, rmsg.Text)
		}
		// check if we have files to upload (from slack, telegram or mattermost)
		if len(msg.Extra["file"]) > 0 {
			b.handleUploadFile(&msg, chatid)
		}
	}

	// edit the message if we have a msg ID
	if msg.ID != "" {
		msgid, err := strconv.Atoi(msg.ID)
		if err != nil {
			return "", err
		}
		m := tgbotapi.NewEditMessageText(chatid, msgid, msg.Username+msg.Text)
		if b.Config.MessageFormat == "HTML" {
			b.Log.Debug("Using mode HTML")
			m.ParseMode = tgbotapi.ModeHTML
		}
		if b.Config.MessageFormat == "Markdown" {
			b.Log.Debug("Using mode markdown")
			m.ParseMode = tgbotapi.ModeMarkdown
		}
		_, err = b.c.Send(m)
		if err != nil {
			return "", err
		}
		return "", nil
	}

	// Post normal message
	return b.sendMessage(chatid, msg.Username, msg.Text)
}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		b.Log.Debugf("== Receiving event: %#v", update.Message)

		if update.Message == nil && update.ChannelPost == nil {
			b.Log.Error("Getting nil messages, this shouldn't happen.")
			continue
		}

		var message *tgbotapi.Message

		rmsg := config.Message{Account: b.Account, Extra: make(map[string][]interface{})}

		// handle channels
		if update.ChannelPost != nil {
			message = update.ChannelPost
		}

		// edited channel message
		if update.EditedChannelPost != nil && !b.Config.EditDisable {
			message = update.EditedChannelPost
			rmsg.Text = rmsg.Text + message.Text + b.Config.EditSuffix
		}

		// handle groups
		if update.Message != nil {
			message = update.Message
		}

		// edited group message
		if update.EditedMessage != nil && !b.Config.EditDisable {
			message = update.EditedMessage
			rmsg.Text = rmsg.Text + message.Text + b.Config.EditSuffix
		}

		// set the ID's from the channel or group message
		rmsg.ID = strconv.Itoa(message.MessageID)
		rmsg.UserID = strconv.Itoa(message.From.ID)
		rmsg.Channel = strconv.FormatInt(message.Chat.ID, 10)

		// handle username
		if message.From != nil {
			if b.Config.UseFirstName {
				rmsg.Username = message.From.FirstName
			}
			if rmsg.Username == "" {
				rmsg.Username = message.From.UserName
				if rmsg.Username == "" {
					rmsg.Username = message.From.FirstName
				}
			}
			// only download avatars if we have a place to upload them (configured mediaserver)
			if b.General.MediaServerUpload != "" {
				b.handleDownloadAvatar(message.From.ID, rmsg.Channel)
			}
		}

		// if we really didn't find a username, set it to unknown
		if rmsg.Username == "" {
			rmsg.Username = "unknown"
		}

		// handle any downloads
		err := b.handleDownload(message, &rmsg)
		if err != nil {
			b.Log.Errorf("download failed: %s", err)
		}

		// handle forwarded messages
		if message.ForwardFrom != nil {
			usernameForward := ""
			if b.Config.UseFirstName {
				usernameForward = message.ForwardFrom.FirstName
			}
			if usernameForward == "" {
				usernameForward = message.ForwardFrom.UserName
				if usernameForward == "" {
					usernameForward = message.ForwardFrom.FirstName
				}
			}
			if usernameForward == "" {
				usernameForward = "unknown"
			}
			rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
		}

		// quote the previous message
		if message.ReplyToMessage != nil {
			usernameReply := ""
			if message.ReplyToMessage.From != nil {
				if b.Config.UseFirstName {
					usernameReply = message.ReplyToMessage.From.FirstName
				}
				if usernameReply == "" {
					usernameReply = message.ReplyToMessage.From.UserName
					if usernameReply == "" {
						usernameReply = message.ReplyToMessage.From.FirstName
					}
				}
			}
			if usernameReply == "" {
				usernameReply = "unknown"
			}
			rmsg.Text = rmsg.Text + " (re @" + usernameReply + ":" + message.ReplyToMessage.Text + ")"
		}

		if rmsg.Text != "" || len(rmsg.Extra) > 0 {
			rmsg.Avatar = helper.GetAvatar(b.avatarMap, strconv.Itoa(message.From.ID), b.General)

			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
			b.Log.Debugf("<= Message is %#v", rmsg)
			b.Remote <- rmsg
		}
	}
}

func (b *Btelegram) getFileDirectURL(id string) string {
	res, err := b.c.GetFileDirectURL(id)
	if err != nil {
		return ""
	}
	return res
}

// handleDownloadAvatar downloads the avatar of userid from channel
// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
// logs an error message if it fails
func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
	rmsg := config.Message{Username: "system", Text: "avatar", Channel: channel, Account: b.Account, UserID: strconv.Itoa(userid), Event: config.EVENT_AVATAR_DOWNLOAD, Extra: make(map[string][]interface{})}
	if _, ok := b.avatarMap[strconv.Itoa(userid)]; !ok {
		photos, err := b.c.GetUserProfilePhotos(tgbotapi.UserProfilePhotosConfig{UserID: userid, Limit: 1})
		if err != nil {
			b.Log.Errorf("Userprofile download failed for %#v %s", userid, err)
		}

		if len(photos.Photos) > 0 {
			photo := photos.Photos[0][0]
			url := b.getFileDirectURL(photo.FileID)
			name := strconv.Itoa(userid) + ".png"
			b.Log.Debugf("trying to download %#v fileid %#v with size %#v", name, photo.FileID, photo.FileSize)

			err := helper.HandleDownloadSize(b.Log, &rmsg, name, int64(photo.FileSize), b.General)
			if err != nil {
				b.Log.Error(err)
				return
			}
			data, err := helper.DownloadFile(url)
			if err != nil {
				b.Log.Errorf("download %s failed %#v", url, err)
				return
			}
			helper.HandleDownloadData(b.Log, &rmsg, name, rmsg.Text, "", data, b.General)
			b.Remote <- rmsg
		}
	}
}

// handleDownloadFile handles file download
func (b *Btelegram) handleDownload(message *tgbotapi.Message, rmsg *config.Message) error {
	size := 0
	var url, name, text string

	if message.Sticker != nil {
		v := message.Sticker
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		if !strings.HasSuffix(name, ".webp") {
			name = name + ".webp"
		}
		text = " " + url
	}
	if message.Video != nil {
		v := message.Video
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		text = " " + url
	}
	if message.Photo != nil {
		photos := *message.Photo
		size = photos[len(photos)-1].FileSize
		url = b.getFileDirectURL(photos[len(photos)-1].FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		text = " " + url
	}
	if message.Document != nil {
		v := message.Document
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		name = v.FileName
		text = " " + v.FileName + " : " + url
	}
	if message.Voice != nil {
		v := message.Voice
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		text = " " + url
		if !strings.HasSuffix(name, ".ogg") {
			name = name + ".ogg"
		}
	}
	if message.Audio != nil {
		v := message.Audio
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		urlPart := strings.Split(url, "/")
		name = urlPart[len(urlPart)-1]
		text = " " + url
	}
	// if name is empty we didn't match a thing to download
	if name == "" {
		return nil
	}
	// use the URL instead of native upload
	if b.Config.UseInsecureURL {
		b.Log.Debugf("Setting message text to :%s", text)
		rmsg.Text = rmsg.Text + text
		return nil
	}
	// if we have a file attached, download it (in memory) and put a pointer to it in msg.Extra
	err := helper.HandleDownloadSize(b.Log, rmsg, name, int64(size), b.General)
	if err != nil {
		return err
	}
	data, err := helper.DownloadFile(url)
	if err != nil {
		return err
	}
	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
	return nil
}

// handleUploadFile handles native upload of files
func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64) (string, error) {
	var c tgbotapi.Chattable
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		file := tgbotapi.FileBytes{fi.Name, *fi.Data}
		re := regexp.MustCompile(".(jpg|png)$")
		if re.MatchString(fi.Name) {
			c = tgbotapi.NewPhotoUpload(chatid, file)
		} else {
			c = tgbotapi.NewDocumentUpload(chatid, file)
		}
		_, err := b.c.Send(c)
		if err != nil {
			b.Log.Errorf("file upload failed: %#v", err)
		}
		if fi.Comment != "" {
			b.sendMessage(chatid, msg.Username, fi.Comment)
		}
	}
	return "", nil
}

func (b *Btelegram) sendMessage(chatid int64, username, text string) (string, error) {
	m := tgbotapi.NewMessage(chatid, "")
	m.Text = username + text
	if b.Config.MessageFormat == "HTML" {
		b.Log.Debug("Using mode HTML")
		username = html.EscapeString(username)
		m.Text = username + text
		m.ParseMode = tgbotapi.ModeHTML
	}
	if b.Config.MessageFormat == "Markdown" {
		b.Log.Debug("Using mode markdown")
		m.ParseMode = tgbotapi.ModeMarkdown
	}
	res, err := b.c.Send(m)
	if err != nil {
		return "", err
	}
	return strconv.Itoa(res.MessageID), nil
}

func (b *Btelegram) cacheAvatar(msg *config.Message) (string, error) {
	fi := msg.Extra["file"][0].(config.FileInfo)
	/* if we have a sha we have successfully uploaded the file to the media server,
	so we can now cache the sha */
	if fi.SHA != "" {
		b.Log.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID)
		b.avatarMap[msg.UserID] = fi.SHA
	}
	return "", nil
}
<|endoftext|>


<commit_before>package btelegram

import (
	"strconv"

	"github.com/42wim/matterbridge/bridge/config"
	log "github.com/Sirupsen/logrus"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type Btelegram struct {
	c       *tgbotapi.BotAPI
	Config  *config.Protocol
	Remote  chan config.Message
	Account string
}

var flog *log.Entry
var protocol = "telegram"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(cfg config.Protocol, account string, c chan config.Message) *Btelegram {
	b := &Btelegram{}
	b.Config = &cfg
	b.Remote = c
	b.Account = account
	return b
}

func (b *Btelegram) Connect() error {
	var err error
	flog.Info("Connecting")
	b.c, err = tgbotapi.NewBotAPI(b.Config.Token)
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	updates, err := b.c.GetUpdatesChan(tgbotapi.NewUpdate(0))
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	go b.handleRecv(updates)
	return nil
}

func (b *Btelegram) Disconnect() error {
	return nil

}

func (b *Btelegram) JoinChannel(channel config.ChannelInfo) error {
	return nil
}

func (b *Btelegram) Send(msg config.Message) (string, error) {
	flog.Debugf("Receiving %#v", msg)
	chatid, err := strconv.ParseInt(msg.Channel, 10, 64)
	if err != nil {
		return "", err
	}

	if b.Config.MessageFormat == "HTML" {
		msg.Text = makeHTML(msg.Text)
	}

	if msg.Event == config.EVENT_MSG_DELETE {
		if msg.ID == "" {
			return "", nil
		}
		msgid, err := strconv.Atoi(msg.ID)
		if err != nil {
			return "", err
		}
		_, err = b.c.DeleteMessage(tgbotapi.DeleteMessageConfig{ChatID: chatid, MessageID: msgid})
		return "", err
	}

	// edit the message if we have a msg ID
	if msg.ID != "" {
		msgid, err := strconv.Atoi(msg.ID)
		if err != nil {
			return "", err
		}
		m := tgbotapi.NewEditMessageText(chatid, msgid, msg.Username+msg.Text)
		_, err = b.c.Send(m)
		if err != nil {
			return "", err
		}
		return "", nil
	}

	m := tgbotapi.NewMessage(chatid, msg.Username+msg.Text)
	if b.Config.MessageFormat == "HTML" {
		m.ParseMode = tgbotapi.ModeHTML
	}
	res, err := b.c.Send(m)
	if err != nil {
		return "", err
	}
	return strconv.Itoa(res.MessageID), nil

}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		flog.Debugf("Receiving from telegram: %#v", update.Message)
		var message *tgbotapi.Message
		username := ""
		channel := ""
		text := ""
		// handle channels
		if update.ChannelPost != nil {
			message = update.ChannelPost
		}
		if update.EditedChannelPost != nil && !b.Config.EditDisable {
			message = update.EditedChannelPost
			message.Text = message.Text + b.Config.EditSuffix
		}
		// handle groups
		if update.Message != nil {
			message = update.Message
		}
		if update.EditedMessage != nil && !b.Config.EditDisable {
			message = update.EditedMessage
			message.Text = message.Text + b.Config.EditSuffix
		}
		if message.From != nil {
			if b.Config.UseFirstName {
				username = message.From.FirstName
			}
			if username == "" {
				username = message.From.UserName
				if username == "" {
					username = message.From.FirstName
				}
			}
			text = message.Text
			channel = strconv.FormatInt(message.Chat.ID, 10)
		}

		if username == "" {
			username = "unknown"
		}
		if message.Sticker != nil && b.Config.UseInsecureURL {
			text = text + " " + b.getFileDirectURL(message.Sticker.FileID)
		}
		if message.Video != nil && b.Config.UseInsecureURL {
			text = text + " " + b.getFileDirectURL(message.Video.FileID)
		}
		if message.Photo != nil && b.Config.UseInsecureURL {
			photos := *message.Photo
			// last photo is the biggest
			text = text + " " + b.getFileDirectURL(photos[len(photos)-1].FileID)
		}
		if message.Document != nil && b.Config.UseInsecureURL {
			text = text + " " + message.Document.FileName + " : " + b.getFileDirectURL(message.Document.FileID)
		}

		// quote the previous message
		if message.ReplyToMessage != nil {
			usernameReply := ""
			if message.ReplyToMessage.From != nil {
				if b.Config.UseFirstName {
					usernameReply = message.ReplyToMessage.From.FirstName
				}
				if usernameReply == "" {
					usernameReply = message.ReplyToMessage.From.UserName
					if usernameReply == "" {
						usernameReply = message.ReplyToMessage.From.FirstName
					}
				}
			}
			if usernameReply == "" {
				usernameReply = "unknown"
			}
			text = text + " (re @" + usernameReply + ":" + message.ReplyToMessage.Text + ")"
		}

		if text != "" {
			flog.Debugf("Sending message from %s on %s to gateway", username, b.Account)
			msg := config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID), ID: strconv.Itoa(message.MessageID)}
			flog.Debugf("Message is %#v", msg)
			b.Remote <- msg
		}
	}
}

func (b *Btelegram) getFileDirectURL(id string) string {
	res, err := b.c.GetFileDirectURL(id)
	if err != nil {
		return ""
	}
	return res
}
<commit_msg>Download files from telegram and reupload to supported bridges (telegram). #278<commit_after>package btelegram

import (
	"regexp"
	"strconv"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	log "github.com/Sirupsen/logrus"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type Btelegram struct {
	c       *tgbotapi.BotAPI
	Config  *config.Protocol
	Remote  chan config.Message
	Account string
}

var flog *log.Entry
var protocol = "telegram"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(cfg config.Protocol, account string, c chan config.Message) *Btelegram {
	b := &Btelegram{}
	b.Config = &cfg
	b.Remote = c
	b.Account = account
	return b
}

func (b *Btelegram) Connect() error {
	var err error
	flog.Info("Connecting")
	b.c, err = tgbotapi.NewBotAPI(b.Config.Token)
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	updates, err := b.c.GetUpdatesChan(tgbotapi.NewUpdate(0))
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	go b.handleRecv(updates)
	return nil
}

func (b *Btelegram) Disconnect() error {
	return nil

}

func (b *Btelegram) JoinChannel(channel config.ChannelInfo) error {
	return nil
}

func (b *Btelegram) Send(msg config.Message) (string, error) {
	flog.Debugf("Receiving %#v", msg)
	chatid, err := strconv.ParseInt(msg.Channel, 10, 64)
	if err != nil {
		return "", err
	}

	if b.Config.MessageFormat == "HTML" {
		msg.Text = makeHTML(msg.Text)
	}

	if msg.Event == config.EVENT_MSG_DELETE {
		if msg.ID == "" {
			return "", nil
		}
		msgid, err := strconv.Atoi(msg.ID)
		if err != nil {
			return "", err
		}
		_, err = b.c.DeleteMessage(tgbotapi.DeleteMessageConfig{ChatID: chatid, MessageID: msgid})
		return "", err
	}

	// edit the message if we have a msg ID
	if msg.ID != "" {
		msgid, err := strconv.Atoi(msg.ID)
		if err != nil {
			return "", err
		}
		m := tgbotapi.NewEditMessageText(chatid, msgid, msg.Username+msg.Text)
		_, err = b.c.Send(m)
		if err != nil {
			return "", err
		}
		return "", nil
	}

	if msg.Extra != nil {
		// check if we have files to upload (from slack, telegram or mattermost)
		if len(msg.Extra["file"]) > 0 {
			var c tgbotapi.Chattable
			for _, f := range msg.Extra["file"] {
				fi := f.(config.FileInfo)
				file := tgbotapi.FileBytes{fi.Name, *fi.Data}
				re := regexp.MustCompile(".(jpg|png)$")
				if re.MatchString(fi.Name) {
					c = tgbotapi.NewPhotoUpload(chatid, file)
				} else {
					c = tgbotapi.NewDocumentUpload(chatid, file)
				}
				_, err := b.c.Send(c)
				if err != nil {
					log.Errorf("file upload failed: %#v")
				}
			}
		}
	}

	m := tgbotapi.NewMessage(chatid, msg.Username+msg.Text)
	if b.Config.MessageFormat == "HTML" {
		m.ParseMode = tgbotapi.ModeHTML
	}
	res, err := b.c.Send(m)
	if err != nil {
		return "", err
	}
	return strconv.Itoa(res.MessageID), nil

}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		flog.Debugf("Receiving from telegram: %#v", update.Message)
		var message *tgbotapi.Message
		username := ""
		channel := ""
		text := ""

		fmsg := config.Message{Extra: make(map[string][]interface{})}

		// handle channels
		if update.ChannelPost != nil {
			message = update.ChannelPost
		}
		if update.EditedChannelPost != nil && !b.Config.EditDisable {
			message = update.EditedChannelPost
			message.Text = message.Text + b.Config.EditSuffix
		}
		// handle groups
		if update.Message != nil {
			message = update.Message
		}
		if update.EditedMessage != nil && !b.Config.EditDisable {
			message = update.EditedMessage
			message.Text = message.Text + b.Config.EditSuffix
		}
		if message.From != nil {
			if b.Config.UseFirstName {
				username = message.From.FirstName
			}
			if username == "" {
				username = message.From.UserName
				if username == "" {
					username = message.From.FirstName
				}
			}
			text = message.Text
			channel = strconv.FormatInt(message.Chat.ID, 10)
		}

		if username == "" {
			username = "unknown"
		}
		if message.Sticker != nil {
			b.handleDownload(message.Sticker, &fmsg)
		}
		if message.Video != nil {
			b.handleDownload(message.Video, &fmsg)
		}
		if message.Photo != nil && b.Config.UseInsecureURL {
			b.handleDownload(message.Photo, &fmsg)
		}
		if message.Document != nil && b.Config.UseInsecureURL {
			b.handleDownload(message.Sticker, &fmsg)
			text = text + " " + message.Document.FileName + " : " + b.getFileDirectURL(message.Document.FileID)
		}

		// quote the previous message
		if message.ReplyToMessage != nil {
			usernameReply := ""
			if message.ReplyToMessage.From != nil {
				if b.Config.UseFirstName {
					usernameReply = message.ReplyToMessage.From.FirstName
				}
				if usernameReply == "" {
					usernameReply = message.ReplyToMessage.From.UserName
					if usernameReply == "" {
						usernameReply = message.ReplyToMessage.From.FirstName
					}
				}
			}
			if usernameReply == "" {
				usernameReply = "unknown"
			}
			text = text + " (re @" + usernameReply + ":" + message.ReplyToMessage.Text + ")"
		}

		if text != "" || len(fmsg.Extra) > 0 {
			flog.Debugf("Sending message from %s on %s to gateway", username, b.Account)
			msg := config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID), ID: strconv.Itoa(message.MessageID)}
			flog.Debugf("Message is %#v", msg)
			b.Remote <- msg
		}
	}
}

func (b *Btelegram) getFileDirectURL(id string) string {
	res, err := b.c.GetFileDirectURL(id)
	if err != nil {
		return ""
	}
	return res
}

func (b *Btelegram) handleDownload(file interface{}, msg *config.Message) {
	size := 0
	url := ""
	name := ""
	text := ""
	switch v := file.(type) {
	case *tgbotapi.Sticker:
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		name = "sticker"
		text = " " + url
	case *tgbotapi.Video:
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		name = "video"
		text = " " + url
	case *[]tgbotapi.PhotoSize:
		photos := *v
		size = photos[len(photos)-1].FileSize
		url = b.getFileDirectURL(photos[len(photos)-1].FileID)
		name = "photo"
		text = " " + url
	case *tgbotapi.Document:
		size = v.FileSize
		url = b.getFileDirectURL(v.FileID)
		name = v.FileName
		text = " " + v.FileName + " : " + url
	}
	if b.Config.UseInsecureURL {
		msg.Text = text
		return
	}
	// if we have a file attached, download it (in memory) and put a pointer to it in msg.Extra
	// limit to 1MB for now
	if size <= 1000000 {
		data, err := helper.DownloadFile(url)
		if err != nil {
			flog.Errorf("download %s failed %#v", url, err)
		} else {
			msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{Name: name, Data: data})
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package biome

var byId [256]*Type

// Biomes
var (
	// Snowy
	FrozenOcean        = newBiome(10, 0.0, 0.5)
	FrozenRiver        = newBiome(11, 0.0, 0.5)
	IcePlains          = newBiome(12, 0.0, 0.5)
	IcePlainsSpikes    = newBiome(140, 0.0, 0.5)
	ColdBeach          = newBiome(26, 0.05, 0.3)
	ColdTaiga          = newBiome(30, 0.0, 0.4)
	ColdTaigaMountains = newBiome(158, 0.0, 0.4)
	// Cold
	ExtremeHills              = newBiome(3, 0.2, 0.3)
	ExtremeHillsMountains     = newBiome(131, 0.2, 0.3)
	Taiga                     = newBiome(5, 0.25, 0.8)
	TaigaM                    = newBiome(133, 0.25, 0.8)
	TheEnd                    = newBiome(9, 0.5, 0.5)
	MegaTaiga                 = newBiome(32, 0.3, 0.8)
	MegaSpruceTaiga           = newBiome(160, 0.5, 0.5)
	ExtremeHillsPlus          = newBiome(34, 0.2, 0.3)
	ExtremeHillsPlusMountains = newBiome(162, 0.2, 0.3)
	StoneBeach                = newBiome(25, 0.2, 0.3)
	// Medium/Lush
	Plains                = newBiome(1, 0.5, 0.5)
	SunflowerPlains       = newBiome(129, 0.5, 0.5)
	Forest                = newBiome(4, 0.5, 0.5)
	FlowerForest          = newBiome(132, 0.5, 0.5)
	Swampland             = newBiome(6, 0.8, 0.9)
	SwamplandMountains    = newBiome(134, 0.8, 0.9)
	River                 = newBiome(7, 0.5, 0.5)
	MushroomIsland        = newBiome(14, 0.9, 1.0)
	MushroomISlandShore   = newBiome(15, 0.9, 1.0)
	Beach                 = newBiome(16, 0.8, 0.4)
	Jungle                = newBiome(21, 0.95, 0.8)
	JungleMountains       = newBiome(149, 0.95, 0.9)
	JungleEdge            = newBiome(23, 0.95, 0.8)
	JungleEdgeMountains   = newBiome(151, 0.95, 0.8)
	BirchForest           = newBiome(27, 0.5, 0.5)
	BirchForestMountains  = newBiome(155, 0.5, 0.5)
	RoofedForest          = newBiome(29, 0.5, 0.5)
	RoofedForestMountains = newBiome(157, 0.5, 0.5)
	// Dry/Warm
	Desert                     = newBiome(2, 1.0, 0.0)
	DesertMountain             = newBiome(130, 1.0, 0.0)
	Hell                       = newBiome(8, 1.0, 0.0)
	Savanna                    = newBiome(35, 1.0, 0.0)
	SavannaMountains           = newBiome(163, 1.0, 0.0)
	Mesa                       = newBiome(37, 0.5, 0.5)
	MesaBryce                  = newBiome(165, 0.5, 0.5)
	SavannaPlateau             = newBiome(36, 1.0, 0.0)
	MesaPlateauForest          = newBiome(38, 0.5, 0.5)
	MesaPlateau                = newBiome(39, 0.5, 0.5)
	SavannaPlateauMountains    = newBiome(164, 1.0, 0.0)
	MesaPlateauForestMountains = newBiome(166, 0.5, 0.5)
	MesaPlateauMountains       = newBiome(167, 0.5, 0.5)
	// Neutral
	Ocean                     = newBiome(0, 0.5, 0.5)
	DeepOcean                 = newBiome(24, 0.5, 0.5)
	IceMountains              = newBiome(13, 0.0, 0.5)
	DesertHills               = newBiome(17, 1.0, 0.0)
	ForestHills               = newBiome(18, 0.45, 0.3)
	TaigaHills                = newBiome(19, 0.25, 0.8)
	JungleHills               = newBiome(22, 0.95, 0.9)
	BirchForestHills          = newBiome(28, 0.5, 0.5)
	ColdTaigaHills            = newBiome(31, 0.5, 0.5)
	MegaTaigaHills            = newBiome(33, 0.3, 0.8)
	BirchForestHillsMountains = newBiome(156, 0.5, 0.5)
	MegaSpruceTaigaHills      = newBiome(161, 0.5, 0.5)
	// Custom
	Invalid = newBiome(255, 0.0, 0.0)
)

func ById(id byte) *Type {
	return byId[id]
}

type Type struct {
	ID                    int
	Temperature, Moisture float64
	ColorIndex            int
}

func newBiome(id int, temperature, moisture float64) *Type {
	b := &Type{
		ID:          id,
		Temperature: temperature,
		Moisture:    moisture * temperature,
	}
	bx := int((1.0 - temperature) * 255.0)
	by := int((1.0 - moisture) * 255.0)
	b.ColorIndex = bx | (by << 8)
	byId[id] = b
	return b
}

func init() {
	for i := range byId {
		if byId[i] == nil {
			byId[i] = Invalid
		}
	}
}
<commit_msg>world/biome: fix all biomes' values and fix the color index computing<commit_after>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package biome

var byId [256]*Type

// Biomes
var (
	Ocean               = newBiome(0, 0.5, 0.5)
	Plains              = newBiome(1, 0.8, 0.4)
	Desert              = newBiome(2, 2.0, 0.0)
	ExtremeHills        = newBiome(3, 0.2, 0.3)
	Forest              = newBiome(4, 0.7, 0.8)
	Taiga               = newBiome(5, 0.05, 0.8)
	Swampland           = newBiome(6, 0.8, 0.9)
	River               = newBiome(7, 0.5, 0.5)
	Hell                = newBiome(8, 2.0, 0.0)
	TheEnd              = newBiome(9, 0.5, 0.5)
	FrozenOcean         = newBiome(10, 0.0, 0.5)
	FrozenRiver         = newBiome(11, 0.0, 0.5)
	IcePlains           = newBiome(12, 0.0, 0.5)
	IceMountains        = newBiome(13, 0.0, 0.5)
	MushroomIsland      = newBiome(14, 0.9, 1.0)
	MushroomISlandShore = newBiome(15, 0.9, 1.0)
	Beach               = newBiome(16, 0.8, 0.4)
	DesertHills         = newBiome(17, 2.0, 0.0)
	ForestHills         = newBiome(18, 0.7, 0.8)
	TaigaHills          = newBiome(19, 0.2, 0.7)
	ExtremeHillsEdge    = newBiome(20, 0.2, 0.3)
	Jungle              = newBiome(21, 1.2, 0.9)
	JungleHills         = newBiome(22, 1.2, 0.9)
	JungleEdge          = newBiome(23, 0.95, 0.8)
	DeepOcean           = newBiome(24, 0.5, 0.5)
	StoneBeach          = newBiome(25, 0.2, 0.3)
	ColdBeach           = newBiome(26, 0.05, 0.3)
	BirchForest         = newBiome(27, 0.6, 0.6)
	BirchForestHills    = newBiome(28, 0.6, 0.6)
	RoofedForest        = newBiome(29, 0.7, 0.8)
	ColdTaiga           = newBiome(30, -0.5, 0.4)
	ColdTaigaHills      = newBiome(31, -0.5, 0.4)
	MegaTaiga           = newBiome(32, 0.3, 0.8)
	MegaTaigaHills      = newBiome(33, 0.3, 0.8)
	ExtremeHillsPlus    = newBiome(34, 0.2, 0.3)
	Savanna             = newBiome(35, 1.2, 0.0)
	SavannaPlateau      = newBiome(36, 1.0, 0.0)
	Mesa                = newBiome(37, 2.0, 0.0)
	MesaPlateauForest   = newBiome(38, 2.0, 0.0)
	MesaPlateau         = newBiome(39, 2.0, 0.0)

	SunflowerPlains            = newBiome(129, 0.8, 0.4)
	DesertMountain             = newBiome(130, 2.0, 0.0)
	ExtremeHillsMountains      = newBiome(131, 0.2, 0.3)
	FlowerForest               = newBiome(132, 0.7, 0.8)
	TaigaM                     = newBiome(133, 0.05, 0.8)
	SwamplandMountains         = newBiome(134, 0.8, 0.9)
	IcePlainsSpikes            = newBiome(140, 0.0, 0.5)
	JungleMountains            = newBiome(149, 1.2, 0.9)
	JungleEdgeMountains        = newBiome(151, 0.95, 0.8)
	BirchForestMountains       = newBiome(155, 0.6, 0.6)
	BirchForestHillsMountains  = newBiome(156, 0.6, 0.6)
	RoofedForestMountains      = newBiome(157, 0.7, 0.8)
	ColdTaigaMountains         = newBiome(158, -0.5, 0.4)
	MegaSpruceTaiga            = newBiome(160, 0.25, 0.8)
	MegaSpruceTaigaHills       = newBiome(161, 0.3, 0.8)
	ExtremeHillsPlusMountains  = newBiome(162, 0.2, 0.3)
	SavannaMountains           = newBiome(163, 1.2, 0.0)
	SavannaPlateauMountains    = newBiome(164, 1.0, 0.0)
	MesaBryce                  = newBiome(165, 2.0, 0.0)
	MesaPlateauForestMountains = newBiome(166, 2.0, 0.0)
	MesaPlateauMountains       = newBiome(167, 2.0, 0.0)
	//
	Invalid = newBiome(255, 0.0, 0.0)
)

func ById(id byte) *Type {
	if val := byId[id]; val != nil {
		return val
	}
	return Invalid
}

type Type struct {
	ID                    int
	Temperature, Moisture float64
	ColorIndex            int
}

func newBiome(id int, temperature, moisture float64) *Type {
	b := &Type{
		ID:          id,
		Temperature: clamp(temperature, 0, 1),
		Moisture:    clamp(moisture, 0, 1),
	}
	b.Moisture *= b.Temperature
	bx := int((1.0 - b.Temperature) * 255.0)
	by := int((1.0 - b.Moisture) * 255.0)
	b.ColorIndex = bx | (by << 8)
	byId[id] = b
	return b
}

func clamp(x, l, h float64) float64 {
	if x < l {
		return l
	}
	if x > h {
		return h
	}
	return x
}

func init() {
	for i := range byId {
		if byId[i] == nil {
			byId[i] = Invalid
		}
	}
}
<|endoftext|>


<commit_before>package core

// transfer2go data core module, request implementation
// Author: Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"bytes"
	"container/heap"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/utils"
)

// AgentStatus data type
type AgentStatus struct {
	Url       string            `json:"url"`      // agent url
	Name      string            `json:"name"`     // agent name or alias
	TimeStamp int64             `json:"ts"`       // time stamp
	Catalog   string            `json:"catalog"`  // underlying TFC catalog
	Protocol  string            `json:"protocol"` // underlying transfer protocol
	Backend   string            `json:"backend"`  // underlying transfer backend
	Tool      string            `json:"tool"`     // underlying transfer tool, e.g. xrdcp
	ToolOpts  string            `json:"toolopts"` // options for backend tool
	Agents    map[string]string `json:"agents"`   // list of known agents
	Addrs     []string          `json:"addrs"`    // list of all IP addresses
	Metrics   map[string]int64  `json:"metrics"`  // agent metrics
}

// Processor is an object who process' given task
// The logic of the Processor should be implemented.
type Processor struct {
}

// Request interface defines a task process
type Request interface {
	Process(*TransferRequest) error
}

// RequestFunc is a function type that implements the Request interface
type RequestFunc func(*TransferRequest) error

// Decorator wraps a request with extra behavior
type Decorator func(Request) Request

// DefaultProcessor is a default processor instance
var DefaultProcessor = &Processor{}

// String provides string representation of given agent status
func (a *AgentStatus) String() string {
	return fmt.Sprintf("<Agent name=%s url=%s catalog=%s protocol=%s backend=%s tool=%s toolOpts=%s agents=%v addrs=%v metrics(%v)>", a.Name, a.Url, a.Catalog, a.Protocol, a.Backend, a.Tool, a.ToolOpts, a.Agents, a.Addrs, a.Metrics)
}

// Process defines execution process for a given task
func (e *Processor) Process(t *TransferRequest) error {
	return nil
}

// Process is a method of TransferRequest
func (f RequestFunc) Process(t *TransferRequest) error {
	return f(t)
}

// filleTransferRequest creates HTTP request to transfer a given file name
// https://matt.aimonetti.net/posts/2013/07/01/golang-multipart-file-upload-example/
func fileTransferRequest(c CatalogEntry, tr *TransferRequest) (*http.Request, error) {
	file, err := os.Open(c.Pfn)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile("data", filepath.Base(c.Pfn))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	err = writer.Close()
	if err != nil {
		return nil, err
	}

	url := fmt.Sprintf("%s/upload", tr.DstUrl)
	req, err := http.NewRequest("POST", url, body)
	req.Header.Set("Content-Type", writer.FormDataContentType())
	req.Header.Set("Pfn", c.Pfn)
	req.Header.Set("Lfn", c.Lfn)
	req.Header.Set("Bytes", fmt.Sprintf("%d", c.Bytes))
	req.Header.Set("Hash", c.Hash)
	req.Header.Set("Src", tr.SrcAlias)
	req.Header.Set("Dst", tr.DstAlias)
	return req, err
}

// helper function to perform transfer via HTTP protocol
func httpTransfer(c CatalogEntry, t *TransferRequest) (string, error) {
	// create file transfer request
	request, err := fileTransferRequest(c, t)
	if err != nil {
		return "", err
	}
	client := utils.HttpClient()
	resp, err := client.Do(request)
	defer resp.Body.Close()

	var r CatalogEntry
	err = json.NewDecoder(resp.Body).Decode(&r)
	if err != nil {
		return "", err
	}
	return r.Pfn, nil
}

// Store returns a Decorator that stores request
func Store() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			t.Id = time.Now().Unix()
			item := &Item{
				Value:    *t,
				priority: t.Priority,
			}
			fmt.Println(*t)
			err := TFC.InsertRequest(*t)
			if err != nil {
				return err
			} else {
				heap.Push(&RequestQueue, item)
			}
			log.WithFields(log.Fields{
				"Request": t,
			}).Println("Request Saved")
			return r.Process(t)
		})
	}
}

// Delete returns a Decorator that deletes request from heap
func Delete() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			// Delete request from PriorityQueue. The complexity is O(n) where n = heap.Len()
			index := -1
			var err error

			for _, item := range RequestQueue {
				if item.Value.Id == t.Id {
					index = item.index
					break
				}
			}

			if index < RequestQueue.Len() && index >= 0 {
				err = TFC.UpdateRequest(t.Id, "deleted")
				if err != nil {
					t.Status = "error"
					return err
				} else {
					// TODO: May be we need to add lock over here.
					heap.Remove(&RequestQueue, index)
					t.Status = "deleted"
					log.WithFields(log.Fields{
						"Request": t,
					}).Println("Request Deleted")
				}
			} else {
				t.Status = "error"
				err = errors.New("Can't find request in heap")
				return err
			}

			return r.Process(t)
		})
	}
}

// Transfer returns a Decorator that performs request transfers by pull model
func PullTransfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"Request": t.String(),
			}).Println("Request Transfer")
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}
			// if both are up then send acknowledge message to destination on /pullack url.
			body, err := json.Marshal(t)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/pull", t.DstUrl)
			resp = utils.FetchResponse(url, body)
			// check return status code
			if resp.StatusCode != 200 {
				return fmt.Errorf("Response %s, error=%s", resp.Status, string(resp.Data))
			}
			return r.Process(t)
		})
	}
}

// Transfer returns a Decorator that performs request transfers
func PushTransfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"Request": t.String(),
			}).Println("Request Transfer")
			var records []CatalogEntry
			// Consider those requests which are failed in previous iteration.
			// If it is nil then request must be passing through first iteration.
			if t.FailedRecords != nil {
				records = t.FailedRecords
			} else {
				records = TFC.Records(*t)
			}
			if len(records) == 0 {
				// file does not exists in TFC, nothing to do, return immediately
				log.WithFields(log.Fields{
					"TransferRequest": t,
				}).Warn("Does not match anything in TFC of this agent\n", t)
				return r.Process(t)
			}
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}

			// TODO: I need to implement bulk transfer for all files in found records
			// so far I loop over them individually and transfer one by one
			var trRecords []CatalogEntry // list of successfully transferred records
			var failedRecords []CatalogEntry
			// Overwrite the previous error status
			t.Status = ""
			for _, rec := range records {

				time0 := time.Now().Unix()

				AgentMetrics.Bytes.Inc(rec.Bytes)

				// if protocol is not given use default one: HTTP
				var rpfn string // remote PFN
				if srcAgent.Protocol == "" || srcAgent.Protocol == "http" {
					log.WithFields(log.Fields{
						"dstAgent": dstAgent.String(),
					}).Println("Transfer via HTTP protocol to", dstAgent.String())
					rpfn, err = httpTransfer(rec, t)
					if err != nil {
						log.WithFields(log.Fields{
							"TransferRequest": t.String(),
							"Record":          rec.String(),
							"Err":             err,
						}).Error("Transfer", rec.String(), t.String(), err)
						t.Status = err.Error()
						failedRecords = append(failedRecords, rec)
						continue // if we fail on single record we continue with others
					}
				} else {
					// construct remote PFN by using destination agent backend and record LFN
					rpfn = fmt.Sprintf("%s%s", dstAgent.Backend, rec.Lfn)
					// perform transfer with the help of backend tool
					var cmd *exec.Cmd
					if srcAgent.ToolOpts == "" {
						cmd = exec.Command(srcAgent.Tool, rec.Pfn, rpfn)
					} else {
						cmd = exec.Command(srcAgent.Tool, srcAgent.ToolOpts, rec.Pfn, rpfn)
					}
					log.WithFields(log.Fields{
						"Command": cmd,
					}).Println("Transfer command")
					err = cmd.Run()
					if err != nil {
						log.WithFields(log.Fields{
							"Tool":         srcAgent.Tool,
							"Tool options": srcAgent.ToolOpts,
							"PFN":          rec.Pfn,
							"Remote PFN":   rpfn,
							"Err":          err,
						}).Error("Transfer")
						t.Status = err.Error()
						failedRecords = append(failedRecords, rec)
						continue // if we fail on single record we continue with others
					}
				}
				r := CatalogEntry{Dataset: rec.Dataset, Block: rec.Block, Lfn: rec.Lfn, Pfn: rpfn, Bytes: rec.Bytes, Hash: rec.Hash, TransferTime: (time.Now().Unix() - time0), Timestamp: time.Now().Unix()}
				trRecords = append(trRecords, r)

				// record how much we transferred
				AgentMetrics.TotalBytes.Inc(r.Bytes) // keep growing
				AgentMetrics.Total.Inc(1)            // keep growing
				AgentMetrics.Bytes.Dec(rec.Bytes)    // decrement since we're done

			}
			// Add entry for remote TFC after transfer is completed
			url = fmt.Sprintf("%s/tfc", t.DstUrl)
			d, e := json.Marshal(trRecords)
			if e != nil {
				return e
			}
			resp = utils.FetchResponse(url, d) // POST request
			if resp.Error != nil {
				return resp.Error
			}
			t.FailedRecords = failedRecords
			return r.Process(t)
		})
	}
}

// Logging returns a Decorator that logs client requests
func Logging(l *log.Logger) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			l.Println("TransferRequest", t)
			return r.Process(t)
		})
	}
}

// Pause returns a Decorator that pauses request for a given time interval
func Pause(interval time.Duration) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			if interval > 0 {
				log.WithFields(log.Fields{
					"Request":  t,
					"Interval": interval,
				}).Println("TransferRequest is paused by")
				time.Sleep(interval)
			}
			return r.Process(t)
		})
	}
}

// Tracer returns a Decorator that traces given request
func Tracer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"TransferRequest": t,
			}).Println("Trace")
			return r.Process(t)
		})
	}
}

// Decorate decorates a Request r with all given Decorators
func Decorate(r Request, ds ...Decorator) Request {
	decorated := r
	for _, decorate := range ds {
		decorated = decorate(decorated)
	}
	return decorated
}
<commit_msg>Do chunck wise transfer<commit_after>package core

// transfer2go data core module, request implementation
// Author: Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"container/heap"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/utils"
)

// AgentStatus data type
type AgentStatus struct {
	Url       string            `json:"url"`      // agent url
	Name      string            `json:"name"`     // agent name or alias
	TimeStamp int64             `json:"ts"`       // time stamp
	Catalog   string            `json:"catalog"`  // underlying TFC catalog
	Protocol  string            `json:"protocol"` // underlying transfer protocol
	Backend   string            `json:"backend"`  // underlying transfer backend
	Tool      string            `json:"tool"`     // underlying transfer tool, e.g. xrdcp
	ToolOpts  string            `json:"toolopts"` // options for backend tool
	Agents    map[string]string `json:"agents"`   // list of known agents
	Addrs     []string          `json:"addrs"`    // list of all IP addresses
	Metrics   map[string]int64  `json:"metrics"`  // agent metrics
}

// Processor is an object who process' given task
// The logic of the Processor should be implemented.
type Processor struct {
}

// Request interface defines a task process
type Request interface {
	Process(*TransferRequest) error
}

// RequestFunc is a function type that implements the Request interface
type RequestFunc func(*TransferRequest) error

// Decorator wraps a request with extra behavior
type Decorator func(Request) Request

// DefaultProcessor is a default processor instance
var DefaultProcessor = &Processor{}

// String provides string representation of given agent status
func (a *AgentStatus) String() string {
	return fmt.Sprintf("<Agent name=%s url=%s catalog=%s protocol=%s backend=%s tool=%s toolOpts=%s agents=%v addrs=%v metrics(%v)>", a.Name, a.Url, a.Catalog, a.Protocol, a.Backend, a.Tool, a.ToolOpts, a.Agents, a.Addrs, a.Metrics)
}

// Process defines execution process for a given task
func (e *Processor) Process(t *TransferRequest) error {
	return nil
}

// Process is a method of TransferRequest
func (f RequestFunc) Process(t *TransferRequest) error {
	return f(t)
}

// filleTransferRequest creates HTTP request to transfer a given file name
// https://matt.aimonetti.net/posts/2013/07/01/golang-multipart-file-upload-example/
func fileTransferRequest(c CatalogEntry, tr *TransferRequest) (*http.Response, error) {
	file, err := os.Open(c.Pfn)
	defer file.Close()
	if err != nil {
		return nil, err
	}

	pipeOut, pipeIn := io.Pipe()
	writer := multipart.NewWriter(pipeIn)
	// do the request concurrently
	var resp *http.Response
	done := make(chan error)
	go func() {
		// prepare request
		url := fmt.Sprintf("%s/upload", tr.DstUrl)
		req, err := http.NewRequest("POST", url, pipeOut)
		if err != nil {
			done <- err
			return
		}

		req.Header.Set("Content-Type", writer.FormDataContentType())
		req.Header.Set("Pfn", c.Pfn)
		req.Header.Set("Lfn", c.Lfn)
		req.Header.Set("Bytes", fmt.Sprintf("%d", c.Bytes))
		req.Header.Set("Hash", c.Hash)
		req.Header.Set("Src", tr.SrcAlias)
		req.Header.Set("Dst", tr.DstAlias)
		req.Header.Set("Content-Type", writer.FormDataContentType())
		client := utils.HttpClient()
		resp, err = client.Do(req)
		if err != nil {
			done <- err
			return
		}

		done <- nil
	}()

	part, err := writer.CreateFormFile("data", filepath.Base(c.Pfn))
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(part, file)
	if err != nil {
		return nil, err
	}

	err = writer.Close()
	if err != nil {
		return nil, err
	}

	err = pipeIn.Close()
	if err != nil {
		return nil, err
	}

	err = <-done
	if err != nil {
		return nil, err
	}

	return resp, nil
}

// helper function to perform transfer via HTTP protocol
func httpTransfer(c CatalogEntry, t *TransferRequest) (string, error) {
	// create file transfer request
	resp, err := fileTransferRequest(c, t)
	if err != nil {
		return "", err
	}

	var r CatalogEntry
	err = json.NewDecoder(resp.Body).Decode(&r)
	if err != nil {
		return "", err
	}
	return r.Pfn, nil
}

// Store returns a Decorator that stores request
func Store() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			t.Id = time.Now().Unix()
			item := &Item{
				Value:    *t,
				priority: t.Priority,
			}
			fmt.Println(*t)
			err := TFC.InsertRequest(*t)
			if err != nil {
				return err
			} else {
				heap.Push(&RequestQueue, item)
			}
			log.WithFields(log.Fields{
				"Request": t,
			}).Println("Request Saved")
			return r.Process(t)
		})
	}
}

// Delete returns a Decorator that deletes request from heap
func Delete() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			// Delete request from PriorityQueue. The complexity is O(n) where n = heap.Len()
			index := -1
			var err error

			for _, item := range RequestQueue {
				if item.Value.Id == t.Id {
					index = item.index
					break
				}
			}

			if index < RequestQueue.Len() && index >= 0 {
				err = TFC.UpdateRequest(t.Id, "deleted")
				if err != nil {
					t.Status = "error"
					return err
				} else {
					// TODO: May be we need to add lock over here.
					heap.Remove(&RequestQueue, index)
					t.Status = "deleted"
					log.WithFields(log.Fields{
						"Request": t,
					}).Println("Request Deleted")
				}
			} else {
				t.Status = "error"
				err = errors.New("Can't find request in heap")
				return err
			}

			return r.Process(t)
		})
	}
}

// Transfer returns a Decorator that performs request transfers by pull model
func PullTransfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"Request": t.String(),
			}).Println("Request Transfer")
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}
			// if both are up then send acknowledge message to destination on /pullack url.
			body, err := json.Marshal(t)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/pull", t.DstUrl)
			resp = utils.FetchResponse(url, body)
			// check return status code
			if resp.StatusCode != 200 {
				return fmt.Errorf("Response %s, error=%s", resp.Status, string(resp.Data))
			}
			return r.Process(t)
		})
	}
}

// Transfer returns a Decorator that performs request transfers
func PushTransfer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"Request": t.String(),
			}).Println("Request Transfer")
			var records []CatalogEntry
			// Consider those requests which are failed in previous iteration.
			// If it is nil then request must be passing through first iteration.
			if t.FailedRecords != nil {
				records = t.FailedRecords
			} else {
				records = TFC.Records(*t)
			}
			if len(records) == 0 {
				// file does not exists in TFC, nothing to do, return immediately
				log.WithFields(log.Fields{
					"TransferRequest": t,
				}).Warn("Does not match anything in TFC of this agent\n", t)
				return r.Process(t)
			}
			// obtain information about source and destination agents
			url := fmt.Sprintf("%s/status", t.DstUrl)
			resp := utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var dstAgent AgentStatus
			err := json.Unmarshal(resp.Data, &dstAgent)
			if err != nil {
				return err
			}
			url = fmt.Sprintf("%s/status", t.SrcUrl)
			resp = utils.FetchResponse(url, []byte{})
			if resp.Error != nil {
				return resp.Error
			}
			var srcAgent AgentStatus
			err = json.Unmarshal(resp.Data, &srcAgent)
			if err != nil {
				return err
			}

			// TODO: I need to implement bulk transfer for all files in found records
			// so far I loop over them individually and transfer one by one
			var trRecords []CatalogEntry // list of successfully transferred records
			var failedRecords []CatalogEntry
			// Overwrite the previous error status
			t.Status = ""
			for _, rec := range records {

				time0 := time.Now().Unix()

				AgentMetrics.Bytes.Inc(rec.Bytes)

				// if protocol is not given use default one: HTTP
				var rpfn string // remote PFN
				if srcAgent.Protocol == "" || srcAgent.Protocol == "http" {
					log.WithFields(log.Fields{
						"dstAgent": dstAgent.String(),
					}).Println("Transfer via HTTP protocol to", dstAgent.String())
					rpfn, err = httpTransfer(rec, t)
					if err != nil {
						log.WithFields(log.Fields{
							"TransferRequest": t.String(),
							"Record":          rec.String(),
							"Err":             err,
						}).Error("Transfer", rec.String(), t.String(), err)
						t.Status = err.Error()
						failedRecords = append(failedRecords, rec)
						continue // if we fail on single record we continue with others
					}
				} else {
					// construct remote PFN by using destination agent backend and record LFN
					rpfn = fmt.Sprintf("%s%s", dstAgent.Backend, rec.Lfn)
					// perform transfer with the help of backend tool
					var cmd *exec.Cmd
					if srcAgent.ToolOpts == "" {
						cmd = exec.Command(srcAgent.Tool, rec.Pfn, rpfn)
					} else {
						cmd = exec.Command(srcAgent.Tool, srcAgent.ToolOpts, rec.Pfn, rpfn)
					}
					log.WithFields(log.Fields{
						"Command": cmd,
					}).Println("Transfer command")
					err = cmd.Run()
					if err != nil {
						log.WithFields(log.Fields{
							"Tool":         srcAgent.Tool,
							"Tool options": srcAgent.ToolOpts,
							"PFN":          rec.Pfn,
							"Remote PFN":   rpfn,
							"Err":          err,
						}).Error("Transfer")
						t.Status = err.Error()
						failedRecords = append(failedRecords, rec)
						continue // if we fail on single record we continue with others
					}
				}
				r := CatalogEntry{Dataset: rec.Dataset, Block: rec.Block, Lfn: rec.Lfn, Pfn: rpfn, Bytes: rec.Bytes, Hash: rec.Hash, TransferTime: (time.Now().Unix() - time0), Timestamp: time.Now().Unix()}
				trRecords = append(trRecords, r)

				// record how much we transferred
				AgentMetrics.TotalBytes.Inc(r.Bytes) // keep growing
				AgentMetrics.Total.Inc(1)            // keep growing
				AgentMetrics.Bytes.Dec(rec.Bytes)    // decrement since we're done

			}
			// Add entry for remote TFC after transfer is completed
			url = fmt.Sprintf("%s/tfc", t.DstUrl)
			d, e := json.Marshal(trRecords)
			if e != nil {
				return e
			}
			resp = utils.FetchResponse(url, d) // POST request
			if resp.Error != nil {
				return resp.Error
			}
			t.FailedRecords = failedRecords
			return r.Process(t)
		})
	}
}

// Logging returns a Decorator that logs client requests
func Logging(l *log.Logger) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			l.Println("TransferRequest", t)
			return r.Process(t)
		})
	}
}

// Pause returns a Decorator that pauses request for a given time interval
func Pause(interval time.Duration) Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			if interval > 0 {
				log.WithFields(log.Fields{
					"Request":  t,
					"Interval": interval,
				}).Println("TransferRequest is paused by")
				time.Sleep(interval)
			}
			return r.Process(t)
		})
	}
}

// Tracer returns a Decorator that traces given request
func Tracer() Decorator {
	return func(r Request) Request {
		return RequestFunc(func(t *TransferRequest) error {
			log.WithFields(log.Fields{
				"TransferRequest": t,
			}).Println("Trace")
			return r.Process(t)
		})
	}
}

// Decorate decorates a Request r with all given Decorators
func Decorate(r Request, ds ...Decorator) Request {
	decorated := r
	for _, decorate := range ds {
		decorated = decorate(decorated)
	}
	return decorated
}
<|endoftext|>