<commit_before>package coolmaze

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/cloud/storage"
)

// This is for sending a file resource.
// The resource transits on Google Cloud Storage.
// This endpoint produces 2 short-lived signed urls:
// - 1 for uploading the resource (from mobile to GCS)
// - 1 for downloading the resource (from GCS to desktop browser)

func init() {
	http.HandleFunc("/new-gcs-urls", gcsUrlsHandler)

	// This is important for randomString below
	rndOnce.Do(randomize)

	var err error
	pkey, err = ioutil.ReadFile(pemFile)
	if err != nil {
		// ..but i don't have a Context to yell at...
		// log.Errorf(c, "%v", err)
	}
}

const (
	// This GCS bucket is used for temporary storage between
	// source mobile and target desktop.
	bucket         = "cool-maze-transit"
	serviceAccount = "mobile-to-gcs@cool-maze.iam.gserviceaccount.com"
	// This (secret) file was generated by command
	//    openssl pkcs12 -in Cool-Maze-2e343b6677b7.p12 -passin pass:notasecret -out Mobile-to-GCS.pem -nodes
	pemFile = "Mobile-to-GCS.pem"
	// Cooperative limit (not attack-proof)
	MB            = 1024 * 1024
	uploadMaxSize = 21*MB - 1
)

var pkey []byte

func gcsUrlsHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if r.Method != "POST" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintln(w, "Only POST method is accepted")
		return
	}
	// Warning: this contentType will be part of the crypted
	// signature, and the client will have to match it exactly
	contentType := r.FormValue("type")

	// Check intended filesize
	fileSizeStr := r.FormValue("filesize")
	if fileSizeStr == "" {
		log.Warningf(c, "Missing mandatory parameter: filesize")
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintln(w, "Mandatory parameter: filesize")
		return
	}
	fileSize, err := strconv.Atoi(fileSizeStr)
	if err != nil {
		log.Warningf(c, "Invalid filesize value %q", fileSizeStr)
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintln(w, "Invalid parameter: filesize")
		return
	}
	if fileSize > uploadMaxSize {
		log.Warningf(c, "Can't give upload URL for resource of size %d. Max allowed size is %d.", fileSize, uploadMaxSize)
		w.WriteHeader(http.StatusPreconditionFailed)
		response := Response{
			"success": false,
			"message": fmt.Sprintf("Can't upload resource of size %dMB. Max allowed size is %dMB.", fileSize/MB, uploadMaxSize/MB),
		}
		fmt.Fprintln(w, response)
		return
	}

	urlPut, urlGet, err := createUrls(c, contentType, fileSize)
	if err != nil {
		log.Errorf(c, "%v", err)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, `{"success": false}`)
		return
	}
	response := Response{
		"success": true,
		"urlPut":  urlPut,
		"urlGet":  urlGet,
	}
	fmt.Fprintln(w, response)
}

func createUrls(c context.Context, contentType string, fileSize int) (urlPut, urlGet string, err error) {
	objectName := randomGcsObjectName()
	log.Infof(c, "Creating urls for tmp object name %s with content-type [%s] having size %d", objectName, contentType, fileSize)

	urlPut, err = storage.SignedURL(bucket, objectName, &storage.SignedURLOptions{
		GoogleAccessID: serviceAccount,
		PrivateKey:     pkey,
		Method:         "PUT",
		Expires:        time.Now().Add(9 * time.Minute),
		ContentType:    contentType,
	})
	if err != nil {
		return
	}

	urlGet, err = storage.SignedURL(bucket, objectName, &storage.SignedURLOptions{
		GoogleAccessID: serviceAccount,
		PrivateKey:     pkey,
		Method:         "GET",
		Expires:        time.Now().Add(10 * time.Minute),
	})

	return
}
<commit_msg>issues/32 Hash-based file cache: backend part (untested).<commit_after>package coolmaze

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
	"google.golang.org/cloud/storage"
)

// This is for sending a file resource.
// The resource transits on Google Cloud Storage.
// This endpoint produces 2 short-lived signed urls:
// - 1 for uploading the resource (from mobile to GCS)
// - 1 for downloading the resource (from GCS to desktop browser)

func init() {
	http.HandleFunc("/new-gcs-urls", gcsUrlsHandler)

	// This is important for randomString below
	rndOnce.Do(randomize)

	var err error
	pkey, err = ioutil.ReadFile(pemFile)
	if err != nil {
		// ..but i don't have a Context to yell at...
		// log.Errorf(c, "%v", err)
	}
}

const (
	// This GCS bucket is used for temporary storage between
	// source mobile and target desktop.
	bucket         = "cool-maze-transit"
	serviceAccount = "mobile-to-gcs@cool-maze.iam.gserviceaccount.com"
	// This (secret) file was generated by command
	//    openssl pkcs12 -in Cool-Maze-2e343b6677b7.p12 -passin pass:notasecret -out Mobile-to-GCS.pem -nodes
	pemFile = "Mobile-to-GCS.pem"
	// Cooperative limit (not attack-proof)
	MB            = 1024 * 1024
	uploadMaxSize = 21*MB - 1
)

var pkey []byte

// The /new-gcs-urls user usually advertises the following info about
// intended file to be uploaded:
// - Content type  (it becomes a part of the signed URL)
// - Size  (since #101)
// - Hash, optional (since #32)
// - TODO : Filename (#63)
func gcsUrlsHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if r.Method != "POST" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintln(w, "Only POST method is accepted")
		return
	}
	// Warning: this contentType will be part of the crypted
	// signature, and the client will have to match it exactly
	contentType := r.FormValue("type")

	// Check intended filesize
	fileSizeStr := r.FormValue("filesize")
	if fileSizeStr == "" {
		log.Warningf(c, "Missing mandatory parameter: filesize")
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintln(w, "Mandatory parameter: filesize")
		return
	}
	fileSize, err := strconv.Atoi(fileSizeStr)
	if err != nil {
		log.Warningf(c, "Invalid filesize value %q", fileSizeStr)
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintln(w, "Invalid parameter: filesize")
		return
	}
	if fileSize > uploadMaxSize {
		log.Warningf(c, "Can't give upload URL for resource of size %d. Max allowed size is %d.", fileSize, uploadMaxSize)
		w.WriteHeader(http.StatusPreconditionFailed)
		response := Response{
			"success": false,
			"message": fmt.Sprintf("Can't upload resource of size %dMB. Max allowed size is %dMB.", fileSize/MB, uploadMaxSize/MB),
		}
		fmt.Fprintln(w, response)
		return
	}

	// Mobile source computed hash of resource before uploading it.
	// Optional.
	// See #32
	hash := r.FormValue("hash")
	if found, urlGetExisting := findByHash(c, hash); found {
		response := Response{
			"success":  true,
			"existing": true,
			"urlGet":   urlGetExisting,
			// No "urlPut" because source won't need to upload anything.
		}
		fmt.Fprintln(w, response)
		return
	}

	urlPut, urlGet, err := createUrls(c, contentType, fileSize, hash)
	if err != nil {
		log.Errorf(c, "%v", err)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, `{"success": false}`)
		return
	}

	response := Response{
		"success":  true,
		"existing": false,
		"urlPut":   urlPut,
		"urlGet":   urlGet,
	}
	fmt.Fprintln(w, response)
}

func createUrls(c context.Context, contentType string, fileSize int, hash string) (urlPut, urlGet string, err error) {
	objectName := randomGcsObjectName()
	log.Infof(c, "Creating urls for tmp object name %s with content-type [%s] having size %d", objectName, contentType, fileSize)

	urlPut, err = storage.SignedURL(bucket, objectName, &storage.SignedURLOptions{
		GoogleAccessID: serviceAccount,
		PrivateKey:     pkey,
		Method:         "PUT",
		Expires:        time.Now().Add(9 * time.Minute),
		ContentType:    contentType,
	})
	if err != nil {
		return
	}

	urlGet, err = storage.SignedURL(bucket, objectName, &storage.SignedURLOptions{
		GoogleAccessID: serviceAccount,
		PrivateKey:     pkey,
		Method:         "GET",
		Expires:        time.Now().Add(10 * time.Minute),
	})

	if hash != "" {
		// #32 memorize Hash->ObjectName in Memcache, in case the same file is sent again.
		cacheKey := "objectName_for_" + hash
		cacheItem := &memcache.Item{
			Key:   cacheKey,
			Value: []byte(objectName),
		}
		err := memcache.Set(c, cacheItem)
		if err != nil {
			log.Warningf(c, "Failed setting cache[%v] : %v", cacheKey, err)
		}
	}

	return
}

func findByHash(c context.Context, hash string) (bool, string) {
	// Look in Memcache.
	// Memcache entries are volatile, someday we may want to use Datastore too.
	cacheKey := "objectName_for_" + hash
	var cacheItem *memcache.Item
	var errMC error
	cacheItem, errMC = memcache.Get(c, cacheKey)
	if errMC == memcache.ErrCacheMiss {
		return false, ""
	}
	if errMC != nil {
		log.Warningf(c, "Problem with memcache: %v", errMC)
		return false, ""
	}
	// The value is a GCS object name
	cachedObjectName := string(cacheItem.Value)

	urlGet, err := storage.SignedURL(bucket, cachedObjectName, &storage.SignedURLOptions{
		GoogleAccessID: serviceAccount,
		PrivateKey:     pkey,
		Method:         "GET",
		Expires:        time.Now().Add(10 * time.Minute),
	})
	if err != nil {
		log.Errorf(c, "Creating GET url to object [%s] for known hash [%s]", cachedObjectName, hash)
		return false, ""
	}
	// TODO if cachedObjectName is already scheduled for deletion, then postpone the deletion.
	// Make sure #31 doesn't delete freshly re-sent files.

	return true, urlGet
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"gopkg.in/redis.v5"
)

// Some string constants.
const (
	MASTER  = "master"
	SLAVE   = "slave"
	UNKNOWN = "unknown"
)

// RedisShim contains info about server name and port and a pointer to the
// underlying redis client.
type RedisShim struct {
	redis  *redis.Client
	server string
	host   string
	port   int
}

// NewRedisShim creates a new shim from a server:port string, where the port
// part is optional.
func NewRedisShim(server string) *RedisShim {
	ri := new(RedisShim)
	ri.server = server
	parts := strings.Split(server, ":")
	ri.host = parts[0]
	ri.port, _ = strconv.Atoi(parts[1])
	ri.redis = redisInstanceFromServerString(server)
	return ri
}

func redisInstanceFromServerString(server string) *redis.Client {
	return redis.NewClient(&redis.Options{Addr: server})
}

func dumpMap(m map[string]string) {
	fmt.Printf("MAP[\n")
	for k, v := range m {
		fmt.Printf("'%s' : '%s'\n", k, v)
	}
	fmt.Printf("]MAP\n")
}

// Info runs the INFO command on the redis server and returns a map of strings.
func (ri *RedisShim) Info() (m map[string]string) {
	m = make(map[string]string)
	cmd := ri.redis.Info("Replication")
	s, err := cmd.Result()
	if err != nil {
		return
	}
	re := regexp.MustCompile("([^:]+):(.*)")
	ss := strings.Split(s, "\n")
	for _, x := range ss {
		match := re.FindStringSubmatch(x)
		if len(match) == 3 {
			k, v := match[1], match[2]
			// there's junk at the end of v
			m[k] = v[0 : len(v)-1]
		}
	}
	return
}

// Role returns the role of the redis server ('master' or 'söave'), or 'unknown'
// if the server cannot be reached.
func (ri *RedisShim) Role() string {
	info := ri.Info()
	if len(info) == 0 {
		return UNKNOWN
	}
	return info["role"]
}

// IsMaster checks whether the redis server is a master.
func (ri *RedisShim) IsMaster() bool {
	return ri.Role() == MASTER
}

// IsSlave checks whether the redis server is a slave.
func (ri *RedisShim) IsSlave() bool {
	return ri.Role() == SLAVE
}

// IsAvailable checks whether the server is available.
func (ri *RedisShim) IsAvailable() bool {
	cmd := ri.redis.Ping()
	_, err := cmd.Result()
	return err == nil
}

// MakeMaster sends SALVEOF no one to the redis server.
func (ri *RedisShim) MakeMaster() error {
	cmd := ri.redis.SlaveOf("no", "one")
	_, err := cmd.Result()
	return err
}

// IsSlaveOf checks whether the redis server is a slave of some other server.
func (ri *RedisShim) IsSlaveOf(host string, port int) bool {
	info := ri.Info()
	return info["role"] == SLAVE && info["master_host"] == host && info["master_port"] == strconv.Itoa(port)
}

// RedisMakeSlave makes the redis server slave of some other server.
func (ri *RedisShim) RedisMakeSlave(host string, port int) error {
	cmd := ri.redis.SlaveOf(host, strconv.Itoa(port))
	_, err := cmd.Result()
	return err
}
<commit_msg>redis config server: log errors when redis commands fail<commit_after>package main

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"gopkg.in/redis.v5"
)

// Some string constants.
const (
	MASTER  = "master"
	SLAVE   = "slave"
	UNKNOWN = "unknown"
)

// RedisShim contains info about server name and port and a pointer to the
// underlying redis client.
type RedisShim struct {
	redis  *redis.Client
	server string
	host   string
	port   int
}

// NewRedisShim creates a new shim from a server:port string, where the port
// part is optional.
func NewRedisShim(server string) *RedisShim {
	ri := new(RedisShim)
	ri.server = server
	parts := strings.Split(server, ":")
	ri.host = parts[0]
	ri.port, _ = strconv.Atoi(parts[1])
	ri.redis = redisInstanceFromServerString(server)
	return ri
}

func redisInstanceFromServerString(server string) *redis.Client {
	return redis.NewClient(&redis.Options{Addr: server})
}

func dumpMap(m map[string]string) {
	fmt.Printf("MAP[\n")
	for k, v := range m {
		fmt.Printf("'%s' : '%s'\n", k, v)
	}
	fmt.Printf("]MAP\n")
}

// Info runs the INFO command on the redis server and returns a map of strings.
func (ri *RedisShim) Info() (m map[string]string) {
	m = make(map[string]string)
	cmd := ri.redis.Info("Replication")
	s, err := cmd.Result()
	if err != nil {
		logError("could not obtain redis info from %s: %s", ri.server, err)
		return
	}
	re := regexp.MustCompile("([^:]+):(.*)")
	ss := strings.Split(s, "\n")
	for _, x := range ss {
		match := re.FindStringSubmatch(x)
		if len(match) == 3 {
			k, v := match[1], match[2]
			// there's junk at the end of v
			m[k] = v[0 : len(v)-1]
		}
	}
	return
}

// Role returns the role of the redis server ('master' or 'söave'), or 'unknown'
// if the server cannot be reached.
func (ri *RedisShim) Role() string {
	info := ri.Info()
	if len(info) == 0 {
		return UNKNOWN
	}
	return info["role"]
}

// IsMaster checks whether the redis server is a master.
func (ri *RedisShim) IsMaster() bool {
	return ri.Role() == MASTER
}

// IsSlave checks whether the redis server is a slave.
func (ri *RedisShim) IsSlave() bool {
	return ri.Role() == SLAVE
}

// IsAvailable checks whether the server is available.
func (ri *RedisShim) IsAvailable() bool {
	cmd := ri.redis.Ping()
	_, err := cmd.Result()
	if err != nil {
		logError("pinging server %s failed: %s", ri.server, err)
		return false
	}
	return true
}

// MakeMaster sends SALVEOF no one to the redis server.
func (ri *RedisShim) MakeMaster() error {
	cmd := ri.redis.SlaveOf("no", "one")
	_, err := cmd.Result()
	if err != nil {
		logError("could not make %s a slave of no one: %s", ri.server, err)
	}
	return err
}

// IsSlaveOf checks whether the redis server is a slave of some other server.
func (ri *RedisShim) IsSlaveOf(host string, port int) bool {
	info := ri.Info()
	return info["role"] == SLAVE && info["master_host"] == host && info["master_port"] == strconv.Itoa(port)
}

// RedisMakeSlave makes the redis server slave of some other server.
func (ri *RedisShim) RedisMakeSlave(host string, port int) error {
	cmd := ri.redis.SlaveOf(host, strconv.Itoa(port))
	_, err := cmd.Result()
	if err != nil {
		logError("could not make %s a slave of %s:%d: %s", ri.server, host, port, err)
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"koding/kites/tunnel/protocol"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
)

var serverAddr = "127.0.0.1:7000"
var localAddr = "127.0.0.1:5000"
var registerPath = "_kdtunnel_"

func init() {
	log.SetOutput(os.Stdout)
	log.SetPrefix("tunnel-client ")
	log.SetFlags(log.Lmicroseconds)
}

type TunnelClient struct {
	remoteConn *httputil.ServerConn
	localConn  *httputil.ClientConn
	registered bool
}

func NewTunnelClient(localAddr string) *TunnelClient {
	remoteConn, err := net.Dial("tcp", serverAddr)
	if err != nil {
		log.Fatalln("dial remote err: %s", err)
	}

	localConn, err := net.Dial("tcp", localAddr)
	if err != nil {
		log.Fatalln("dial local err: %s", err)
	}

	return &TunnelClient{
		remoteConn: httputil.NewServerConn(remoteConn, nil),
		localConn:  httputil.NewClientConn(localConn, nil),
	}
}

func main() {
	tunnel := NewTunnelClient(localAddr)
	err := tunnel.Register()
	if err != nil {
		log.Println(err)
		return
	}

	for {
		req, err := tunnel.remoteConn.Read()
		if err != nil {
			fmt.Println("Server read", err)
			return
		}

		go tunnel.handleReq(req)
	}
}

func (t *TunnelClient) handleReq(req *http.Request) {
	fmt.Println(req.RemoteAddr, req.URL.String(), req.Host, req.RequestURI)

	resp, err := t.localConn.Do(req)
	if err != nil {
		fmt.Println("could not do request")
	}

	t.remoteConn.Write(req, resp)
}

// Register registered the tunnel client to the TunnelServer via an CONNECT request.
// It returns an error if the connect request is not successful.
func (t *TunnelClient) Register() error {
	conn, buffer := t.remoteConn.Hijack()

	remoteAddr := fmt.Sprintf("http://%s%s", serverAddr, protocol.RegisterPath)
	req, err := http.NewRequest("CONNECT", remoteAddr, nil)
	if err != nil {
		return fmt.Errorf("CONNECT", err)
	}

	req.Header.Set("Username", "fatih")
	req.Write(conn)

	resp, err := http.ReadResponse(buffer, req)
	if err != nil {
		return fmt.Errorf("read response", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 && resp.Status != protocol.Connected {
		return fmt.Errorf("Non-200 response from proxy server: %s", resp.Status)
	}

	fmt.Println(resp.Status)

	// hijack detaches the server, after doing raw tcp communication
	// attach it again to our tunnelclient
	t.remoteConn = httputil.NewServerConn(conn, nil)
	t.registered = true
	return nil
}
<commit_msg>tunnel: make client package ready<commit_after>package main

import (
	"fmt"
	"koding/kites/tunnel/protocol"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
)

var serverAddr = "127.0.0.1:7000"
var localAddr = "127.0.0.1:5000"
var registerPath = "_kdtunnel_"

func init() {
	log.SetOutput(os.Stdout)
	log.SetPrefix("tunnel-client ")
	log.SetFlags(log.Lmicroseconds)
}

type TunnelClient struct {
	remoteConn *httputil.ServerConn
	localConn  *httputil.ClientConn
	registered bool
}

func NewTunnelClient(localAddr string) *TunnelClient {
	remoteConn, err := net.Dial("tcp", serverAddr)
	if err != nil {
		log.Fatalln("dial remote err: %s", err)
	}

	localConn, err := net.Dial("tcp", localAddr)
	if err != nil {
		log.Fatalln("dial local err: %s", err)
	}

	tunnel := &TunnelClient{
		remoteConn: httputil.NewServerConn(remoteConn, nil),
		localConn:  httputil.NewClientConn(localConn, nil),
	}

	err = tunnel.Register()
	if err != nil {
		log.Fatalln(err)
	}

	fmt.Printf("Tunnel established from %s to %s\n", remoteConn.RemoteAddr(), localConn.RemoteAddr())
	return tunnel
}

func main() {
	tunnel := NewTunnelClient(localAddr)
	tunnel.Start()
}

func (t *TunnelClient) Start() {
	for {
		req, err := t.remoteConn.Read()
		if err != nil {
			fmt.Println("Server read", err)
			return
		}

		go t.handleReq(req)
	}
}

func (t *TunnelClient) handleReq(req *http.Request) {
	resp, err := t.localConn.Do(req)
	if err != nil {
		fmt.Println("could not do request")
	}

	t.remoteConn.Write(req, resp)
}

// Register registered the tunnel client to the TunnelServer via an CONNECT request.
// It returns an error if the connect request is not successful.
func (t *TunnelClient) Register() error {
	conn, buffer := t.remoteConn.Hijack()

	remoteAddr := fmt.Sprintf("http://%s%s", serverAddr, protocol.RegisterPath)
	req, err := http.NewRequest("CONNECT", remoteAddr, nil)
	if err != nil {
		return fmt.Errorf("CONNECT", err)
	}

	req.Header.Set("Username", "fatih")
	req.Write(conn)

	resp, err := http.ReadResponse(buffer, req)
	if err != nil {
		return fmt.Errorf("read response", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 && resp.Status != protocol.Connected {
		return fmt.Errorf("Non-200 response from proxy server: %s", resp.Status)
	}

	// hijack detaches the server, after doing raw tcp communication
	// attach it again to our tunnelclient
	t.remoteConn = httputil.NewServerConn(conn, nil)
	t.registered = true
	return nil
}
<|endoftext|>


<commit_before>package main

import (
  "encoding/json"
  "fmt"
  logging "github.com/op/go-logging"
  "github.com/streadway/amqp"
  . "koding/db/models"
  helper "koding/db/mongodb/modelhelper"
  "koding/messaging/rabbitmq"
  "labix.org/v2/mgo"
  stdlog "log"
  "os"
  "strings"
)

type Status string

const (
  DELETE Status = "delete"
  MERGE  Status = "merge"
)

var (
  EXCHANGE_NAME     = "topicModifierExchange"
  WORKER_QUEUE_NAME = "topicModifierWorkerQueue"
  log               = logging.MustGetLogger("TopicModifier")
)

type TagModifierData struct {
  TagId  string `json:"tagId"`
  Status Status `json:"status"`
}

func init() {
  configureLogger()
}

func main() {
  exchange := rabbitmq.Exchange{
    Name:    EXCHANGE_NAME,
    Type:    "fanout",
    Durable: true,
  }

  queue := rabbitmq.Queue{
    Name:    WORKER_QUEUE_NAME,
    Durable: true,
  }

  binding := rabbitmq.BindingOptions{
    RoutingKey: "",
  }

  consumerOptions := rabbitmq.ConsumerOptions{
    Tag: "TopicModifier",
  }

  consumer, err := rabbitmq.NewConsumer(exchange, queue, binding, consumerOptions)
  if err != nil {
    log.Error("%v", err)
    return
  }

  defer consumer.Shutdown()
  err = consumer.QOS(3)
  if err != nil {
    panic(err)
  }

  defer PUBLISHER.Shutdown()

  log.Info("Tag Modifier worker started")
  consumer.RegisterSignalHandler()
  consumer.Consume(messageConsumer)
}

func configureLogger() {
  logging.SetLevel(logging.INFO, "TopicModifier")
  log.Module = "TopicModifier"
  logging.SetFormatter(logging.MustStringFormatter("%{level:-3s} ▶ %{message}"))
  stderrBackend := logging.NewLogBackend(os.Stderr, "", stdlog.LstdFlags|stdlog.Lshortfile)
  stderrBackend.Color = true
  logging.SetBackend(stderrBackend)
}

var messageConsumer = func(delivery amqp.Delivery) {

  modifierData := &TagModifierData{}
  if err := json.Unmarshal([]byte(delivery.Body), modifierData); err != nil {
    log.Error("Wrong Post Format", err, delivery)
  }

  tagId := modifierData.TagId
  switch modifierData.Status {
  default:
    log.Error("Unknown modification status %s", modifierData.Status)
  case DELETE:
    deleteTags(tagId)
  case MERGE:
    mergeTags(tagId)
  }
  delivery.Ack(false)

}

func deleteTags(tagId string) {
  log.Println("delete")
  if tag := FindTagById(tagId); (tag != Tag{}) {
    log.Println("Valid tag")
    rels := FindRelationships(bson.M{"targetId": bson.ObjectIdHex(tagId), "as": "tag"})
    updatePosts(rels, "")
    updateRelationships(rels, &Tag{})
  }
}

func mergeTags(tagId string) {
  log.Println("merge")

  if tag := FindTagById(tagId); (tag != Tag{}) {
    log.Println("Valid tag")

    synonym := FindSynonym(tagId)
    tagRels := FindRelationships(bson.M{"targetId": bson.ObjectIdHex(tagId), "as": "tag"})
    if len(tagRels) > 0 {
      updatePosts(tagRels, synonym.Id.Hex())
      updateRelationships(tagRels, &synonym)
    }

    postRels := FindRelationships(bson.M{"sourceId": bson.ObjectIdHex(tagId), "as": "post"})
    if len(postRels) > 0 {
      updateRelationships(postRels, &synonym)
      updateCounts(&tag, &synonym)
    }
    synonym.Counts.Followers += updateFollowers(&tag, &synonym)
    UpdateTag(&synonym)
    tag.Counts = TagCount{} // reset counts
    UpdateTag(&tag)
  }
}

func updatePosts(rels []Relationship, newTagId string) {
  var newTag string
  if newTagId != "" {
    newTag = fmt.Sprintf("|#:JTag:%v|", newTagId)
  }

  for _, rel := range rels {
    tagId := rel.TargetId.Hex()
    post := FindPostWithId(rel.SourceId.Hex())
    modifiedTag := fmt.Sprintf("|#:JTag:%v|", tagId)
    post.Body = strings.Replace(post.Body, modifiedTag, newTag, -1)
    UpdatePost(&post)
  }
  log.Printf("%v Posts updated", len(rels))
}

func updateRelationships(rels []Relationship, synonym *Tag) {
  for _, rel := range rels {
    removeRelationship(rel)
    if (synonym != &Tag{}) {
      if rel.TargetName == "JTag" {
        rel.TargetId = synonym.Id
      } else {
        rel.SourceId = synonym.Id
      }
      rel.Id = bson.NewObjectId()
      createRelationship(rel)
    }
  }
}

func updateCounts(tag *Tag, synonym *Tag) {
  synonym.Counts.Following += tag.Counts.Following // does this have any meaning?
  // synonym.Counts.Followers += tag.Counts.Followers
  synonym.Counts.Post += tag.Counts.Post
  synonym.Counts.Tagged += tag.Counts.Tagged
}

func updateFollowers(tag *Tag, synonym *Tag) int {
  var arr [2]bson.M
  arr[0] = bson.M{
    "targetId": tag.Id,
    "as":       "follower",
  }
  arr[1] = bson.M{
    "sourceId": tag.Id,
    "as":       "follower",
  }
  rels := FindRelationships(bson.M{"$or": arr})
  var removeRels []Relationship
  var updateRels []Relationship

  for _, rel := range rels {
    query := bson.M{
      "as": "follower",
    }

    if rel.SourceName == "JTag" {
      query["sourceId"] = synonym.Id
      query["targetId"] = rel.TargetId
    } else {
      query["sourceId"] = rel.SourceId
      query["targetId"] = synonym.Id
    }
    // for filtering already existing relationships
    _, err := FindRelationship(query)

    // what if there is really an error
    if err == nil {
      updateRels = append(updateRels, rel)
    } else {
      removeRels = append(removeRels, rel)
    }
  }

  log.Printf("%v users are already following new topic", len(removeRels))
  if len(removeRels) > 0 {
    updateRelationships(removeRels, &Tag{})
  }
  log.Printf("%v users followed new topic", len(updateRels))
  if len(updateRels) > 0 {
    updateRelationships(updateRels, synonym)
  }

  return len(updateRels)
}

func createRelationship(relationship Relationship) {
  CreateGraphRelationship(relationship)
  CreateRelationship(relationship)
}

func removeRelationship(relationship Relationship) {
  RemoveGraphRelationship(relationship)
  RemoveRelationship(relationship)

}
<commit_msg>Moderation: Topic Merge follower update method is changed<commit_after>package main

import (
  "encoding/json"
  "fmt"
  logging "github.com/op/go-logging"
  "github.com/streadway/amqp"
  . "koding/db/models"
  helper "koding/db/mongodb/modelhelper"
  "koding/messaging/rabbitmq"
  "labix.org/v2/mgo"
  stdlog "log"
  "os"
  "strings"
)

type Status string

const (
  DELETE Status = "delete"
  MERGE  Status = "merge"
)

var (
  EXCHANGE_NAME     = "topicModifierExchange"
  WORKER_QUEUE_NAME = "topicModifierWorkerQueue"
  log               = logging.MustGetLogger("TopicModifier")
)

type TagModifierData struct {
  TagId  string `json:"tagId"`
  Status Status `json:"status"`
}

func init() {
  configureLogger()
}

func main() {
  exchange := rabbitmq.Exchange{
    Name:    EXCHANGE_NAME,
    Type:    "fanout",
    Durable: true,
  }

  queue := rabbitmq.Queue{
    Name:    WORKER_QUEUE_NAME,
    Durable: true,
  }

  binding := rabbitmq.BindingOptions{
    RoutingKey: "",
  }

  consumerOptions := rabbitmq.ConsumerOptions{
    Tag: "TopicModifier",
  }

  consumer, err := rabbitmq.NewConsumer(exchange, queue, binding, consumerOptions)
  if err != nil {
    log.Error("%v", err)
    return
  }

  defer consumer.Shutdown()
  err = consumer.QOS(3)
  if err != nil {
    panic(err)
  }

  defer PUBLISHER.Shutdown()

  log.Info("Tag Modifier worker started")
  consumer.RegisterSignalHandler()
  consumer.Consume(messageConsumer)
}

func configureLogger() {
  logging.SetLevel(logging.INFO, "TopicModifier")
  log.Module = "TopicModifier"
  logging.SetFormatter(logging.MustStringFormatter("%{level:-3s} ▶ %{message}"))
  stderrBackend := logging.NewLogBackend(os.Stderr, "", stdlog.LstdFlags|stdlog.Lshortfile)
  stderrBackend.Color = true
  logging.SetBackend(stderrBackend)
}

var messageConsumer = func(delivery amqp.Delivery) {

  modifierData := &TagModifierData{}
  if err := json.Unmarshal([]byte(delivery.Body), modifierData); err != nil {
    log.Error("Wrong Post Format", err, delivery)
  }

  tagId := modifierData.TagId
  switch modifierData.Status {
  default:
    log.Error("Unknown modification status %s", modifierData.Status)
  case DELETE:
    deleteTags(tagId)
  case MERGE:
    mergeTags(tagId)
  }
  delivery.Ack(false)

}

func deleteTags(tagId string) {
  log.Println("delete")
  if tag := FindTagById(tagId); (tag != Tag{}) {
    log.Println("Valid tag")
    rels := FindRelationships(bson.M{"targetId": bson.ObjectIdHex(tagId), "as": "tag"})
    updatePosts(rels, "")
    updateRelationships(rels, &Tag{})
  }
}

func mergeTags(tagId string) {
  log.Println("merge")

  if tag := FindTagById(tagId); (tag != Tag{}) {
    log.Println("Valid tag")

    synonym := FindSynonym(tagId)
    tagRels := FindRelationships(bson.M{"targetId": bson.ObjectIdHex(tagId), "as": "tag"})
    if len(tagRels) > 0 {
      updatePosts(tagRels, synonym.Id.Hex())
      updateRelationships(tagRels, &synonym)
    }

    postRels := FindRelationships(bson.M{"sourceId": bson.ObjectIdHex(tagId), "as": "post"})
    if len(postRels) > 0 {
      updateRelationships(postRels, &synonym)
      updateCounts(&tag, &synonym)
    }
    synonym.Counts.Followers += updateFollowers(&tag, &synonym)
    UpdateTag(&synonym)
    tag.Counts = TagCount{} // reset counts
    UpdateTag(&tag)
  }
}

func updatePosts(rels []Relationship, newTagId string) {
  var newTag string
  if newTagId != "" {
    newTag = fmt.Sprintf("|#:JTag:%v|", newTagId)
  }

  for _, rel := range rels {
    tagId := rel.TargetId.Hex()
    post := FindPostWithId(rel.SourceId.Hex())
    modifiedTag := fmt.Sprintf("|#:JTag:%v|", tagId)
    post.Body = strings.Replace(post.Body, modifiedTag, newTag, -1)
    UpdatePost(&post)
  }
  log.Printf("%v Posts updated", len(rels))
}

func updateRelationships(rels []Relationship, synonym *Tag) {
  for _, rel := range rels {
    removeRelationship(rel)
    if (synonym != &Tag{}) {
      if rel.TargetName == "JTag" {
        rel.TargetId = synonym.Id
      } else {
        rel.SourceId = synonym.Id
      }
      rel.Id = bson.NewObjectId()
      createRelationship(rel)
    }
  }
}

func updateCounts(tag *Tag, synonym *Tag) {
  synonym.Counts.Following += tag.Counts.Following // does this have any meaning?
  // synonym.Counts.Followers += tag.Counts.Followers
  synonym.Counts.Post += tag.Counts.Post
  synonym.Counts.Tagged += tag.Counts.Tagged
}

func updateFollowers(tag *Tag, synonym *Tag) int {
  selector := helper.Selector{
    "sourceId":   tag.Id,
    "as":         "follower",
    "targetName": "JAccount",
  }

  rels := helper.GetRelationships(selector)
  var oldFollowers []Relationship
  var newFollowers []Relationship

  for _, rel := range rels {
    selector["sourceId"] = synonym.Id
    selector["targetId"] = rel.TargetId

    // checking if relationship already exists for the synonym
    _, err := helper.GetRelationship(selector)
    //because there are two relations as account -> follower -> tag and
    //tag -> follower -> account, we have added
    if err != nil {
      if err == mgo.ErrNotFound {
        newFollowers = append(newFollowers, rel)
      } else {
        log.Error(err.Error())
        return 0
      }
    } else {
      oldFollowers = append(oldFollowers, rel)
    }
  }

  log.Info("%v users are already following new topic", len(oldFollowers))
  if len(oldFollowers) > 0 {
    updateTagRelationships(oldFollowers, &Tag{})
  }
  log.Info("%v users followed new topic", len(newFollowers))
  if len(newFollowers) > 0 {
    updateTagRelationships(newFollowers, synonym)
  }

  return len(newFollowers)
}

func createRelationship(relationship Relationship) {
  CreateGraphRelationship(relationship)
  CreateRelationship(relationship)
}

func removeRelationship(relationship Relationship) {
  RemoveGraphRelationship(relationship)
  RemoveRelationship(relationship)

}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"path"
	"time"

	"vitess.io/vitess/go/vt/log"
)

// DefaultCell : If no cell name is passed, then use following
const DefaultCell = "zone1"

var (
	keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders")
)

// LocalProcessCluster Testcases need to use this to iniate a cluster
type LocalProcessCluster struct {
	Keyspaces          []Keyspace
	Cell               string
	BaseTabletUID      int
	Hostname           string
	TopoPort           int
	TmpDirectory       string
	OriginalVTDATAROOT string
	CurrentVTDATAROOT  string

	VtgateMySQLPort int
	VtgateGrpcPort  int
	VtctldHTTPPort  int

	// standalone executable
	VtctlclientProcess VtctlClientProcess
	VtctlProcess       VtctlProcess

	// background executable processes
	topoProcess   EtcdProcess
	vtctldProcess VtctldProcess
	VtgateProcess VtgateProcess

	nextPortForProcess int

	//Extra arguments for vtTablet
	VtTabletExtraArgs []string

	//Extra arguments for vtGate
	VtGateExtraArgs []string
}

// Keyspace : Cluster accepts keyspace to launch it
type Keyspace struct {
	Name      string
	SchemaSQL string
	VSchema   string
	Shards    []Shard
}

// Shard with associated vttablets
type Shard struct {
	Name      string
	Vttablets []Vttablet
}

// Vttablet stores the properties needed to start a vttablet process
type Vttablet struct {
	Type      string
	TabletUID int
	HTTPPort  int
	GrpcPort  int
	MySQLPort int

	// background executable processes
	mysqlctlProcess MysqlctlProcess
	vttabletProcess VttabletProcess
}

// StartTopo starts topology server
func (cluster *LocalProcessCluster) StartTopo() (err error) {
	if cluster.Cell == "" {
		cluster.Cell = DefaultCell
	}
	cluster.TopoPort = cluster.GetAndReservePort()
	cluster.TmpDirectory = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/tmp_%d", cluster.GetAndReservePort()))
	cluster.topoProcess = *EtcdProcessInstance(cluster.TopoPort, cluster.GetAndReservePort(), cluster.Hostname, "global")
	log.Info(fmt.Sprintf("Starting etcd server on port : %d", cluster.TopoPort))
	if err = cluster.topoProcess.Setup(); err != nil {
		log.Error(err.Error())
		return
	}

	log.Info("Creating topo dirs")
	if err = cluster.topoProcess.ManageTopoDir("mkdir", "/vitess/global"); err != nil {
		log.Error(err.Error())
		return
	}

	if err = cluster.topoProcess.ManageTopoDir("mkdir", "/vitess/"+cluster.Cell); err != nil {
		log.Error(err.Error())
		return
	}

	log.Info("Adding cell info")
	cluster.VtctlProcess = *VtctlProcessInstance(cluster.topoProcess.Port, cluster.Hostname)
	if err = cluster.VtctlProcess.AddCellInfo(cluster.Cell); err != nil {
		log.Error(err)
		return
	}

	cluster.vtctldProcess = *VtctldProcessInstance(cluster.GetAndReservePort(), cluster.GetAndReservePort(), cluster.topoProcess.Port, cluster.Hostname, cluster.TmpDirectory)
	log.Info(fmt.Sprintf("Starting vtctld server on port : %d", cluster.vtctldProcess.Port))
	cluster.VtctldHTTPPort = cluster.vtctldProcess.Port
	if err = cluster.vtctldProcess.Setup(cluster.Cell); err != nil {
		log.Error(err.Error())
		return
	}

	cluster.VtctlclientProcess = *VtctlClientProcessInstance("localhost", cluster.vtctldProcess.GrpcPort, cluster.TmpDirectory)
	return
}

// StartUnshardedKeyspace starts unshared keyspace with shard name as "0"
func (cluster *LocalProcessCluster) StartUnshardedKeyspace(keyspace Keyspace, replicaCount int, rdonly bool) error {
	return cluster.StartKeyspace(keyspace, []string{"0"}, replicaCount, rdonly)
}

// StartKeyspace starts required number of shard and the corresponding tablets
// keyspace : struct containing keyspace name, Sqlschema to apply, VSchema to apply
// shardName : list of shard names
// replicaCount: total number of replicas excluding master and rdonly
// rdonly: whether readonly tablets needed
func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames []string, replicaCount int, rdonly bool) (err error) {
	totalTabletsRequired := replicaCount + 1 // + 1 is for master
	if rdonly {
		totalTabletsRequired = totalTabletsRequired + 1 // + 1 for rdonly
	}
	shards := make([]Shard, 0)
	log.Info("Starting keyspace : " + keyspace.Name)
	_ = cluster.VtctlProcess.CreateKeyspace(keyspace.Name)
	for _, shardName := range shardNames {
		shard := &Shard{
			Name: shardName,
		}
		log.Info("Starting shard : " + shardName)
		for i := 0; i < totalTabletsRequired; i++ {
			// instantiate vttable object with reserved ports
			tablet := &Vttablet{
				TabletUID: cluster.GetAndReserveTabletUID(),
				HTTPPort:  cluster.GetAndReservePort(),
				GrpcPort:  cluster.GetAndReservePort(),
				MySQLPort: cluster.GetAndReservePort(),
			}
			if i == 0 { // Make the first one as master
				tablet.Type = "master"
			} else if i == totalTabletsRequired-1 && rdonly { // Make the last one as rdonly if rdonly flag is passed
				tablet.Type = "rdonly"
			}
			// Start Mysqlctl process
			log.Info(fmt.Sprintf("Starting mysqlctl for table uid %d, mysql port %d", tablet.TabletUID, tablet.MySQLPort))
			tablet.mysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, cluster.TmpDirectory)
			if err = tablet.mysqlctlProcess.Start(); err != nil {
				log.Error(err.Error())
				return
			}

			// start vttablet process
			tablet.vttabletProcess = *VttabletProcessInstance(tablet.HTTPPort,
				tablet.GrpcPort,
				tablet.TabletUID,
				cluster.Cell,
				shardName,
				keyspace.Name,
				cluster.vtctldProcess.Port,
				tablet.Type,
				cluster.topoProcess.Port,
				cluster.Hostname,
				cluster.TmpDirectory,
				cluster.VtTabletExtraArgs)
			log.Info(fmt.Sprintf("Starting vttablet for tablet uid %d, grpc port %d", tablet.TabletUID, tablet.GrpcPort))

			if err = tablet.vttabletProcess.Setup(); err != nil {
				log.Error(err.Error())
				return
			}

			shard.Vttablets = append(shard.Vttablets, *tablet)
		}

		// Make first tablet as master
		if err = cluster.VtctlclientProcess.InitShardMaster(keyspace.Name, shardName, cluster.Cell, shard.Vttablets[0].TabletUID); err != nil {
			log.Error(err.Error())
			return
		}

		shards = append(shards, *shard)
	}
	keyspace.Shards = shards
	cluster.Keyspaces = append(cluster.Keyspaces, keyspace)

	// Apply Schema SQL
	if err = cluster.VtctlclientProcess.ApplySchema(keyspace.Name, keyspace.SchemaSQL); err != nil {
		log.Error(err.Error())
		return
	}

	//Apply VSchema
	if keyspace.VSchema != "" {
		if err = cluster.VtctlclientProcess.ApplyVSchema(keyspace.Name, keyspace.VSchema); err != nil {
			log.Error(err.Error())
			return
		}
	}

	log.Info("Done creating keyspace : " + keyspace.Name)
	return
}

// StartVtgate starts vtgate
func (cluster *LocalProcessCluster) StartVtgate() (err error) {
	vtgateHTTPPort := cluster.GetAndReservePort()
	vtgateGrpcPort := cluster.GetAndReservePort()
	cluster.VtgateMySQLPort = cluster.GetAndReservePort()
	log.Info(fmt.Sprintf("Starting vtgate on port %d", vtgateHTTPPort))
	cluster.VtgateProcess = *VtgateProcessInstance(
		vtgateHTTPPort,
		vtgateGrpcPort,
		cluster.VtgateMySQLPort,
		cluster.Cell,
		cluster.Cell,
		cluster.Hostname,
		"MASTER,REPLICA",
		cluster.topoProcess.Port,
		cluster.TmpDirectory,
		cluster.VtGateExtraArgs)

	log.Info(fmt.Sprintf("Vtgate started, connect to mysql using : mysql -h 127.0.0.1 -P %d", cluster.VtgateMySQLPort))
	err = cluster.VtgateProcess.Setup()
	if err != nil {
		return
	}
	cluster.WaitForTabletsToHealthyInVtgate()
	return nil
}

// NewCluster instantiates a new cluster
func NewCluster(cell string, hostname string) *LocalProcessCluster {
	cluster := &LocalProcessCluster{Cell: cell, Hostname: hostname}
	cluster.OriginalVTDATAROOT = os.Getenv("VTDATAROOT")
	cluster.CurrentVTDATAROOT = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("vtroot_%d", cluster.GetAndReservePort()))
	_ = createDirectory(cluster.CurrentVTDATAROOT, 0700)
	_ = os.Setenv("VTDATAROOT", cluster.CurrentVTDATAROOT)
	rand.Seed(time.Now().UTC().UnixNano())
	return cluster
}

// ReStartVtgate starts vtgate with updated configs
func (cluster *LocalProcessCluster) ReStartVtgate() (err error) {
	err = cluster.VtgateProcess.TearDown()
	if err != nil {
		log.Error(err.Error())
		return
	}
	err = cluster.StartVtgate()
	if err != nil {
		log.Error(err.Error())
		return
	}
	return err
}

// WaitForTabletsToHealthyInVtgate waits for all tablets in all shards to be healthy as per vtgate
func (cluster *LocalProcessCluster) WaitForTabletsToHealthyInVtgate() {
	var isRdOnlyPresent bool
	for _, keyspace := range cluster.Keyspaces {
		for _, shard := range keyspace.Shards {
			isRdOnlyPresent = false
			_ = cluster.VtgateProcess.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.master", keyspace.Name, shard.Name))
			_ = cluster.VtgateProcess.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.replica", keyspace.Name, shard.Name))
			for _, tablet := range shard.Vttablets {
				if tablet.Type == "rdonly" {
					isRdOnlyPresent = true
				}
			}
			if isRdOnlyPresent {
				_ = cluster.VtgateProcess.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.rdonly", keyspace.Name, shard.Name))
			}
		}
	}
}

// Teardown brings down the cluster by invoking teardown for individual processes
func (cluster *LocalProcessCluster) Teardown() (err error) {
	if err = cluster.VtgateProcess.TearDown(); err != nil {
		log.Error(err.Error())
		return
	}

	for _, keyspace := range cluster.Keyspaces {
		for _, shard := range keyspace.Shards {
			for _, tablet := range shard.Vttablets {
				if err = tablet.mysqlctlProcess.Stop(); err != nil {
					log.Error(err.Error())
					return
				}

				if err = tablet.vttabletProcess.TearDown(); err != nil {
					log.Error(err.Error())
					return
				}
			}
		}
	}

	if err = cluster.vtctldProcess.TearDown(); err != nil {
		log.Error(err.Error())
		return
	}

	if err = cluster.topoProcess.TearDown(cluster.Cell, cluster.OriginalVTDATAROOT, cluster.CurrentVTDATAROOT, *keepData); err != nil {
		log.Error(err.Error())
		return
	}
	return err
}

// GetAndReservePort gives port for required process
func (cluster *LocalProcessCluster) GetAndReservePort() int {
	if cluster.nextPortForProcess == 0 {
		cluster.nextPortForProcess = getRandomNumber(20000, 15000)
	}
	cluster.nextPortForProcess = cluster.nextPortForProcess + 1
	return cluster.nextPortForProcess
}

// GetAndReserveTabletUID gives tablet uid
func (cluster *LocalProcessCluster) GetAndReserveTabletUID() int {
	if cluster.BaseTabletUID == 0 {
		cluster.BaseTabletUID = getRandomNumber(10000, 0)
	}
	cluster.BaseTabletUID = cluster.BaseTabletUID + 1
	return cluster.BaseTabletUID
}

func getRandomNumber(maxNumber int32, baseNumber int) int {
	return int(rand.Int31n(maxNumber)) + baseNumber
}
<commit_msg>Propagate err to top<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"path"
	"time"

	"vitess.io/vitess/go/vt/log"
)

// DefaultCell : If no cell name is passed, then use following
const DefaultCell = "zone1"

var (
	keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders")
)

// LocalProcessCluster Testcases need to use this to iniate a cluster
type LocalProcessCluster struct {
	Keyspaces          []Keyspace
	Cell               string
	BaseTabletUID      int
	Hostname           string
	TopoPort           int
	TmpDirectory       string
	OriginalVTDATAROOT string
	CurrentVTDATAROOT  string

	VtgateMySQLPort int
	VtgateGrpcPort  int
	VtctldHTTPPort  int

	// standalone executable
	VtctlclientProcess VtctlClientProcess
	VtctlProcess       VtctlProcess

	// background executable processes
	topoProcess   EtcdProcess
	vtctldProcess VtctldProcess
	VtgateProcess VtgateProcess

	nextPortForProcess int

	//Extra arguments for vtTablet
	VtTabletExtraArgs []string

	//Extra arguments for vtGate
	VtGateExtraArgs []string
}

// Keyspace : Cluster accepts keyspace to launch it
type Keyspace struct {
	Name      string
	SchemaSQL string
	VSchema   string
	Shards    []Shard
}

// Shard with associated vttablets
type Shard struct {
	Name      string
	Vttablets []Vttablet
}

// Vttablet stores the properties needed to start a vttablet process
type Vttablet struct {
	Type      string
	TabletUID int
	HTTPPort  int
	GrpcPort  int
	MySQLPort int

	// background executable processes
	mysqlctlProcess MysqlctlProcess
	vttabletProcess VttabletProcess
}

// StartTopo starts topology server
func (cluster *LocalProcessCluster) StartTopo() (err error) {
	if cluster.Cell == "" {
		cluster.Cell = DefaultCell
	}
	cluster.TopoPort = cluster.GetAndReservePort()
	cluster.TmpDirectory = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/tmp_%d", cluster.GetAndReservePort()))
	cluster.topoProcess = *EtcdProcessInstance(cluster.TopoPort, cluster.GetAndReservePort(), cluster.Hostname, "global")
	log.Info(fmt.Sprintf("Starting etcd server on port : %d", cluster.TopoPort))
	if err = cluster.topoProcess.Setup(); err != nil {
		log.Error(err.Error())
		return
	}

	log.Info("Creating topo dirs")
	if err = cluster.topoProcess.ManageTopoDir("mkdir", "/vitess/global"); err != nil {
		log.Error(err.Error())
		return
	}

	if err = cluster.topoProcess.ManageTopoDir("mkdir", "/vitess/"+cluster.Cell); err != nil {
		log.Error(err.Error())
		return
	}

	log.Info("Adding cell info")
	cluster.VtctlProcess = *VtctlProcessInstance(cluster.topoProcess.Port, cluster.Hostname)
	if err = cluster.VtctlProcess.AddCellInfo(cluster.Cell); err != nil {
		log.Error(err)
		return
	}

	cluster.vtctldProcess = *VtctldProcessInstance(cluster.GetAndReservePort(), cluster.GetAndReservePort(), cluster.topoProcess.Port, cluster.Hostname, cluster.TmpDirectory)
	log.Info(fmt.Sprintf("Starting vtctld server on port : %d", cluster.vtctldProcess.Port))
	cluster.VtctldHTTPPort = cluster.vtctldProcess.Port
	if err = cluster.vtctldProcess.Setup(cluster.Cell); err != nil {
		log.Error(err.Error())
		return
	}

	cluster.VtctlclientProcess = *VtctlClientProcessInstance("localhost", cluster.vtctldProcess.GrpcPort, cluster.TmpDirectory)
	return
}

// StartUnshardedKeyspace starts unshared keyspace with shard name as "0"
func (cluster *LocalProcessCluster) StartUnshardedKeyspace(keyspace Keyspace, replicaCount int, rdonly bool) error {
	return cluster.StartKeyspace(keyspace, []string{"0"}, replicaCount, rdonly)
}

// StartKeyspace starts required number of shard and the corresponding tablets
// keyspace : struct containing keyspace name, Sqlschema to apply, VSchema to apply
// shardName : list of shard names
// replicaCount: total number of replicas excluding master and rdonly
// rdonly: whether readonly tablets needed
func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames []string, replicaCount int, rdonly bool) (err error) {
	totalTabletsRequired := replicaCount + 1 // + 1 is for master
	if rdonly {
		totalTabletsRequired = totalTabletsRequired + 1 // + 1 for rdonly
	}
	shards := make([]Shard, 0)
	log.Info("Starting keyspace : " + keyspace.Name)
	_ = cluster.VtctlProcess.CreateKeyspace(keyspace.Name)
	for _, shardName := range shardNames {
		shard := &Shard{
			Name: shardName,
		}
		log.Info("Starting shard : " + shardName)
		for i := 0; i < totalTabletsRequired; i++ {
			// instantiate vttable object with reserved ports
			tablet := &Vttablet{
				TabletUID: cluster.GetAndReserveTabletUID(),
				HTTPPort:  cluster.GetAndReservePort(),
				GrpcPort:  cluster.GetAndReservePort(),
				MySQLPort: cluster.GetAndReservePort(),
			}
			if i == 0 { // Make the first one as master
				tablet.Type = "master"
			} else if i == totalTabletsRequired-1 && rdonly { // Make the last one as rdonly if rdonly flag is passed
				tablet.Type = "rdonly"
			}
			// Start Mysqlctl process
			log.Info(fmt.Sprintf("Starting mysqlctl for table uid %d, mysql port %d", tablet.TabletUID, tablet.MySQLPort))
			tablet.mysqlctlProcess = *MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, cluster.TmpDirectory)
			if err = tablet.mysqlctlProcess.Start(); err != nil {
				log.Error(err.Error())
				return
			}

			// start vttablet process
			tablet.vttabletProcess = *VttabletProcessInstance(tablet.HTTPPort,
				tablet.GrpcPort,
				tablet.TabletUID,
				cluster.Cell,
				shardName,
				keyspace.Name,
				cluster.vtctldProcess.Port,
				tablet.Type,
				cluster.topoProcess.Port,
				cluster.Hostname,
				cluster.TmpDirectory,
				cluster.VtTabletExtraArgs)
			log.Info(fmt.Sprintf("Starting vttablet for tablet uid %d, grpc port %d", tablet.TabletUID, tablet.GrpcPort))

			if err = tablet.vttabletProcess.Setup(); err != nil {
				log.Error(err.Error())
				return
			}

			shard.Vttablets = append(shard.Vttablets, *tablet)
		}

		// Make first tablet as master
		if err = cluster.VtctlclientProcess.InitShardMaster(keyspace.Name, shardName, cluster.Cell, shard.Vttablets[0].TabletUID); err != nil {
			log.Error(err.Error())
			return
		}

		shards = append(shards, *shard)
	}
	keyspace.Shards = shards
	cluster.Keyspaces = append(cluster.Keyspaces, keyspace)

	// Apply Schema SQL
	if err = cluster.VtctlclientProcess.ApplySchema(keyspace.Name, keyspace.SchemaSQL); err != nil {
		log.Error(err.Error())
		return
	}

	//Apply VSchema
	if keyspace.VSchema != "" {
		if err = cluster.VtctlclientProcess.ApplyVSchema(keyspace.Name, keyspace.VSchema); err != nil {
			log.Error(err.Error())
			return
		}
	}

	log.Info("Done creating keyspace : " + keyspace.Name)
	return
}

// StartVtgate starts vtgate
func (cluster *LocalProcessCluster) StartVtgate() (err error) {
	vtgateHTTPPort := cluster.GetAndReservePort()
	vtgateGrpcPort := cluster.GetAndReservePort()
	cluster.VtgateMySQLPort = cluster.GetAndReservePort()
	log.Info(fmt.Sprintf("Starting vtgate on port %d", vtgateHTTPPort))
	cluster.VtgateProcess = *VtgateProcessInstance(
		vtgateHTTPPort,
		vtgateGrpcPort,
		cluster.VtgateMySQLPort,
		cluster.Cell,
		cluster.Cell,
		cluster.Hostname,
		"MASTER,REPLICA",
		cluster.topoProcess.Port,
		cluster.TmpDirectory,
		cluster.VtGateExtraArgs)

	log.Info(fmt.Sprintf("Vtgate started, connect to mysql using : mysql -h 127.0.0.1 -P %d", cluster.VtgateMySQLPort))
	if err = cluster.VtgateProcess.Setup(); err != nil {
		return err
	}
	if err = cluster.WaitForTabletsToHealthyInVtgate(); err != nil {
		return err
	}
	return nil
}

// NewCluster instantiates a new cluster
func NewCluster(cell string, hostname string) *LocalProcessCluster {
	cluster := &LocalProcessCluster{Cell: cell, Hostname: hostname}
	cluster.OriginalVTDATAROOT = os.Getenv("VTDATAROOT")
	cluster.CurrentVTDATAROOT = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("vtroot_%d", cluster.GetAndReservePort()))
	_ = createDirectory(cluster.CurrentVTDATAROOT, 0700)
	_ = os.Setenv("VTDATAROOT", cluster.CurrentVTDATAROOT)
	rand.Seed(time.Now().UTC().UnixNano())
	return cluster
}

// ReStartVtgate starts vtgate with updated configs
func (cluster *LocalProcessCluster) ReStartVtgate() (err error) {
	err = cluster.VtgateProcess.TearDown()
	if err != nil {
		log.Error(err.Error())
		return
	}
	err = cluster.StartVtgate()
	if err != nil {
		log.Error(err.Error())
		return
	}
	return err
}

// WaitForTabletsToHealthyInVtgate waits for all tablets in all shards to be healthy as per vtgate
func (cluster *LocalProcessCluster) WaitForTabletsToHealthyInVtgate() (err error) {
	var isRdOnlyPresent bool
	for _, keyspace := range cluster.Keyspaces {
		for _, shard := range keyspace.Shards {
			isRdOnlyPresent = false
			if err = cluster.VtgateProcess.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.master", keyspace.Name, shard.Name)); err != nil {
				return err
			}
			if err = cluster.VtgateProcess.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.replica", keyspace.Name, shard.Name)); err != nil {
				return err
			}
			for _, tablet := range shard.Vttablets {
				if tablet.Type == "rdonly" {
					isRdOnlyPresent = true
				}
			}
			if isRdOnlyPresent {
				err = cluster.VtgateProcess.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.rdonly", keyspace.Name, shard.Name))
			}
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// Teardown brings down the cluster by invoking teardown for individual processes
func (cluster *LocalProcessCluster) Teardown() (err error) {
	if err = cluster.VtgateProcess.TearDown(); err != nil {
		log.Error(err.Error())
		return
	}

	for _, keyspace := range cluster.Keyspaces {
		for _, shard := range keyspace.Shards {
			for _, tablet := range shard.Vttablets {
				if err = tablet.mysqlctlProcess.Stop(); err != nil {
					log.Error(err.Error())
					return
				}

				if err = tablet.vttabletProcess.TearDown(); err != nil {
					log.Error(err.Error())
					return
				}
			}
		}
	}

	if err = cluster.vtctldProcess.TearDown(); err != nil {
		log.Error(err.Error())
		return
	}

	if err = cluster.topoProcess.TearDown(cluster.Cell, cluster.OriginalVTDATAROOT, cluster.CurrentVTDATAROOT, *keepData); err != nil {
		log.Error(err.Error())
		return
	}
	return err
}

// GetAndReservePort gives port for required process
func (cluster *LocalProcessCluster) GetAndReservePort() int {
	if cluster.nextPortForProcess == 0 {
		cluster.nextPortForProcess = getRandomNumber(20000, 15000)
	}
	cluster.nextPortForProcess = cluster.nextPortForProcess + 1
	return cluster.nextPortForProcess
}

// GetAndReserveTabletUID gives tablet uid
func (cluster *LocalProcessCluster) GetAndReserveTabletUID() int {
	if cluster.BaseTabletUID == 0 {
		cluster.BaseTabletUID = getRandomNumber(10000, 0)
	}
	cluster.BaseTabletUID = cluster.BaseTabletUID + 1
	return cluster.BaseTabletUID
}

func getRandomNumber(maxNumber int32, baseNumber int) int {
	return int(rand.Int31n(maxNumber)) + baseNumber
}
<|endoftext|>


<commit_before>// Package semver provides an implementation of Semantic Versioning.
package semver

import (
	"errors"
	"strings"
)

var (
	// ErrInvalidOperator is returned when the parser operator is not valid.
	ErrInvalidOperator = errors.New("semver: invalid operator")
)

// Constraint represents a version constraint for a semver version number.
type Constraint struct {
	Operator string
	Version  *Version
}

// NewConstraint returns a new constraint using the '=' operator and a default
// Version object. See NewVersion for the default version number.
func NewConstraint() *Constraint {
	return &Constraint{
		Operator: "=",
		Version:  NewVersion(),
	}
}

// ParseConstraint attempts to parse a semver constraint containing an operator
// and a semver string.
// If the operator is not valid, ErrInvalidOperator is returned.
// If the string is of an invalid semver format, ErrInvalidFormat is returned.
//
// Examples:
//		 =2.0.0 (Equals 2.0.0)
//		 >2.0.0 (Greater than 2.0.0)
//		 <2.0.0 (Less than 2.0.0)
//		>=2.0.0 (Greater than or equal to 2.0.0)
//		<=2.0.0 (Less than or equal to 2.0.0)
func ParseConstraint(str string) (*Constraint, error) {
	c := NewConstraint()
	c.Operator = ""

	operator := ""
	for _, r := range str {
		if !isOperator(r) {
			break
		}
		operator += string(r)
	}
	str = str[len(c.Operator):]

	operators := []string{"=", ">", "<", ">=", "<="}
	for _, o := range operators {
		if o == operator {
			c.Operator = o
		}
	}

	if len(c.Operator) == 0 {
		return nil, ErrInvalidOperator
	}

	v, err := ParseVersion(strings.TrimPrefix(str, c.Operator))
	if err != nil {
		return nil, err
	}
	c.Version = v

	return c, nil
}

// Match attempts to match the given Version with the constraint.
// Valid operators are:
//		 = means equal
//		 > means greater than
//		 < means less than
//		>= means greater than or equal
//		<= means less than or equal
func (c *Constraint) Match(v *Version) bool {
	switch c.Operator {
	case "=":
		return v.Equals(c.Version)

	case ">":
		return v.GreaterThan(c.Version)

	case "<":
		return v.LessThan(c.Version)

	case ">=":
		return v.GreaterThan(c.Version) || v.Equals(c.Version)

	case "<=":
		return v.LessThan(c.Version) || v.Equals(c.Version)
	}

	return false
}

func isOperator(ch rune) bool {
	operators := "=><"
	return strings.ContainsRune(operators, ch)
}
<commit_msg>Remove unnecessary call to strings.TrimPrefix<commit_after>// Package semver provides an implementation of Semantic Versioning.
package semver

import (
	"errors"
	"strings"
)

var (
	// ErrInvalidOperator is returned when the parser operator is not valid.
	ErrInvalidOperator = errors.New("semver: invalid operator")
)

// Constraint represents a version constraint for a semver version number.
type Constraint struct {
	Operator string
	Version  *Version
}

// NewConstraint returns a new constraint using the '=' operator and a default
// Version object. See NewVersion for the default version number.
func NewConstraint() *Constraint {
	return &Constraint{
		Operator: "=",
		Version:  NewVersion(),
	}
}

// ParseConstraint attempts to parse a semver constraint containing an operator
// and a semver string.
// If the operator is not valid, ErrInvalidOperator is returned.
// If the string is of an invalid semver format, ErrInvalidFormat is returned.
//
// Examples:
//		 =2.0.0 (Equals 2.0.0)
//		 >2.0.0 (Greater than 2.0.0)
//		 <2.0.0 (Less than 2.0.0)
//		>=2.0.0 (Greater than or equal to 2.0.0)
//		<=2.0.0 (Less than or equal to 2.0.0)
func ParseConstraint(str string) (*Constraint, error) {
	c := NewConstraint()
	c.Operator = ""

	operator := ""
	for _, r := range str {
		if !isOperator(r) {
			break
		}
		operator += string(r)
	}
	str = str[len(operator):]

	operators := []string{"=", ">", "<", ">=", "<="}
	for _, o := range operators {
		if o == operator {
			c.Operator = o
		}
	}

	if len(c.Operator) == 0 {
		return nil, ErrInvalidOperator
	}

	v, err := ParseVersion(str)
	if err != nil {
		return nil, err
	}
	c.Version = v

	return c, nil
}

// Match attempts to match the given Version with the constraint.
// Valid operators are:
//		 = means equal
//		 > means greater than
//		 < means less than
//		>= means greater than or equal
//		<= means less than or equal
func (c *Constraint) Match(v *Version) bool {
	switch c.Operator {
	case "=":
		return v.Equals(c.Version)

	case ">":
		return v.GreaterThan(c.Version)

	case "<":
		return v.LessThan(c.Version)

	case ">=":
		return v.GreaterThan(c.Version) || v.Equals(c.Version)

	case "<=":
		return v.LessThan(c.Version) || v.Equals(c.Version)
	}

	return false
}

func isOperator(ch rune) bool {
	operators := "=><"
	return strings.ContainsRune(operators, ch)
}
<|endoftext|>


<commit_before>package api

import (
	"net/http"

	"github.com/golang/protobuf/jsonpb"
	"github.com/julienschmidt/httprouter"

	"github.com/octavore/press/proto/press/api"
	"github.com/octavore/press/proto/press/models"
	"github.com/octavore/press/server/router"
)

func (m *Module) ListRoutes(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	routes, err := m.DB.ListRoutes()
	if err != nil {
		return err
	}
	return router.Proto(rw, &api.ListRouteResponse{
		Routes: routes,
	})
}

func (m *Module) ListRoutesByPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	routes, err := m.DB.ListRoutes()
	if err != nil {
		return err
	}
	pageUUID := par.ByName("uuid")
	filteredRoutes := []*models.Route{}
	for _, route := range routes {
		if route.GetPageUuid() == pageUUID {
			filteredRoutes = append(filteredRoutes, route)
		}
	}
	return router.Proto(rw, &api.ListRouteResponse{
		Routes: filteredRoutes,
	})
}

func (m *Module) UpdateRoute(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	route := &models.Route{}
	err := jsonpb.Unmarshal(req.Body, route)
	if err != nil {
		return err
	}
	err = m.DB.UpdateRoute(route)
	if err != nil {
		return err
	}
	return router.Proto(rw, route)
}

func (m *Module) DeleteRoute(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	routeUUID := par.ByName("uuid")
	r, err := m.DB.GetRoute(routeUUID)
	if err != nil {
		return err
	}
	return m.DB.DeleteRoute(r)
}

func (m *Module) UpdateRoutesByPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	pageUUID := par.ByName("uuid")
	pb := &api.UpdatePageRoutesRequest{}
	err := jsonpb.Unmarshal(req.Body, pb)
	if err != nil {
		return err
	}

	newList := map[string]*models.Route{}
	for _, route := range pb.GetRoutes() {
		route.Target = &models.Route_PageUuid{PageUuid: pageUUID}
		newList[route.GetUuid()] = route
	}

	routes, err := m.DB.ListRoutes()
	if err != nil {
		return err
	}

	filteredRoutes := []*models.Route{}
	for _, route := range routes {
		if route.GetPageUuid() == pageUUID {
			if newList[route.GetUuid()] == nil {
				err = m.DB.DeleteRoute(route)
				if err != nil {
					return err
				}
			} else {
				delete(newList, route.GetUuid())
			}
		}
	}

	for _, route := range newList {
		err = m.DB.UpdateRoute(route)
		if err != nil {
			return err
		}
	}

	return router.Proto(rw, &api.ListRouteResponse{
		Routes: filteredRoutes,
	})
}
<commit_msg>server: Fix list updating.<commit_after>package api

import (
	"net/http"

	"github.com/golang/protobuf/jsonpb"
	"github.com/julienschmidt/httprouter"

	"github.com/octavore/press/proto/press/api"
	"github.com/octavore/press/proto/press/models"
	"github.com/octavore/press/server/router"
)

func (m *Module) ListRoutes(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	routes, err := m.DB.ListRoutes()
	if err != nil {
		return err
	}
	return router.Proto(rw, &api.ListRouteResponse{
		Routes: routes,
	})
}

func (m *Module) ListRoutesByPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	routes, err := m.DB.ListRoutes()
	if err != nil {
		return err
	}
	pageUUID := par.ByName("uuid")
	filteredRoutes := []*models.Route{}
	for _, route := range routes {
		if route.GetPageUuid() == pageUUID {
			filteredRoutes = append(filteredRoutes, route)
		}
	}
	return router.Proto(rw, &api.ListRouteResponse{
		Routes: filteredRoutes,
	})
}

func (m *Module) UpdateRoute(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	route := &models.Route{}
	err := jsonpb.Unmarshal(req.Body, route)
	if err != nil {
		return err
	}
	err = m.DB.UpdateRoute(route)
	if err != nil {
		return err
	}
	return router.Proto(rw, route)
}

func (m *Module) DeleteRoute(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	routeUUID := par.ByName("uuid")
	r, err := m.DB.GetRoute(routeUUID)
	if err != nil {
		return err
	}
	return m.DB.DeleteRoute(r)
}

func (m *Module) UpdateRoutesByPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	pageUUID := par.ByName("uuid")
	pb := &api.UpdatePageRoutesRequest{}
	err := jsonpb.Unmarshal(req.Body, pb)
	if err != nil {
		return err
	}

	newList := map[string]*models.Route{}
	for _, route := range pb.GetRoutes() {
		route.Target = &models.Route_PageUuid{PageUuid: pageUUID}
		newList[route.GetUuid()] = route
	}

	routes, err := m.DB.ListRoutes()
	if err != nil {
		return err
	}

	filteredRoutes := []*models.Route{}
	for _, route := range routes {
		if route.GetPageUuid() == pageUUID {
			if newList[route.GetUuid()] == nil {
				err = m.DB.DeleteRoute(route)
				if err != nil {
					return err
				}
			}
		}
	}

	for _, route := range newList {
		err = m.DB.UpdateRoute(route)
		if err != nil {
			return err
		}
	}

	return router.Proto(rw, &api.ListRouteResponse{
		Routes: filteredRoutes,
	})
}
<|endoftext|>


<commit_before>package manager

import (
	"encoding/json"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/runtime"
	"github.com/micro/go-micro/v2/store"
	"github.com/micro/micro/v2/internal/namespace"
)

var (
	// eventTTL is the duration events will perist in the store before expiring
	eventTTL = time.Minute * 10
	// eventPollFrequency is the max frequency the manager will check for new events in the store
	eventPollFrequency = time.Minute
)

// eventPrefix is prefixed to the key for event records
const eventPrefix = "event/"

// publishEvent will write the event to the global store and immediately process the event
func (m *manager) publishEvent(eType runtime.EventType, srv *runtime.Service, opts *runtime.CreateOptions) error {
	e := &runtime.Event{
		ID:      uuid.New().String(),
		Type:    eType,
		Service: srv,
		Options: opts,
	}

	bytes, err := json.Marshal(e)
	if err != nil {
		return err
	}

	record := &store.Record{
		Key:    eventPrefix + e.ID,
		Value:  bytes,
		Expiry: eventTTL,
	}

	if err := m.options.Store.Write(record); err != nil {
		return err
	}

	go m.processEvent(record.Key)
	return nil
}

// watchEvents polls the store for events periodically and processes them if they have not already
// done so
func (m *manager) watchEvents() {
	ticker := time.NewTicker(eventPollFrequency)

	for {
		// get the keys of the events
		events, err := m.options.Store.Read(eventPrefix, store.ReadPrefix())
		if err != nil {
			logger.Warn("Error listing events: %v", err)
			continue
		}

		// loop through every event
		for _, ev := range events {
			logger.Debugf("Process Event: %v", ev.Key)
			m.processEvent(ev.Key)
		}

		<-ticker.C
	}
}

// processEvent will take an event key, verify it hasn't been consumed and then execute it. We pass
// the key and not the ID since the global store and the memory store use the same key prefix so there
// is not point stripping and then re-prefixing.
func (m *manager) processEvent(key string) {
	// check to see if the event has been processed before
	if _, err := m.cache.Read(key); err != store.ErrNotFound {
		return
	}

	// lookup the event
	recs, err := m.options.Store.Read(key)
	if err != nil {
		logger.Warnf("Error finding event %v: %v", key, err)
		return
	}
	var ev *runtime.Event
	if err := json.Unmarshal(recs[0].Value, &ev); err != nil {
		logger.Warnf("Error unmarshaling event %v: %v", key, err)
	}

	// determine the namespace
	ns := namespace.DefaultNamespace
	if ev.Options != nil && len(ev.Options.Namespace) > 0 {
		ns = ev.Options.Namespace
	}

	// log the event
	logger.Infof("Procesing %v event for service %v:%v in namespace %v", ev.Type, ev.Service.Name, ev.Service.Version, ns)

	// apply the event to the managed runtime
	switch ev.Type {
	case runtime.Delete:
		err = m.Runtime.Delete(ev.Service, runtime.DeleteNamespace(ns))
	case runtime.Update:
		err = m.Runtime.Update(ev.Service, runtime.UpdateNamespace(ns))
	case runtime.Create:
		err = m.Runtime.Create(ev.Service,
			runtime.CreateImage(ev.Options.Image),
			runtime.CreateType(ev.Options.Type),
			runtime.CreateNamespace(ns),
			runtime.WithArgs(ev.Options.Args...),
			runtime.WithCommand(ev.Options.Command...),
			runtime.WithEnv(m.runtimeEnv(ev.Options)),
		)
	}

	// if there was an error update the status in the cache
	if err != nil {
		logger.Warnf("Error procesing %v event for service %v:%v in namespace %v: %v,", ev.Type, ev.Service.Name, ev.Service.Version, ns, err)
		ev.Service.Metadata = map[string]string{"status": "error", "error": err.Error()}
		m.cacheStatus(ns, ev.Service)
	}

	// write to the store indicating the event has been consumed. We double the ttl to safely know the
	// event will expire before this record
	m.cache.Write(&store.Record{Key: key, Expiry: eventTTL * 2})
}

// runtimeEnv returns the environment variables which should  be used when creating a service.
func (m *manager) runtimeEnv(options *runtime.CreateOptions) []string {
	setEnv := func(p []string, env map[string]string) {
		for _, v := range p {
			parts := strings.Split(v, "=")
			if len(parts) <= 1 {
				continue
			}
			env[parts[0]] = strings.Join(parts[1:], "=")
		}
	}

	// overwrite any values
	env := map[string]string{}

	// set the env vars provided
	setEnv(options.Env, env)

	// override with vars from the Profile
	setEnv(m.options.Profile, env)

	// temp: set the auth namespace. this will be removed once he namespace can be determined from certs.
	if len(options.Namespace) > 0 {
		env["MICRO_AUTH_NAMESPACE"] = options.Namespace
	}

	// create a new env
	var vars []string
	for k, v := range env {
		vars = append(vars, k+"="+v)
	}

	// setup the runtime env
	return vars
}
<commit_msg>Set service status after update/create<commit_after>package manager

import (
	"encoding/json"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/micro/go-micro/v2/logger"
	"github.com/micro/go-micro/v2/runtime"
	"github.com/micro/go-micro/v2/store"
	"github.com/micro/micro/v2/internal/namespace"
)

var (
	// eventTTL is the duration events will perist in the store before expiring
	eventTTL = time.Minute * 10
	// eventPollFrequency is the max frequency the manager will check for new events in the store
	eventPollFrequency = time.Minute
)

// eventPrefix is prefixed to the key for event records
const eventPrefix = "event/"

// publishEvent will write the event to the global store and immediately process the event
func (m *manager) publishEvent(eType runtime.EventType, srv *runtime.Service, opts *runtime.CreateOptions) error {
	e := &runtime.Event{
		ID:      uuid.New().String(),
		Type:    eType,
		Service: srv,
		Options: opts,
	}

	bytes, err := json.Marshal(e)
	if err != nil {
		return err
	}

	record := &store.Record{
		Key:    eventPrefix + e.ID,
		Value:  bytes,
		Expiry: eventTTL,
	}

	if err := m.options.Store.Write(record); err != nil {
		return err
	}

	go m.processEvent(record.Key)
	return nil
}

// watchEvents polls the store for events periodically and processes them if they have not already
// done so
func (m *manager) watchEvents() {
	ticker := time.NewTicker(eventPollFrequency)

	for {
		// get the keys of the events
		events, err := m.options.Store.Read(eventPrefix, store.ReadPrefix())
		if err != nil {
			logger.Warn("Error listing events: %v", err)
			continue
		}

		// loop through every event
		for _, ev := range events {
			logger.Debugf("Process Event: %v", ev.Key)
			m.processEvent(ev.Key)
		}

		<-ticker.C
	}
}

// processEvent will take an event key, verify it hasn't been consumed and then execute it. We pass
// the key and not the ID since the global store and the memory store use the same key prefix so there
// is not point stripping and then re-prefixing.
func (m *manager) processEvent(key string) {
	// check to see if the event has been processed before
	if _, err := m.cache.Read(key); err != store.ErrNotFound {
		return
	}

	// lookup the event
	recs, err := m.options.Store.Read(key)
	if err != nil {
		logger.Warnf("Error finding event %v: %v", key, err)
		return
	}
	var ev *runtime.Event
	if err := json.Unmarshal(recs[0].Value, &ev); err != nil {
		logger.Warnf("Error unmarshaling event %v: %v", key, err)
	}

	// determine the namespace
	ns := namespace.DefaultNamespace
	if ev.Options != nil && len(ev.Options.Namespace) > 0 {
		ns = ev.Options.Namespace
	}

	// log the event
	logger.Infof("Procesing %v event for service %v:%v in namespace %v", ev.Type, ev.Service.Name, ev.Service.Version, ns)

	// apply the event to the managed runtime
	switch ev.Type {
	case runtime.Delete:
		err = m.Runtime.Delete(ev.Service, runtime.DeleteNamespace(ns))
	case runtime.Update:
		err = m.Runtime.Update(ev.Service, runtime.UpdateNamespace(ns))
	case runtime.Create:
		err = m.Runtime.Create(ev.Service,
			runtime.CreateImage(ev.Options.Image),
			runtime.CreateType(ev.Options.Type),
			runtime.CreateNamespace(ns),
			runtime.WithArgs(ev.Options.Args...),
			runtime.WithCommand(ev.Options.Command...),
			runtime.WithEnv(m.runtimeEnv(ev.Options)),
		)
	}

	// if there was an error update the status in the cache
	if err != nil {
		logger.Warnf("Error procesing %v event for service %v:%v in namespace %v: %v,", ev.Type, ev.Service.Name, ev.Service.Version, ns, err)
		ev.Service.Metadata = map[string]string{"status": "error", "error": err.Error()}
		m.cacheStatus(ns, ev.Service)
	} else if ev.Type != runtime.Delete {
		ev.Service.Metadata = map[string]string{"status": "starting"}
		m.cacheStatus(ns, ev.Service)
	}

	// write to the store indicating the event has been consumed. We double the ttl to safely know the
	// event will expire before this record
	m.cache.Write(&store.Record{Key: key, Expiry: eventTTL * 2})
}

// runtimeEnv returns the environment variables which should  be used when creating a service.
func (m *manager) runtimeEnv(options *runtime.CreateOptions) []string {
	setEnv := func(p []string, env map[string]string) {
		for _, v := range p {
			parts := strings.Split(v, "=")
			if len(parts) <= 1 {
				continue
			}
			env[parts[0]] = strings.Join(parts[1:], "=")
		}
	}

	// overwrite any values
	env := map[string]string{}

	// set the env vars provided
	setEnv(options.Env, env)

	// override with vars from the Profile
	setEnv(m.options.Profile, env)

	// temp: set the auth namespace. this will be removed once he namespace can be determined from certs.
	if len(options.Namespace) > 0 {
		env["MICRO_AUTH_NAMESPACE"] = options.Namespace
	}

	// create a new env
	var vars []string
	for k, v := range env {
		vars = append(vars, k+"="+v)
	}

	// setup the runtime env
	return vars
}
<|endoftext|>


<commit_before>package internal

import (
	"fmt"

	"code.cloudfoundry.org/bbs"
	loggingclient "code.cloudfoundry.org/diego-logging-client"
	"code.cloudfoundry.org/executor"
	"code.cloudfoundry.org/executor/depot/log_streamer"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/rep"
)

type evacuationLRPProcessor struct {
	bbsClient              bbs.InternalClient
	containerDelegate      ContainerDelegate
	metronClient           loggingclient.IngressClient
	cellID                 string
	evacuationTTLInSeconds uint64
	evacuatedContainers    map[string]struct{}
}

func newEvacuationLRPProcessor(bbsClient bbs.InternalClient, containerDelegate ContainerDelegate, metronClient loggingclient.IngressClient, cellID string, evacuationTTLInSeconds uint64) LRPProcessor {
	return &evacuationLRPProcessor{
		bbsClient:              bbsClient,
		containerDelegate:      containerDelegate,
		metronClient:           metronClient,
		cellID:                 cellID,
		evacuationTTLInSeconds: evacuationTTLInSeconds,
		evacuatedContainers:    make(map[string]struct{}),
	}
}

func (p *evacuationLRPProcessor) Process(logger lager.Logger, container executor.Container) {
	logger = logger.Session("evacuation-lrp-processor", lager.Data{
		"container-guid":  container.Guid,
		"container-state": container.State,
	})
	logger.Debug("start")

	streamer := log_streamer.New(
		container.RunInfo.LogConfig.Guid,
		container.RunInfo.LogConfig.SourceName,
		container.RunInfo.LogConfig.Index,
		p.metronClient,
	)

	lrpKey, err := rep.ActualLRPKeyFromTags(container.Tags)
	if err != nil {
		logger.Error("failed-to-generate-lrp-key", err)
		return
	}

	instanceKey, err := rep.ActualLRPInstanceKeyFromContainer(container, p.cellID)
	if err != nil {
		logger.Error("failed-to-generate-instance-key", err)
		return
	}

	lrpContainer := newLRPContainer(lrpKey, instanceKey, container)

	switch lrpContainer.Container.State {
	case executor.StateReserved:
		p.processReservedContainer(logger, lrpContainer)
	case executor.StateInitializing:
		p.processInitializingContainer(logger, lrpContainer)
	case executor.StateCreated:
		p.processCreatedContainer(logger, lrpContainer)
	case executor.StateRunning:
		p.processRunningContainer(logger, lrpContainer, streamer)
	case executor.StateCompleted:
		p.processCompletedContainer(logger, lrpContainer)
	default:
		p.processInvalidContainer(logger, lrpContainer)
	}
}

func (p *evacuationLRPProcessor) processReservedContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-reserved-container")
	p.evacuateClaimedLRPContainer(logger, lrpContainer)
}

func (p *evacuationLRPProcessor) processInitializingContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-initializing-container")
	p.evacuateClaimedLRPContainer(logger, lrpContainer)
}

func (p *evacuationLRPProcessor) processCreatedContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-created-container")
	p.evacuateClaimedLRPContainer(logger, lrpContainer)
}

func (p *evacuationLRPProcessor) processRunningContainer(logger lager.Logger, lrpContainer *lrpContainer, streamer log_streamer.LogStreamer) {
	logger = logger.Session("process-running-container")

	logger.Debug("extracting-net-info-from-container")
	netInfo, err := rep.ActualLRPNetInfoFromContainer(lrpContainer.Container)
	if err != nil {
		logger.Error("failed-extracting-net-info-from-container", err)
		return
	}
	logger.Debug("succeeded-extracting-net-info-from-container")

	if _, ok := p.evacuatedContainers[lrpContainer.Guid]; !ok {
		writeToStream(streamer, fmt.Sprintf("Cell %s requesting replacement for instance %s", p.cellID, lrpContainer.ActualLRPInstanceKey.InstanceGuid))
		p.evacuatedContainers[lrpContainer.Guid] = struct{}{}
	}

	logger.Info("bbs-evacuate-running-actual-lrp", lager.Data{"net_info": netInfo})
	keepContainer, err := p.bbsClient.EvacuateRunningActualLRP(logger, lrpContainer.ActualLRPKey, lrpContainer.ActualLRPInstanceKey, netInfo, p.evacuationTTLInSeconds)
	if keepContainer == false {
		p.containerDelegate.DeleteContainer(logger, lrpContainer.Container.Guid)
	} else if err != nil {
		logger.Error("failed-to-evacuate-running-actual-lrp", err, lager.Data{"lrp-key": lrpContainer.ActualLRPKey})
	}
}

func (p *evacuationLRPProcessor) processCompletedContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-completed-container")

	if lrpContainer.RunResult.Stopped {
		_, err := p.bbsClient.EvacuateStoppedActualLRP(logger, lrpContainer.ActualLRPKey, lrpContainer.ActualLRPInstanceKey)
		if err != nil {
			logger.Error("failed-to-evacuate-stopped-actual-lrp", err, lager.Data{"lrp-key": lrpContainer.ActualLRPKey})
		}
	} else {
		_, err := p.bbsClient.EvacuateCrashedActualLRP(logger, lrpContainer.ActualLRPKey, lrpContainer.ActualLRPInstanceKey, lrpContainer.RunResult.FailureReason)
		if err != nil {
			logger.Error("failed-to-evacuate-crashed-actual-lrp", err, lager.Data{"lrp-key": lrpContainer.ActualLRPKey})
		}
	}

	p.containerDelegate.DeleteContainer(logger, lrpContainer.Guid)
}

func (p *evacuationLRPProcessor) processInvalidContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-invalid-container")
	logger.Error("not-processing-container-in-invalid-state", nil)
}

func (p *evacuationLRPProcessor) evacuateClaimedLRPContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	_, err := p.bbsClient.EvacuateClaimedActualLRP(logger, lrpContainer.ActualLRPKey, lrpContainer.ActualLRPInstanceKey)
	if err != nil {
		logger.Error("failed-to-unclaim-actual-lrp", err, lager.Data{"lrp-key": lrpContainer.ActualLRPKey})
	}

	p.containerDelegate.DeleteContainer(logger, lrpContainer.Container.Guid)
}

func writeToStream(streamer log_streamer.LogStreamer, msg string) {
	fmt.Fprintf(streamer.Stdout(), msg)
	streamer.Flush()
}
<commit_msg>use a sync.Map to avoid a panic when evacuating multiple LRPs<commit_after>package internal

import (
	"fmt"
	"sync"

	"code.cloudfoundry.org/bbs"
	loggingclient "code.cloudfoundry.org/diego-logging-client"
	"code.cloudfoundry.org/executor"
	"code.cloudfoundry.org/executor/depot/log_streamer"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/rep"
)

type evacuationLRPProcessor struct {
	bbsClient              bbs.InternalClient
	containerDelegate      ContainerDelegate
	metronClient           loggingclient.IngressClient
	cellID                 string
	evacuationTTLInSeconds uint64
	evacuatedContainers    sync.Map
}

func newEvacuationLRPProcessor(bbsClient bbs.InternalClient, containerDelegate ContainerDelegate, metronClient loggingclient.IngressClient, cellID string, evacuationTTLInSeconds uint64) LRPProcessor {
	return &evacuationLRPProcessor{
		bbsClient:              bbsClient,
		containerDelegate:      containerDelegate,
		metronClient:           metronClient,
		cellID:                 cellID,
		evacuationTTLInSeconds: evacuationTTLInSeconds,
	}
}

func (p *evacuationLRPProcessor) Process(logger lager.Logger, container executor.Container) {
	logger = logger.Session("evacuation-lrp-processor", lager.Data{
		"container-guid":  container.Guid,
		"container-state": container.State,
	})
	logger.Debug("start")

	streamer := log_streamer.New(
		container.RunInfo.LogConfig.Guid,
		container.RunInfo.LogConfig.SourceName,
		container.RunInfo.LogConfig.Index,
		p.metronClient,
	)

	lrpKey, err := rep.ActualLRPKeyFromTags(container.Tags)
	if err != nil {
		logger.Error("failed-to-generate-lrp-key", err)
		return
	}

	instanceKey, err := rep.ActualLRPInstanceKeyFromContainer(container, p.cellID)
	if err != nil {
		logger.Error("failed-to-generate-instance-key", err)
		return
	}

	lrpContainer := newLRPContainer(lrpKey, instanceKey, container)

	switch lrpContainer.Container.State {
	case executor.StateReserved:
		p.processReservedContainer(logger, lrpContainer)
	case executor.StateInitializing:
		p.processInitializingContainer(logger, lrpContainer)
	case executor.StateCreated:
		p.processCreatedContainer(logger, lrpContainer)
	case executor.StateRunning:
		p.processRunningContainer(logger, lrpContainer, streamer)
	case executor.StateCompleted:
		p.processCompletedContainer(logger, lrpContainer)
	default:
		p.processInvalidContainer(logger, lrpContainer)
	}
}

func (p *evacuationLRPProcessor) processReservedContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-reserved-container")
	p.evacuateClaimedLRPContainer(logger, lrpContainer)
}

func (p *evacuationLRPProcessor) processInitializingContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-initializing-container")
	p.evacuateClaimedLRPContainer(logger, lrpContainer)
}

func (p *evacuationLRPProcessor) processCreatedContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-created-container")
	p.evacuateClaimedLRPContainer(logger, lrpContainer)
}

func (p *evacuationLRPProcessor) processRunningContainer(logger lager.Logger, lrpContainer *lrpContainer, streamer log_streamer.LogStreamer) {
	logger = logger.Session("process-running-container")

	logger.Debug("extracting-net-info-from-container")
	netInfo, err := rep.ActualLRPNetInfoFromContainer(lrpContainer.Container)
	if err != nil {
		logger.Error("failed-extracting-net-info-from-container", err)
		return
	}
	logger.Debug("succeeded-extracting-net-info-from-container")

	if _, ok := p.evacuatedContainers.LoadOrStore(lrpContainer.Guid, struct{}{}); !ok {
		writeToStream(streamer, fmt.Sprintf("Cell %s requesting replacement for instance %s", p.cellID, lrpContainer.ActualLRPInstanceKey.InstanceGuid))
	}

	logger.Info("bbs-evacuate-running-actual-lrp", lager.Data{"net_info": netInfo})
	keepContainer, err := p.bbsClient.EvacuateRunningActualLRP(logger, lrpContainer.ActualLRPKey, lrpContainer.ActualLRPInstanceKey, netInfo, p.evacuationTTLInSeconds)
	if keepContainer == false {
		p.containerDelegate.DeleteContainer(logger, lrpContainer.Container.Guid)
	} else if err != nil {
		logger.Error("failed-to-evacuate-running-actual-lrp", err, lager.Data{"lrp-key": lrpContainer.ActualLRPKey})
	}
}

func (p *evacuationLRPProcessor) processCompletedContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-completed-container")

	if lrpContainer.RunResult.Stopped {
		_, err := p.bbsClient.EvacuateStoppedActualLRP(logger, lrpContainer.ActualLRPKey, lrpContainer.ActualLRPInstanceKey)
		if err != nil {
			logger.Error("failed-to-evacuate-stopped-actual-lrp", err, lager.Data{"lrp-key": lrpContainer.ActualLRPKey})
		}
	} else {
		_, err := p.bbsClient.EvacuateCrashedActualLRP(logger, lrpContainer.ActualLRPKey, lrpContainer.ActualLRPInstanceKey, lrpContainer.RunResult.FailureReason)
		if err != nil {
			logger.Error("failed-to-evacuate-crashed-actual-lrp", err, lager.Data{"lrp-key": lrpContainer.ActualLRPKey})
		}
	}

	p.containerDelegate.DeleteContainer(logger, lrpContainer.Guid)
}

func (p *evacuationLRPProcessor) processInvalidContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	logger = logger.Session("process-invalid-container")
	logger.Error("not-processing-container-in-invalid-state", nil)
}

func (p *evacuationLRPProcessor) evacuateClaimedLRPContainer(logger lager.Logger, lrpContainer *lrpContainer) {
	_, err := p.bbsClient.EvacuateClaimedActualLRP(logger, lrpContainer.ActualLRPKey, lrpContainer.ActualLRPInstanceKey)
	if err != nil {
		logger.Error("failed-to-unclaim-actual-lrp", err, lager.Data{"lrp-key": lrpContainer.ActualLRPKey})
	}

	p.containerDelegate.DeleteContainer(logger, lrpContainer.Container.Guid)
}

func writeToStream(streamer log_streamer.LogStreamer, msg string) {
	fmt.Fprintf(streamer.Stdout(), msg)
	streamer.Flush()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"strings"

	"github.com/go-yaml"
)

const (
	apiPath = "/api/v3"
)

type issueRange struct {
	from, to int
}

type project struct {
	ServerURL string `yaml:"url"`
	Name      string `yaml:"project"`
	Token     string
	Issues    []string
	// Same as Issues but converted to int by parseConfig
	issues []issueRange
}

// matches checks whether issue is part of p.issues. Always
// true if p.issues is an empty list, otherwise check all entries
// and ranges, if any.
func (p *project) matches(issue int) bool {
	if len(p.issues) == 0 {
		return true
	}
	for _, i := range p.issues {
		if issue >= i.from && issue <= i.to {
			return true
		}
	}
	return false
}

// parseIssues ensure issues items are valid input data, i.e castable
// to int, ranges allowed.
func (p *project) parseIssues() error {
	p.issues = make([]issueRange, 0)
	var x [2]int
	for _, i := range p.Issues {
		vals := strings.Split(i, "-")
		if len(vals) > 2 {
			return fmt.Errorf("only one range separator allowed, '%s' not supported", vals)
		}
		if len(vals) > 1 {
			for k, p := range vals {
				num, err := strconv.ParseUint(p, 10, 64)
				if err != nil {
					return fmt.Errorf("wrong issue range in '%s': expects an integer, not '%s'", i, p)
				}
				x[k] = int(num)
			}
			if x[0] > x[1] {
				return fmt.Errorf("reverse range not allowed in '%s'", i)
			}
		} else {
			// No range
			num, err := strconv.ParseUint(vals[0], 10, 64)
			if err != nil {
				return fmt.Errorf("wrong issue value for '%s': expects an integer, not '%s'", i, vals[0])
			}
			x[0] = int(num)
			x[1] = int(num)
		}
		p.issues = append(p.issues, issueRange{from: x[0], to: x[1]})
	}
	return nil
}

type config struct {
	From, To *project
}

func checkProjectData(p *project, prefix string) error {
	if p == nil {
		return fmt.Errorf("missing %s project's data", prefix)
	}
	if p.ServerURL == "" {
		return fmt.Errorf("missing %s project's server URL", prefix)
	}
	if !strings.HasSuffix(p.ServerURL, apiPath) {
		p.ServerURL += apiPath
	}
	if p.Name == "" {
		return fmt.Errorf("missing %s project's name", prefix)
	}
	if p.Token == "" {
		return fmt.Errorf("missing %s project's token", prefix)
	}
	return nil
}

func parseConfig(name string) (*config, error) {
	data, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, err
	}

	c := new(config)
	if err := yaml.Unmarshal(data, c); err != nil {
		log.Fatal(err)
	}

	if err := checkProjectData(c.From, "source"); err != nil {
		return nil, err
	}
	if err := checkProjectData(c.To, "destination"); err != nil {
		return nil, err
	}
	if err := c.From.parseIssues(); err != nil {
		return nil, err
	}

	return c, nil
}
<commit_msg>Makes cleaner source/target project url. Fixes #2<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"path"
	"strconv"
	"strings"

	"github.com/go-yaml"
)

const (
	apiPath = "/api/v3"
)

type issueRange struct {
	from, to int
}

type project struct {
	ServerURL string `yaml:"url"`
	Name      string `yaml:"project"`
	Token     string
	Issues    []string
	// Same as Issues but converted to int by parseConfig
	issues []issueRange
}

// matches checks whether issue is part of p.issues. Always
// true if p.issues is an empty list, otherwise check all entries
// and ranges, if any.
func (p *project) matches(issue int) bool {
	if len(p.issues) == 0 {
		return true
	}
	for _, i := range p.issues {
		if issue >= i.from && issue <= i.to {
			return true
		}
	}
	return false
}

// parseIssues ensure issues items are valid input data, i.e castable
// to int, ranges allowed.
func (p *project) parseIssues() error {
	p.issues = make([]issueRange, 0)
	var x [2]int
	for _, i := range p.Issues {
		vals := strings.Split(i, "-")
		if len(vals) > 2 {
			return fmt.Errorf("only one range separator allowed, '%s' not supported", vals)
		}
		if len(vals) > 1 {
			for k, p := range vals {
				num, err := strconv.ParseUint(p, 10, 64)
				if err != nil {
					return fmt.Errorf("wrong issue range in '%s': expects an integer, not '%s'", i, p)
				}
				x[k] = int(num)
			}
			if x[0] > x[1] {
				return fmt.Errorf("reverse range not allowed in '%s'", i)
			}
		} else {
			// No range
			num, err := strconv.ParseUint(vals[0], 10, 64)
			if err != nil {
				return fmt.Errorf("wrong issue value for '%s': expects an integer, not '%s'", i, vals[0])
			}
			x[0] = int(num)
			x[1] = int(num)
		}
		p.issues = append(p.issues, issueRange{from: x[0], to: x[1]})
	}
	return nil
}

type config struct {
	From, To *project
}

func checkProjectData(p *project, prefix string) error {
	if p == nil {
		return fmt.Errorf("missing %s project's data", prefix)
	}
	if p.ServerURL == "" {
		return fmt.Errorf("missing %s project's server URL", prefix)
	}
	u, err := url.Parse(p.ServerURL)
	if err != nil {
		return err
	}
	if !strings.HasSuffix(p.ServerURL, apiPath) {
		p.ServerURL = path.Join(u.Host, u.Path, apiPath)
		p.ServerURL = fmt.Sprintf("%s://%s", u.Scheme, p.ServerURL)
	}
	if p.Name == "" {
		return fmt.Errorf("missing %s project's name", prefix)
	}
	if p.Token == "" {
		return fmt.Errorf("missing %s project's token", prefix)
	}
	return nil
}

func parseConfig(name string) (*config, error) {
	data, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, err
	}

	c := new(config)
	if err := yaml.Unmarshal(data, c); err != nil {
		log.Fatal(err)
	}

	if err := checkProjectData(c.From, "source"); err != nil {
		return nil, err
	}
	if err := checkProjectData(c.To, "destination"); err != nil {
		return nil, err
	}
	if err := c.From.parseIssues(); err != nil {
		return nil, err
	}

	return c, nil
}
<|endoftext|>