<commit_before>package loga

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/malnick/logasaurus/config"

	"github.com/mgutz/ansi"
)

type ESRequest struct {
	Size int `json:"size"`
	Sort struct {
		Timestamp string `json:"@timestamp"`
	} `json:"sort"`
	Query struct {
		Filtered struct {
			Query struct {
				QueryString struct {
					AnalyzeWildcard string `json:"analyze_wildcard"`
					Query           string `json:"query"`
				} `json:"query_string"`
			} `json:"query"`
			Filter struct {
				Bool struct {
					Must    []ESMust    `json:"must"`
					MustNot []ESMustNot `json:"must_not"`
				} `json:"bool"`
			} `json:"filter"`
		} `json:"filtered"`
	} `json:"query"`
}

type ESMust struct {
	Range struct {
		Timestamp struct {
			Gte interface{} `json:"gte"`
			Lte interface{} `json:"lte"`
		} `json:"@timestamp"`
	} `json:"range"`
}

type ESMustNot struct{}

func (esRequest *ESRequest) makeRequest(c *config.Config) (ESResponse, error) {
	var esResponse ESResponse

	jsonpost, err := json.MarshalIndent(&esRequest, "", "\t")
	if err != nil {
		return esResponse, err
	}
	log.Debugf("Elastic Search Request:\n %s", string(jsonpost))

	// Craft the request URI
	queryURL := strings.Join([]string{"http://", c.ElasticsearchURL, ":", c.ElasticsearchPort, "/_search?pretty"}, "")
	log.Debug("Query URI: ", queryURL)

	// Make request
	req, err := http.NewRequest("POST", queryURL, bytes.NewBuffer(jsonpost))
	if err != nil {
		return esResponse, err
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return esResponse, err
	}
	defer resp.Body.Close()

	jsonRespBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return esResponse, err
	}
	log.Debugf("Elastic Search Response:\n%s", string(jsonRespBody))

	err = json.Unmarshal(jsonRespBody, &esResponse)
	if err != nil {
		return esResponse, err
	}
	CheckElasticResponse(&esResponse)

	return esResponse, nil
}

// Example good response
var foo = `
{
  "hits" : {
	    "total" : 0,
			"max_score" : null,
			"hits" : [
			{
				"_index" : "logstash-2016.05.07",
				"_type" : "logs",
				"_id" : "AVSNHuWYRX6YZTX2Znoy",
				"_score" : null,
				"_source" : {
					"message" : "May 06 17:47:34 ip-10-0-4-15.us-west-2.compute.internal mesos-master[2866]: I0506 17:47:34.510301  2878 recover.cpp:462] Recover process terminated",
					"@version" : "1",
					"@timestamp" : "2016-05-07T21:28:12.918Z",
					"path" : "/vagrant/test_logs/10.0.4.15/dcos-mesos-master.service.log",
					"host" : "vagrant-ubuntu-trusty-64"
				},
				"sort" : [ 1462656492918 ]
			},
			{
				"_index" : "logstash-2016.05.07",
				"_type" : "logs",
				"_id" : "AVSNHuWYRX6YZTX2ZnoT",
				"_score" : null,
				"_source" : {
					"message" : "May 06 17:47:33 ip-10-0-4-15.us-west-2.compute.internal mesos-master[2866]: I0506 17:47:33.782208  2877 recover.cpp:193] Received a recover response from a replica in EMPTY status",
					"@version" : "1",
					"@timestamp" : "2016-05-07T21:28:12.918Z",
					"path" : "/vagrant/test_logs/10.0.4.15/dcos-mesos-master.service.log",
					"host" : "vagrant-ubuntu-trusty-64"
				}
			]
	}
}
`

type Hit struct {
	Source struct {
		Host    string `json:"host"`
		Message string `json:"message"`
	} `json:"_source"`
}

type ESResponse struct {
	Hits struct {
		Hits []Hit `json:"hits"`
	} `json:"hits"`
	Status int `json:"status"`
}

func (esResponse *ESResponse) printResponse(c config.Config, service string) {
	// Print
	for _, hit := range esResponse.Hits.Hits {
		if c.SearchHost {
			message := hit.Source.Message
			host := ansi.Color(hit.Source.Host, "cyan:black")
			withHost := strings.Join([]string{host, " ", message}, "")
			if c.Highlight {
				highlightQuery(withHost, service)
			} else {
				fmt.Println(withHost)
			}
		} else {
			message := hit.Source.Message
			if c.Highlight {
				highlightQuery(message, service)
			} else {
				fmt.Println(message)
			}
		}
	}
}

func elasticRunner(service string, c config.Config) {
	var (
		esRequest = ESRequest{}
		must      = ESMust{}
		lte       = time.Now().Add(time.Duration(-c.StartTime) * time.Minute)
	)
	for syncCount := 0; syncCount >= 0; syncCount++ {
		// Set time: last 10min or last sync_interval
		if syncCount > 0 {
			must.Range.Timestamp.Gte = lte.Add(time.Duration(-c.SyncInterval) * time.Second)
		} else {
			must.Range.Timestamp.Gte = lte.Add(time.Duration(-c.SyncDepth) * time.Minute)
		}

		must.Range.Timestamp.Lte = lte

		esRequest.Size = c.Count
		esRequest.Sort.Timestamp = "asc"
		esRequest.Query.Filtered.Query.QueryString.AnalyzeWildcard = "true"
		esRequest.Query.Filtered.Query.QueryString.Query = string(service)
		esRequest.Query.Filtered.Filter.Bool.Must = []ESMust{must}

		esResponse, err := esRequest.makeRequest(&c)
		BasicCheckOrExit(err)

		esResponse.printResponse(c, service)
		time.Sleep(time.Second * time.Duration(c.SyncInterval))
	}
}
<commit_msg>cleaning up<commit_after>package loga

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/malnick/logasaurus/config"

	"github.com/mgutz/ansi"
)

type ESRequest struct {
	Size int `json:"size"`
	Sort struct {
		Timestamp string `json:"@timestamp"`
	} `json:"sort"`
	Query struct {
		Filtered struct {
			Query struct {
				QueryString struct {
					AnalyzeWildcard string `json:"analyze_wildcard"`
					Query           string `json:"query"`
				} `json:"query_string"`
			} `json:"query"`
			Filter struct {
				Bool struct {
					Must    []ESMust    `json:"must"`
					MustNot []ESMustNot `json:"must_not"`
				} `json:"bool"`
			} `json:"filter"`
		} `json:"filtered"`
	} `json:"query"`
}

type ESMust struct {
	Range struct {
		Timestamp struct {
			Gte interface{} `json:"gte"`
			Lte interface{} `json:"lte"`
		} `json:"@timestamp"`
	} `json:"range"`
}

type ESMustNot struct{}

func (esRequest *ESRequest) makeRequest(c *config.Config) (ESResponse, error) {
	var esResponse ESResponse

	jsonpost, err := json.MarshalIndent(&esRequest, "", "\t")
	if err != nil {
		return esResponse, err
	}
	log.Debugf("Elastic Search Request:\n %s", string(jsonpost))

	// Craft the request URI
	queryURL := strings.Join([]string{"http://", c.ElasticsearchURL, ":", c.ElasticsearchPort, "/_search?pretty"}, "")
	log.Debug("Query URI: ", queryURL)

	// Make request
	req, err := http.NewRequest("POST", queryURL, bytes.NewBuffer(jsonpost))
	if err != nil {
		return esResponse, err
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return esResponse, err
	}
	defer resp.Body.Close()

	jsonRespBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return esResponse, err
	}
	log.Debugf("Elastic Search Response:\n%s", string(jsonRespBody))

	err = json.Unmarshal(jsonRespBody, &esResponse)
	if err != nil {
		return esResponse, err
	}
	CheckElasticResponse(&esResponse)

	return esResponse, nil
}

type Hit struct {
	Source struct {
		Host    string `json:"host"`
		Message string `json:"message"`
	} `json:"_source"`
}

type ESResponse struct {
	Hits struct {
		Hits []Hit `json:"hits"`
	} `json:"hits"`
	Status int `json:"status"`
}

func (esResponse *ESResponse) printResponse(c config.Config, service string) {
	// Print
	for _, hit := range esResponse.Hits.Hits {
		if c.SearchHost {
			message := hit.Source.Message
			host := ansi.Color(hit.Source.Host, "cyan:black")
			withHost := strings.Join([]string{host, " ", message}, "")
			if c.Highlight {
				highlightQuery(withHost, service)
			} else {
				fmt.Println(withHost)
			}
		} else {
			message := hit.Source.Message
			if c.Highlight {
				highlightQuery(message, service)
			} else {
				fmt.Println(message)
			}
		}
	}
}

func elasticRunner(service string, c config.Config) {
	var (
		esRequest = ESRequest{}
		must      = ESMust{}
		lte       = time.Now().Add(time.Duration(-c.StartTime) * time.Minute)
	)
	for syncCount := 0; syncCount >= 0; syncCount++ {
		// Set time: last 10min or last sync_interval
		if syncCount > 0 {
			must.Range.Timestamp.Gte = lte.Add(time.Duration(-c.SyncInterval) * time.Second)
		} else {
			must.Range.Timestamp.Gte = lte.Add(time.Duration(-c.SyncDepth) * time.Minute)
		}

		must.Range.Timestamp.Lte = lte

		esRequest.Size = c.Count
		esRequest.Sort.Timestamp = "asc"
		esRequest.Query.Filtered.Query.QueryString.AnalyzeWildcard = "true"
		esRequest.Query.Filtered.Query.QueryString.Query = string(service)
		esRequest.Query.Filtered.Filter.Bool.Must = []ESMust{must}

		esResponse, err := esRequest.makeRequest(&c)
		BasicCheckOrExit(err)

		esResponse.printResponse(c, service)
		time.Sleep(time.Second * time.Duration(c.SyncInterval))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ying32/govcl/vcl"
	_ "github.com/ying32/govcl/vcl/exts/winappres"
)

func main() {

	vcl.Application.SetFormScaled(true)
	vcl.Application.Initialize()
	vcl.Application.SetMainFormOnTaskBar(true)
	vcl.Application.SetOnException(func(sender vcl.IObject, e *vcl.Exception) {
		vcl.ShowMessage(e.Message())
	})
	//   Form1.gfm
	vcl.Application.CreateForm(&Form1)
	// 字节加载方式
	vcl.Application.CreateForm(&Form2)

	vcl.Application.Run()

}
<commit_msg>Update basicResForm example<commit_after>package main

import (
	"github.com/ying32/govcl/vcl"
	_ "github.com/ying32/govcl/vcl/exts/winappres"
)

func main() {

	//vcl.Application.SetFormScaled(true)
	//vcl.Application.Initialize()
	//vcl.Application.SetMainFormOnTaskBar(true)
	//vcl.Application.SetOnException(func(sender vcl.IObject, e *vcl.Exception) {
	//	vcl.ShowMessage(e.Message())
	//})
	////   Form1.gfm
	//vcl.Application.CreateForm(&Form1)
	//// 字节加载方式
	//vcl.Application.CreateForm(&Form2)
	//
	//vcl.Application.Run()
	vcl.RunApp(&Form1, &Form2)

}
<|endoftext|>


<commit_before>// +build freebsd
// tun_freebsd.go -- tun interface with cgo for linux / bsd
//

package samtun

/*

#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <net/if.h>
#include <net/if_tun.h>
#include <stdio.h>

int tundev_open(char * ifname) {
  if (strlen(ifname) > IFNAMSIZ) {
    return -1;
  }
  char name[IFNAMSIZ];
  sprintf(name, "/dev/%s", ifname);
  int fd = open(name, O_RDWR);
  if (fd > 0) {
    int i = 0;
    ioctl(fd, TUNSLMODE, &i);
    ioctl(fd, TUNSIFHEAD, &i);
  }
  return fd;
}

int tundev_up(char * ifname, char * addr, char * dstaddr, int mtu) {

  struct ifreq ifr;
  memset(&ifr, 0, sizeof(struct ifreq));
  strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
  int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
  if ( fd > 0 ) {
    if ( ioctl(fd, SIOCGIFINDEX, (void*) &ifr) < 0 ) {
      perror("SIOCGIFINDEX");
      close(fd);
      return -1;
    }
    int idx = ifr.ifr_index;
    ifr.ifr_mtu = mtu;
    if ( ioctl(fd, SIOCSIFMTU, (void*) &ifr) < 0) {
      close(fd);
      perror("SIOCSIFMTU");
      return -1;
    }

    if ( ioctl(fd, SIOCGIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCGIFFLAGS");
      return -1;
    }
    ifr.ifr_flags |= IFF_UP | IFF_RUNNING;
    if ( ioctl(fd, SIOCSIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      return -1;
    }

    struct sockaddr_in dst;
    memset(&dst, 0, sizeof(struct sockaddr_in));
    dst.sin_family = AF_INET;
    if ( ! inet_aton(dstaddr, &dst.sin_addr) ) {
      printf("invalid dstaddr %s\n", dstaddr);
      close(fd);
      return -1;
    }

    struct sockaddr_in src;
    memset(&src, 0, sizeof(struct sockaddr_in));
    src.sin_family = AF_INET;
    if ( ! inet_aton(addr, &src.sin_addr) ) {
      printf("invalid srcaddr %s\n", addr);
      close(fd);
      return -1;
    }
    memset(&ifr, 0, sizeof(struct ifreq));
    if (strlen(ifname) > IFNAMSIZ) {
      return -1;
    }
    char name[IFNAMSIZ];
    sprintf(name, "/dev/%s", ifname);
    strncpy(ifr.ifr_name, name, IFNAMSIZ);
    memcpy(&ifr.ifr_addr, &src, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }
    memcpy(&ifr.ifr_dstaddr, &dst, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFDSTADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }

    close(fd);
    return 0;
  }
  return -1;
}

void tundev_close(int fd) {
  close(fd);
}

*/
import "C"

import (
  "errors"
)

type tunDev C.int

func newTun(ifname, addr, dstaddr string, mtu int) (t tunDev, err error) {
  fd := C.tundev_open(C.CString(ifname))

  if fd == -1 {
    err = errors.New("cannot open tun interface")
  } else {
    if C.tundev_up(C.CString(ifname), C.CString(addr), C.CString(dstaddr), C.int(mtu)) < C.int(0) {
      err = errors.New("cannot put up interface")
    } else {
      return tunDev(fd), nil
    }
  }
  return -1, err
}

func (d tunDev) Close() {
  C.tundev_close(C.int(d))
}
<commit_msg>try fixing freebsd<commit_after>// +build freebsd
// tun_freebsd.go -- tun interface with cgo for linux / bsd
//

package samtun

/*

#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <net/if.h>
#include <net/if_tun.h>
#include <stdio.h>

int tundev_open(char * ifname) {
  if (strlen(ifname) > IFNAMSIZ) {
    return -1;
  }
  char name[IFNAMSIZ];
  sprintf(name, "/dev/%s", ifname);
  int fd = open(name, O_RDWR);
  if (fd > 0) {
    int i = 0;
    ioctl(fd, TUNSLMODE, &i);
    ioctl(fd, TUNSIFHEAD, &i);
  }
  return fd;
}

int tundev_up(char * ifname, char * addr, char * dstaddr, int mtu) {

  struct ifreq ifr;
  memset(&ifr, 0, sizeof(struct ifreq));
  strncpy(ifr.ifr_name, ifname, IFNAMSIZ);
  int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
  if ( fd > 0 ) {
    if ( ioctl(fd, SIOCGIFINDEX, (void*) &ifr) < 0 ) {
      perror("SIOCGIFINDEX");
      close(fd);
      return -1;
    }
    int idx = ifr.ifr_index;
    ifr.ifr_mtu = mtu;
    if ( ioctl(fd, SIOCSIFMTU, (void*) &ifr) < 0) {
      close(fd);
      perror("SIOCSIFMTU");
      return -1;
    }

    if ( ioctl(fd, SIOCGIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCGIFFLAGS");
      return -1;
    }
    ifr.ifr_flags |= IFF_UP | IFF_RUNNING;
    if ( ioctl(fd, SIOCSIFFLAGS, (void*)&ifr) < 0 ) {
      close(fd);
      return -1;
    }

    struct sockaddr_in dst;
    memset(&dst, 0, sizeof(struct sockaddr_in));
    dst.sin_family = AF_INET;
    if ( ! inet_aton(dstaddr, &dst.sin_addr) ) {
      printf("invalid dstaddr %s\n", dstaddr);
      close(fd);
      return -1;
    }

    struct sockaddr_in src;
    memset(&src, 0, sizeof(struct sockaddr_in));
    src.sin_family = AF_INET;
    if ( ! inet_aton(addr, &src.sin_addr) ) {
      printf("invalid srcaddr %s\n", addr);
      close(fd);
      return -1;
    }
    ifr.ifr_index = idx;
    memcpy(&ifr.ifr_addr, &src, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }
    memcpy(&ifr.ifr_dstaddr, &dst, sizeof(struct sockaddr_in));
    if ( ioctl(fd, SIOCSIFDSTADDR, (void*)&ifr) < 0 ) {
      close(fd);
      perror("SIOCSIFADDR");
     return -1;
    }

    close(fd);
    return 0;
  }
  return -1;
}

void tundev_close(int fd) {
  close(fd);
}

*/
import "C"

import (
  "errors"
)

type tunDev C.int

func newTun(ifname, addr, dstaddr string, mtu int) (t tunDev, err error) {
  fd := C.tundev_open(C.CString(ifname))

  if fd == -1 {
    err = errors.New("cannot open tun interface")
  } else {
    if C.tundev_up(C.CString(ifname), C.CString(addr), C.CString(dstaddr), C.int(mtu)) < C.int(0) {
      err = errors.New("cannot put up interface")
    } else {
      return tunDev(fd), nil
    }
  }
  return -1, err
}

func (d tunDev) Close() {
  C.tundev_close(C.int(d))
}
<|endoftext|>


<commit_before>package model

import (
	"fmt"

	"github.com/Konstantin8105/GoFea/dof"
	"github.com/Konstantin8105/GoFea/element"
	"github.com/Konstantin8105/GoFea/finiteElement"
	"github.com/Konstantin8105/GoFea/utils"
	"github.com/Konstantin8105/GoLinAlg/linAlg"
	"github.com/Konstantin8105/GoLinAlg/linAlg/solver"
)

// Solve - solving finite element
func (m *Dim2) Solve() (err error) {

	for caseNumber := 0; caseNumber < len(m.forceCases); caseNumber++ {

		// TODO : check everything
		// TODO : sort  everything
		// TODO : compress loads by number

		// Generate degree of freedom in global system
		var degreeGlobal []dof.AxeNumber
		dofSystem := dof.NewBeam(m.beams, dof.Dim2d)
		for _, beam := range m.beams {
			fe := m.getBeamFiniteElement(beam.Index)
			_, degreeLocal := finiteElement.GetStiffinerGlobalK(fe, &dofSystem, finiteElement.WithoutZeroStiffiner)
			degreeGlobal = append(degreeGlobal, degreeLocal...)
		}
		{
			is := dof.ConvertToInt(degreeGlobal)
			utils.UniqueInt(&is)
			degreeGlobal = dof.ConvertToAxe(is)
		}

		// Create convertor index to axe
		mapIndex := dof.NewMapIndex(&degreeGlobal)

		// Generate global stiffiner matrix [Ko]
		stiffinerKGlobal := m.convertFromLocalToGlobalSystem(&degreeGlobal, &dofSystem, &mapIndex, finiteElement.GetStiffinerGlobalK)

		// Create load vector
		loads := linAlg.NewMatrix64bySize(len(degreeGlobal), 1)
		for _, node := range m.forceCases[caseNumber].nodeForces {
			for _, inx := range node.pointIndexes {
				d := dofSystem.GetDoF(inx)
				if node.nodeForce.Fx != 0.0 {
					h, err := mapIndex.GetByAxe(d[0])
					if err == nil {
						loads.Set(h, 0, node.nodeForce.Fx)
					}
				}
				if node.nodeForce.Fy != 0.0 {
					h, err := mapIndex.GetByAxe(d[1])
					if err == nil {
						loads.Set(h, 0, node.nodeForce.Fy)
					}
				}
				if node.nodeForce.M != 0.0 {
					h, err := mapIndex.GetByAxe(d[2])
					if err == nil {
						loads.Set(h, 0, node.nodeForce.M)
					}
				}
			}
		}

		// Create array degree for support
		// and modify the global stiffiner matrix
		// and load vector
		for _, sup := range m.supports {
			for _, inx := range sup.pointIndexes {
				d := dofSystem.GetDoF(inx)
				var result []dof.AxeNumber
				if sup.support.Dx == true {
					result = append(result, d[0])
				}
				if sup.support.Dy == true {
					result = append(result, d[1])
				}
				if sup.support.M == true {
					result = append(result, d[2])
				}
				// modify stiffiner matrix for correct
				// adding support
				for i := 0; i < len(result); i++ {
					g, err := mapIndex.GetByAxe(result[i])
					if err != nil {
						continue
					}
					for j := 0; j < len(degreeGlobal); j++ {
						h, err := mapIndex.GetByAxe(degreeGlobal[j])
						if err != nil {
							continue
						}
						stiffinerKGlobal.Set(g, h, 0.0)
						stiffinerKGlobal.Set(h, g, 0.0)
					}
					stiffinerKGlobal.Set(g, g, 1.0)
					// modify load vector on support
					loads.Set(g, 0, 0.0)
				}
			}
		}
		//fmt.Println("degreeGlobal = ", degreeGlobal)
		//fmt.Printf("K global = \n%s\n", stiffinerKGlobal)
		//fmt.Printf("Load vector = \n%s\n", loads)

		// Solving system of linear equations for finding
		// the displacement in points in global system
		// TODO: if you have nonlinear elements, then we can use
		// TODO: one global stiffiner matrix for all cases
		lu := solver.NewLUsolver(stiffinerKGlobal)
		x := lu.Solve(loads)
		// TODO: rename global vector of displacement

		fmt.Printf("Global displacement = \n%s\n", x)
		fmt.Println("degreeGlobal = ", degreeGlobal)
		for _, beam := range m.beams {
			fe := m.getBeamFiniteElement(beam.Index)
			klocal, degreeLocal := finiteElement.GetStiffinerGlobalK(fe, &dofSystem, finiteElement.FullInformation)
			fmt.Println("=============")
			fmt.Println("klocalGlobal = ", klocal)
			fmt.Println("degreeLocal = ", degreeLocal)
			globalDisplacement := make([]float64, len(degreeLocal))
			for i := 0; i < len(globalDisplacement); i++ {
				found := false
				for j := 0; j < len(degreeGlobal); j++ {
					if degreeLocal[i] == degreeGlobal[j] {
						found = true
						globalDisplacement[i] = x.Get(j, 0)
						break
					}
				}
				if !found {
					panic("Cannot found dof - MAY BE PINNED. Check")
				}
			}
			fmt.Println("globalDisplacement = ", globalDisplacement)

			t := linAlg.NewMatrix64bySize(10, 10)
			fe.GetCoordinateTransformation(&t)
			fmt.Println("tr.glo --", t)

			// Zo = T_t * Z
			var localDisplacement []float64
			for i := 0; i < t.GetRowSize(); i++ {
				sum := 0.0
				for j := 0; j < t.GetColumnSize(); j++ {
					sum += t.Get(i, j) * globalDisplacement[j]
				}
				localDisplacement = append(localDisplacement, sum)
			}

			fmt.Println("localDisplacement = ", localDisplacement)

			kk := linAlg.NewMatrix64bySize(10, 10)
			fe.GetStiffinerK(&kk)
			fmt.Println("klocalll -->", kk)

			var localForce []float64
			for i := 0; i < kk.GetRowSize(); i++ {
				sum := 0.0
				for j := 0; j < kk.GetRowSize(); j++ {
					sum += kk.Get(i, j) * localDisplacement[j]
				}
				localForce = append(localForce, sum)
			}
			fmt.Println("localForce = ", localForce)
		}

		//TODO: can calculated in parallel local force

		// Generate global mass matrix [Mo]
		massGlobal := m.convertFromLocalToGlobalSystem(&degreeGlobal, &dofSystem, &mapIndex, finiteElement.GetGlobalMass)
		fmt.Println("GlobalMass = ", massGlobal)

		// Calculate matrix [H] = [Ko]^-1 * [Mo]
		if stiffinerKGlobal.GetRowSize() != stiffinerKGlobal.GetColumnSize() {
			panic("Not correct size of global stiffiner matrix")
		}
		n := stiffinerKGlobal.GetRowSize()
		Ho := linAlg.NewMatrix64bySize(n, n)
		buffer := linAlg.NewMatrix64bySize(n, 1)
		for i := 0; i < n; i++ {
			// Create vertical vector from [Mo]
			for j := 0; j < n; j++ {
				buffer.Set(j, 0, massGlobal.Get(j, i))
			}
			// Calculation
			result := lu.Solve(buffer)
			// Add vector to [Ho]
			for j := 0; j < n; j++ {
				Ho.Set(j, i, result.Get(j, 0))
			}
		}
		fmt.Println("[Ho] = ", Ho)

		// Calculation of natural frequency
		eigen := solver.NewEigen(Ho)
		fmt.Println("lambda       = ", eigen.GetRealEigenvalues())
		fmt.Println("eigenvectors = ", eigen.GetV())
	}

	return nil
}

func (m *Dim2) getBeamFiniteElement(inx element.BeamIndex) (fe finiteElement.FiniteElementer) {
	material, err := m.getMaterial(inx)
	if err != nil {
		panic(fmt.Errorf("Cannot found material for beam #%v. Error = %v", inx, err))
	}
	shape, err := m.getShape(inx)
	if err != nil {
		panic(fmt.Errorf("Cannot found shape for beam #%v. Error = %v", inx, err))
	}
	coord, err := m.getCoordinate(inx)
	if err != nil {
		panic(fmt.Errorf("Cannot calculate lenght for beam #%v. Error = %v", inx, err))
	}
	if m.isTruss(inx) {
		f := finiteElement.TrussDim2{
			Material: material,
			Shape:    shape,
			Points:   coord,
		}
		return &f
	} /* else {
		fe := finiteElement.BeamDim2{
			Material: material,
			Shape:    shape,
			Points:   coord,
		}
		err = fe.GetStiffinerK(&buffer)
		if err != nil {
			return err
		}
	}*/
	return nil
}

func (m *Dim2) convertFromLocalToGlobalSystem(degreeGlobal *[]dof.AxeNumber, dofSystem *dof.DoF, mapIndex *dof.MapIndex, f func(finiteElement.FiniteElementer, *dof.DoF, finiteElement.Information) (linAlg.Matrix64, []dof.AxeNumber)) linAlg.Matrix64 {

	globalResult := linAlg.NewMatrix64bySize(len(*degreeGlobal), len(*degreeGlobal))
	for _, beam := range m.beams {
		fe := m.getBeamFiniteElement(beam.Index)
		klocal, degreeLocal := f(fe, dofSystem, finiteElement.WithoutZeroStiffiner)
		// Add local stiffiner matrix to global matrix
		for i := 0; i < len(degreeLocal); i++ {
			g, err := mapIndex.GetByAxe(degreeLocal[i])
			if err != nil {
				continue
			}
			for j := 0; j < len(degreeLocal); j++ {
				h, err := mapIndex.GetByAxe(degreeLocal[j])
				if err != nil {
					continue
				}
				globalResult.Set(g, h, globalResult.Get(g, h)+klocal.Get(i, j))
			}
		}
	}
	return globalResult
}
<commit_msg>Natural frequency prelimnary Ok<commit_after>package model

import (
	"fmt"
	"math"

	"github.com/Konstantin8105/GoFea/dof"
	"github.com/Konstantin8105/GoFea/element"
	"github.com/Konstantin8105/GoFea/finiteElement"
	"github.com/Konstantin8105/GoFea/utils"
	"github.com/Konstantin8105/GoLinAlg/linAlg"
	"github.com/Konstantin8105/GoLinAlg/linAlg/solver"
)

// Solve - solving finite element
func (m *Dim2) Solve() (err error) {

	for caseNumber := 0; caseNumber < len(m.forceCases); caseNumber++ {

		// TODO : check everything
		// TODO : sort  everything
		// TODO : compress loads by number

		// Generate degree of freedom in global system
		var degreeGlobal []dof.AxeNumber
		dofSystem := dof.NewBeam(m.beams, dof.Dim2d)
		for _, beam := range m.beams {
			fe := m.getBeamFiniteElement(beam.Index)
			_, degreeLocal := finiteElement.GetStiffinerGlobalK(fe, &dofSystem, finiteElement.WithoutZeroStiffiner)
			degreeGlobal = append(degreeGlobal, degreeLocal...)
		}
		{
			is := dof.ConvertToInt(degreeGlobal)
			utils.UniqueInt(&is)
			degreeGlobal = dof.ConvertToAxe(is)
		}

		// Create convertor index to axe
		mapIndex := dof.NewMapIndex(&degreeGlobal)

		// Generate global stiffiner matrix [Ko]
		stiffinerKGlobal := m.convertFromLocalToGlobalSystem(&degreeGlobal, &dofSystem, &mapIndex, finiteElement.GetStiffinerGlobalK)

		// Create load vector
		loads := linAlg.NewMatrix64bySize(len(degreeGlobal), 1)
		for _, node := range m.forceCases[caseNumber].nodeForces {
			for _, inx := range node.pointIndexes {
				d := dofSystem.GetDoF(inx)
				if node.nodeForce.Fx != 0.0 {
					h, err := mapIndex.GetByAxe(d[0])
					if err == nil {
						loads.Set(h, 0, node.nodeForce.Fx)
					}
				}
				if node.nodeForce.Fy != 0.0 {
					h, err := mapIndex.GetByAxe(d[1])
					if err == nil {
						loads.Set(h, 0, node.nodeForce.Fy)
					}
				}
				if node.nodeForce.M != 0.0 {
					h, err := mapIndex.GetByAxe(d[2])
					if err == nil {
						loads.Set(h, 0, node.nodeForce.M)
					}
				}
			}
		}

		// Create array degree for support
		// and modify the global stiffiner matrix
		// and load vector
		for _, sup := range m.supports {
			for _, inx := range sup.pointIndexes {
				d := dofSystem.GetDoF(inx)
				var result []dof.AxeNumber
				if sup.support.Dx == true {
					result = append(result, d[0])
				}
				if sup.support.Dy == true {
					result = append(result, d[1])
				}
				if sup.support.M == true {
					result = append(result, d[2])
				}
				// modify stiffiner matrix for correct
				// adding support
				for i := 0; i < len(result); i++ {
					g, err := mapIndex.GetByAxe(result[i])
					if err != nil {
						continue
					}
					for j := 0; j < len(degreeGlobal); j++ {
						h, err := mapIndex.GetByAxe(degreeGlobal[j])
						if err != nil {
							continue
						}
						stiffinerKGlobal.Set(g, h, 0.0)
						stiffinerKGlobal.Set(h, g, 0.0)
					}
					stiffinerKGlobal.Set(g, g, 1.0)
					// modify load vector on support
					loads.Set(g, 0, 0.0)
				}
			}
		}
		//fmt.Println("degreeGlobal = ", degreeGlobal)
		//fmt.Printf("K global = \n%s\n", stiffinerKGlobal)
		//fmt.Printf("Load vector = \n%s\n", loads)

		// Solving system of linear equations for finding
		// the displacement in points in global system
		// TODO: if you have nonlinear elements, then we can use
		// TODO: one global stiffiner matrix for all cases
		lu := solver.NewLUsolver(stiffinerKGlobal)
		x := lu.Solve(loads)
		// TODO: rename global vector of displacement

		fmt.Printf("Global displacement = \n%s\n", x)
		fmt.Println("degreeGlobal = ", degreeGlobal)
		for _, beam := range m.beams {
			fe := m.getBeamFiniteElement(beam.Index)
			klocal, degreeLocal := finiteElement.GetStiffinerGlobalK(fe, &dofSystem, finiteElement.FullInformation)
			fmt.Println("=============")
			fmt.Println("klocalGlobal = ", klocal)
			fmt.Println("degreeLocal = ", degreeLocal)
			globalDisplacement := make([]float64, len(degreeLocal))
			for i := 0; i < len(globalDisplacement); i++ {
				found := false
				for j := 0; j < len(degreeGlobal); j++ {
					if degreeLocal[i] == degreeGlobal[j] {
						found = true
						globalDisplacement[i] = x.Get(j, 0)
						break
					}
				}
				if !found {
					panic("Cannot found dof - MAY BE PINNED. Check")
				}
			}
			fmt.Println("globalDisplacement = ", globalDisplacement)

			t := linAlg.NewMatrix64bySize(10, 10)
			fe.GetCoordinateTransformation(&t)
			fmt.Println("tr.glo --", t)

			// Zo = T_t * Z
			var localDisplacement []float64
			for i := 0; i < t.GetRowSize(); i++ {
				sum := 0.0
				for j := 0; j < t.GetColumnSize(); j++ {
					sum += t.Get(i, j) * globalDisplacement[j]
				}
				localDisplacement = append(localDisplacement, sum)
			}

			fmt.Println("localDisplacement = ", localDisplacement)

			kk := linAlg.NewMatrix64bySize(10, 10)
			fe.GetStiffinerK(&kk)
			fmt.Println("klocalll -->", kk)

			var localForce []float64
			for i := 0; i < kk.GetRowSize(); i++ {
				sum := 0.0
				for j := 0; j < kk.GetRowSize(); j++ {
					sum += kk.Get(i, j) * localDisplacement[j]
				}
				localForce = append(localForce, sum)
			}
			fmt.Println("localForce = ", localForce)
		}

		//TODO: can calculated in parallel local force

		// Generate global mass matrix [Mo]
		n := stiffinerKGlobal.GetRowSize()
		massGlobal := m.convertFromLocalToGlobalSystem(&degreeGlobal, &dofSystem, &mapIndex, finiteElement.GetGlobalMass)
		// m.convertFromLocalToGlobalSystem(&degreeGlobal, &dofSystem, &mapIndex, finiteElement.GetGlobalMass)
		//  linAlg.NewMatrix64bySize(n, n)

		// TODO: Add to matrix mass the nodal mass
		for _, node := range m.forceCases[caseNumber].nodeForces {
			for _, inx := range node.pointIndexes {
				d := dofSystem.GetDoF(inx)
				if node.nodeForce.Fx != 0.0 {
					h, err := mapIndex.GetByAxe(d[0])
					if err == nil {
						massGlobal.Set(h, h, massGlobal.Get(h, h)+math.Abs(node.nodeForce.Fx))
					}
				}
				if node.nodeForce.Fy != 0.0 {
					h, err := mapIndex.GetByAxe(d[1])
					if err == nil {
						massGlobal.Set(h, h, massGlobal.Get(h, h)+math.Abs(node.nodeForce.Fy))
					}
				}
				//if node.nodeForce.M != 0.0 {
				//	h, err := mapIndex.GetByAxe(d[2])
				//	if err == nil {
				//		massGlobal.Set(h, h, massGlobal.Get(h, h)+math.Abs(node.nodeForce.M))
				//		fmt.Println("Add M to mass")
				//	}
				//}
			}
		}

		//TODO: CHECKUING GRAVITY TO MATRIX MASS
		for i := 0; i < massGlobal.GetRowSize(); i++ {
			for j := 0; j < massGlobal.GetColumnSize(); j++ {
				massGlobal.Set(i, j, massGlobal.Get(i, j)/9.806)
			}
		}
		// TODO: ADD to mass WITH OR WITOUT SELFWEIGHT

		// Calculate matrix [H] = [Ko]^-1 * [Mo]
		if stiffinerKGlobal.GetRowSize() != stiffinerKGlobal.GetColumnSize() {
			panic("Not correct size of global stiffiner matrix")
		}
		fmt.Println("GlobalMass = ", massGlobal)
		Ho := linAlg.NewMatrix64bySize(n, n)
		buffer := linAlg.NewMatrix64bySize(n, 1)
		for i := 0; i < n; i++ {
			// Create vertical vector from [Mo]
			for j := 0; j < n; j++ {
				buffer.Set(j, 0, massGlobal.Get(j, i))
			}
			// Calculation
			result := lu.Solve(buffer)
			// Add vector to [Ho]
			for j := 0; j < n; j++ {
				Ho.Set(j, i, result.Get(j, 0))
			}
		}
		fmt.Println("[Ho] = ", Ho)

		// Calculation of natural frequency
		eigen := solver.NewEigen(Ho)
		fmt.Println("lambda       = ", eigen.GetRealEigenvalues())
		fmt.Println("eigenvectors = ", eigen.GetV())
		fmt.Println("getD = ", eigen.GetD())

		value := eigen.GetRealEigenvalues()
		for _, v := range value {
			fmt.Printf("f = %.3v Hz\n", math.Sqrt(1.0/v)/2.0/math.Pi)
		}
	}

	return nil
}

func (m *Dim2) getBeamFiniteElement(inx element.BeamIndex) (fe finiteElement.FiniteElementer) {
	material, err := m.getMaterial(inx)
	if err != nil {
		panic(fmt.Errorf("Cannot found material for beam #%v. Error = %v", inx, err))
	}
	shape, err := m.getShape(inx)
	if err != nil {
		panic(fmt.Errorf("Cannot found shape for beam #%v. Error = %v", inx, err))
	}
	coord, err := m.getCoordinate(inx)
	if err != nil {
		panic(fmt.Errorf("Cannot calculate lenght for beam #%v. Error = %v", inx, err))
	}
	if m.isTruss(inx) {
		f := finiteElement.TrussDim2{
			Material: material,
			Shape:    shape,
			Points:   coord,
		}
		return &f
	} /* else {
		fe := finiteElement.BeamDim2{
			Material: material,
			Shape:    shape,
			Points:   coord,
		}
		err = fe.GetStiffinerK(&buffer)
		if err != nil {
			return err
		}
	}*/
	return nil
}

func (m *Dim2) convertFromLocalToGlobalSystem(degreeGlobal *[]dof.AxeNumber, dofSystem *dof.DoF, mapIndex *dof.MapIndex, f func(finiteElement.FiniteElementer, *dof.DoF, finiteElement.Information) (linAlg.Matrix64, []dof.AxeNumber)) linAlg.Matrix64 {

	globalResult := linAlg.NewMatrix64bySize(len(*degreeGlobal), len(*degreeGlobal))
	for _, beam := range m.beams {
		fe := m.getBeamFiniteElement(beam.Index)
		klocal, degreeLocal := f(fe, dofSystem, finiteElement.WithoutZeroStiffiner)
		// Add local stiffiner matrix to global matrix
		for i := 0; i < len(degreeLocal); i++ {
			g, err := mapIndex.GetByAxe(degreeLocal[i])
			if err != nil {
				continue
			}
			for j := 0; j < len(degreeLocal); j++ {
				h, err := mapIndex.GetByAxe(degreeLocal[j])
				if err != nil {
					continue
				}
				globalResult.Set(g, h, globalResult.Get(g, h)+klocal.Get(i, j))
			}
		}
	}
	return globalResult
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package txserializer provides the vttablet hot row protection.
// See the TxSerializer struct for details.
package txserializer

import (
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/vt/logutil"
	"github.com/youtube/vitess/go/vt/vterrors"

	vtrpcpb "github.com/youtube/vitess/go/vt/proto/vtrpc"
)

var (
	// waits stores how many times a transaction was queued because another
	// transaction was already in flight for the same row (range).
	// The key of the map is the table name of the query.
	waits = stats.NewCounters("TxSerializerWaits")
	// waitsDryRun is similar as "waits": In dry-run mode it records how many
	// transactions would have been queued.
	// The key of the map is the table name of the query.
	waitsDryRun = stats.NewCounters("TxSerializerWaitsDryRun")

	// queueExceeded counts per table how many transactions were rejected because
	// the max queue size per row (range) was exceeded.
	queueExceeded = stats.NewCounters("TxSerializerQueueExceeded")
	// queueExceededDryRun counts in dry-run mode how many transactions would have
	// been rejected due to exceeding the max queue size per row (range).
	queueExceededDryRun = stats.NewCounters("TxSerializerQueueExceededDryRun")
	// globalQueueExceeded is the same as queueExceeded but for the global queue.
	globalQueueExceeded       = stats.NewInt("TxSerializerGlobalQueueExceeded")
	globalQueueExceededDryRun = stats.NewInt("TxSerializerGlobalQueueExceededDryRun")
)

// TxSerializer serializes incoming transactions which target the same row range
// i.e. table name and WHERE clause are identical.
// Additional transactions are queued and woken up in arrival order.
//
// This implementation has some parallels to the sync2.Consolidator class.
// However, there are many substantial differences:
// - Results are not shared between queued transactions.
// - Only one waiting transaction and not all are notified when the current one
//   has finished.
// - Waiting transactions are woken up in FIFO order.
// - Waiting transactions are unblocked if their context is done.
// - Both the local queue (per row range) and global queue (whole process) are
//   limited to avoid that queued transactions can consume the full capacity
//   of vttablet. This is important if the capaciy is finite. For example, the
//   number of RPCs in flight could be limited by the RPC subsystem.
type TxSerializer struct {
	*sync2.ConsolidatorCache

	// Immutable fields.
	dryRun                 bool
	maxQueueSize           int
	maxGlobalQueueSize     int
	concurrentTransactions int

	log                          *logutil.ThrottledLogger
	logDryRun                    *logutil.ThrottledLogger
	logWaitsDryRun               *logutil.ThrottledLogger
	logQueueExceededDryRun       *logutil.ThrottledLogger
	logGlobalQueueExceededDryRun *logutil.ThrottledLogger

	mu         sync.Mutex
	queues     map[string]*queue
	globalSize int
}

// New returns a TxSerializer object.
func New(dryRun bool, maxQueueSize, maxGlobalQueueSize, concurrentTransactions int) *TxSerializer {
	return &TxSerializer{
		ConsolidatorCache:      sync2.NewConsolidatorCache(1000),
		dryRun:                 dryRun,
		maxQueueSize:           maxQueueSize,
		maxGlobalQueueSize:     maxGlobalQueueSize,
		concurrentTransactions: concurrentTransactions,
		log:                          logutil.NewThrottledLogger("HotRowProtection", 5*time.Second),
		logDryRun:                    logutil.NewThrottledLogger("HotRowProtection DryRun", 5*time.Second),
		logWaitsDryRun:               logutil.NewThrottledLogger("HotRowProtection Waits DryRun", 5*time.Second),
		logQueueExceededDryRun:       logutil.NewThrottledLogger("HotRowProtection QueueExceeded DryRun", 5*time.Second),
		logGlobalQueueExceededDryRun: logutil.NewThrottledLogger("HotRowProtection GlobalQueueExceeded DryRun", 5*time.Second),
		queues: make(map[string]*queue),
	}
}

// DoneFunc is returned by Wait() and must be called by the caller.
type DoneFunc func()

// Wait blocks if another transaction for the same range is already in flight.
// It returns when this transaction has its turn.
// "done" is != nil if err == nil and must be called once the transaction is
// done and the next waiting transaction can be unblocked.
// "waited" is true if Wait() had to wait for other transactions.
// "err" is not nil if a) the context is done or b) a queue limit was reached.
func (t *TxSerializer) Wait(ctx context.Context, key, table string) (done DoneFunc, waited bool, err error) {
	t.mu.Lock()
	defer t.mu.Unlock()

	waited, err = t.lockLocked(ctx, key, table)
	if err != nil {
		if waited {
			// Waiting failed early e.g. due a canceled context and we did NOT get the
			// token. Call "done" now because we don't return it to the caller.
			t.unlockLocked(key, false /* returnToken */)
		}
		return nil, waited, err
	}
	return func() { t.unlock(key) }, waited, nil
}

// lockLocked queues this transaction. It will unblock immediately if this
// transaction is the first in the queue or when it got a token.
// The method has the suffix "Locked" to clarify that "t.mu" must be locked.
func (t *TxSerializer) lockLocked(ctx context.Context, key, table string) (bool, error) {
	q, ok := t.queues[key]
	if !ok {
		// First transaction in the queue i.e. we don't wait and return immediately.
		t.queues[key] = newQueue(t.concurrentTransactions)
		t.globalSize++
		return false, nil
	}

	if t.globalSize >= t.maxGlobalQueueSize {
		if t.dryRun {
			globalQueueExceededDryRun.Add(1)
			t.logGlobalQueueExceededDryRun.Warningf("Would have rejected BeginExecute RPC because there are too many queued transactions (%d >= %d)", t.globalSize, t.maxGlobalQueueSize)
		} else {
			globalQueueExceeded.Add(1)
			return false, vterrors.Errorf(vtrpcpb.Code_RESOURCE_EXHAUSTED,
				"hot row protection: too many queued transactions (%d >= %d)", t.globalSize, t.maxGlobalQueueSize)
		}
	}

	if q.size >= t.maxQueueSize {
		if t.dryRun {
			queueExceededDryRun.Add(table, 1)
			t.logQueueExceededDryRun.Warningf("Would have rejected BeginExecute RPC because there are too many queued transactions (%d >= %d) for the same row (table + WHERE clause: '%v')", q.size, t.maxQueueSize, key)
		} else {
			queueExceeded.Add(table, 1)
			return false, vterrors.Errorf(vtrpcpb.Code_RESOURCE_EXHAUSTED,
				"hot row protection: too many queued transactions (%d >= %d) for the same row (table + WHERE clause: '%v')", q.size, t.maxQueueSize, key)
		}
	}

	t.globalSize++
	q.size++
	q.count++
	if q.size == 2 && q.max == 1 {
		// Hot row detected: A second, concurrent transaction is seen for the first
		// time.
		// The first transaction already holds the first token and will return it
		// when it's done and calls "unlock".
		// If more tokens are allowed, add them now. (We delayed adding the tokens
		// until now as an optimization for the default case when there is no hot
		// row.)
		additionalTokens := t.concurrentTransactions - 1
		for i := 1; i <= additionalTokens; i++ {
			q.tokens <- struct{}{}
		}

		// Include first transaction in the count at /debug/hotrows. (It was not
		// recorded on purpose because it did not wait.)
		t.Record(key)
	}
	if q.size > q.max {
		q.max = q.size
	}
	// Publish the number of waits at /debug/hotrows.
	t.Record(key)

	if t.dryRun {
		waitsDryRun.Add(table, 1)
		t.logWaitsDryRun.Warningf("Would have queued BeginExecute RPC for row (range): '%v' because another transaction to the same range is already in progress.", key)
		return false, nil
	}

	// Unlock before the wait and relock before returning because our caller
	// Wait() holds the lock and assumes it still has it.
	t.mu.Unlock()
	defer t.mu.Lock()

	// Non-blocking read of a token.
	select {
	case <-q.tokens:
		// Return waited=false because a token was immediately available.
		return false, nil
	default:
	}

	// Wait for the next available token.
	waits.Add(table, 1)
	select {
	case <-q.tokens:
		return true, nil
	case <-ctx.Done():
		return true, ctx.Err()
	}
}

func (t *TxSerializer) unlock(key string) {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.unlockLocked(key, true)
}

func (t *TxSerializer) unlockLocked(key string, returnToken bool) {
	q := t.queues[key]
	q.size--
	t.globalSize--
	if q.size == 0 {
		// This is the last transaction in flight.
		delete(t.queues, key)

		if q.max > 1 {
			if t.dryRun {
				t.logDryRun.Infof("%v simultaneous transactions (%v in total) for the same row range (%v) would have been queued.", q.max, q.count, key)
			} else {
				t.log.Infof("%v simultaneous transactions (%v in total) for the same row range (%v) were queued.", q.max, q.count, key)
			}
		}

		// Return early because the queue "q" for this "key" will not be used any
		// more.
		return
	}

	// Return token to queue. Wakes up the next queued transaction.
	if !t.dryRun && returnToken {
		q.tokens <- struct{}{}
	}
}

// Pending returns the number of queued transactions (including the ones which
// are currently in flight.)
func (t *TxSerializer) Pending(key string) int {
	t.mu.Lock()
	defer t.mu.Unlock()

	q, ok := t.queues[key]
	if !ok {
		return 0
	}
	return q.size
}

// queue reprents the local queue for a particular row (range).
//
// Note that we don't use a dedicated queue structure for all waiting
// transactions. Instead, we leverage that Go routines waiting for a channel
// are woken up in the order they are queued up. The "tokens" field is said
// channel which has n elements, "tokens", for the number of concurrent
// transactions which can access the tx pool. All queued transactions are
// competing for these tokens.
type queue struct {
	// NOTE: The following fields are guarded by TxSerializer.mu.
	// size counts how many transactions are currently queued/in flight (includes
	// the transactions which are not waiting.)
	size int
	// count is the same as "size", but never gets decremented.
	count int
	// max is the max of "size", i.e. the maximum number of transactions which
	// were simultaneously queued for the same row range.
	max int

	// tokens holds one element for each allowed tx pool slot. E.g. if the channel
	// has a size of 1, only one transaction at a time is allowed through.
	tokens chan struct{}
}

func newQueue(concurrentTransactions int) *queue {
	return &queue{
		size:  1,
		count: 1,
		max:   1,
		// The first available token is not added as an optimization because the
		// caller would immediately remove it anyway.
		// If additional tokens are allowed, we delay adding them until the row
		// range becomes hot and a second in-flight transaction occurs.
		tokens: make(chan struct{}, concurrentTransactions),
	}
}
<commit_msg>vttablet: Hot Row Protection: Reverse the "tokens" queue for the allowed concurrent transactions from pull to push.<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package txserializer provides the vttablet hot row protection.
// See the TxSerializer struct for details.
package txserializer

import (
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/vt/logutil"
	"github.com/youtube/vitess/go/vt/vterrors"

	vtrpcpb "github.com/youtube/vitess/go/vt/proto/vtrpc"
)

var (
	// waits stores how many times a transaction was queued because another
	// transaction was already in flight for the same row (range).
	// The key of the map is the table name of the query.
	waits = stats.NewCounters("TxSerializerWaits")
	// waitsDryRun is similar as "waits": In dry-run mode it records how many
	// transactions would have been queued.
	// The key of the map is the table name of the query.
	waitsDryRun = stats.NewCounters("TxSerializerWaitsDryRun")

	// queueExceeded counts per table how many transactions were rejected because
	// the max queue size per row (range) was exceeded.
	queueExceeded = stats.NewCounters("TxSerializerQueueExceeded")
	// queueExceededDryRun counts in dry-run mode how many transactions would have
	// been rejected due to exceeding the max queue size per row (range).
	queueExceededDryRun = stats.NewCounters("TxSerializerQueueExceededDryRun")
	// globalQueueExceeded is the same as queueExceeded but for the global queue.
	globalQueueExceeded       = stats.NewInt("TxSerializerGlobalQueueExceeded")
	globalQueueExceededDryRun = stats.NewInt("TxSerializerGlobalQueueExceededDryRun")
)

// TxSerializer serializes incoming transactions which target the same row range
// i.e. table name and WHERE clause are identical.
// Additional transactions are queued and woken up in arrival order.
//
// This implementation has some parallels to the sync2.Consolidator class.
// However, there are many substantial differences:
// - Results are not shared between queued transactions.
// - Only one waiting transaction and not all are notified when the current one
//   has finished.
// - Waiting transactions are woken up in FIFO order.
// - Waiting transactions are unblocked if their context is done.
// - Both the local queue (per row range) and global queue (whole process) are
//   limited to avoid that queued transactions can consume the full capacity
//   of vttablet. This is important if the capaciy is finite. For example, the
//   number of RPCs in flight could be limited by the RPC subsystem.
type TxSerializer struct {
	*sync2.ConsolidatorCache

	// Immutable fields.
	dryRun                 bool
	maxQueueSize           int
	maxGlobalQueueSize     int
	concurrentTransactions int

	log                          *logutil.ThrottledLogger
	logDryRun                    *logutil.ThrottledLogger
	logWaitsDryRun               *logutil.ThrottledLogger
	logQueueExceededDryRun       *logutil.ThrottledLogger
	logGlobalQueueExceededDryRun *logutil.ThrottledLogger

	mu         sync.Mutex
	queues     map[string]*queue
	globalSize int
}

// New returns a TxSerializer object.
func New(dryRun bool, maxQueueSize, maxGlobalQueueSize, concurrentTransactions int) *TxSerializer {
	return &TxSerializer{
		ConsolidatorCache:      sync2.NewConsolidatorCache(1000),
		dryRun:                 dryRun,
		maxQueueSize:           maxQueueSize,
		maxGlobalQueueSize:     maxGlobalQueueSize,
		concurrentTransactions: concurrentTransactions,
		log:                          logutil.NewThrottledLogger("HotRowProtection", 5*time.Second),
		logDryRun:                    logutil.NewThrottledLogger("HotRowProtection DryRun", 5*time.Second),
		logWaitsDryRun:               logutil.NewThrottledLogger("HotRowProtection Waits DryRun", 5*time.Second),
		logQueueExceededDryRun:       logutil.NewThrottledLogger("HotRowProtection QueueExceeded DryRun", 5*time.Second),
		logGlobalQueueExceededDryRun: logutil.NewThrottledLogger("HotRowProtection GlobalQueueExceeded DryRun", 5*time.Second),
		queues: make(map[string]*queue),
	}
}

// DoneFunc is returned by Wait() and must be called by the caller.
type DoneFunc func()

// Wait blocks if another transaction for the same range is already in flight.
// It returns when this transaction has its turn.
// "done" is != nil if err == nil and must be called once the transaction is
// done and the next waiting transaction can be unblocked.
// "waited" is true if Wait() had to wait for other transactions.
// "err" is not nil if a) the context is done or b) a queue limit was reached.
func (t *TxSerializer) Wait(ctx context.Context, key, table string) (done DoneFunc, waited bool, err error) {
	t.mu.Lock()
	defer t.mu.Unlock()

	waited, err = t.lockLocked(ctx, key, table)
	if err != nil {
		if waited {
			// Waiting failed early e.g. due a canceled context and we did NOT get the
			// slot. Call "done" now because we don't return it to the caller.
			t.unlockLocked(key, false /* returnSlot */)
		}
		return nil, waited, err
	}
	return func() { t.unlock(key) }, waited, nil
}

// lockLocked queues this transaction. It will unblock immediately if this
// transaction is the first in the queue or when it acquired a slot.
// The method has the suffix "Locked" to clarify that "t.mu" must be locked.
func (t *TxSerializer) lockLocked(ctx context.Context, key, table string) (bool, error) {
	q, ok := t.queues[key]
	if !ok {
		// First transaction in the queue i.e. we don't wait and return immediately.
		t.queues[key] = newQueueForFirstTransaction(t.concurrentTransactions)
		t.globalSize++
		return false, nil
	}

	if t.globalSize >= t.maxGlobalQueueSize {
		if t.dryRun {
			globalQueueExceededDryRun.Add(1)
			t.logGlobalQueueExceededDryRun.Warningf("Would have rejected BeginExecute RPC because there are too many queued transactions (%d >= %d)", t.globalSize, t.maxGlobalQueueSize)
		} else {
			globalQueueExceeded.Add(1)
			return false, vterrors.Errorf(vtrpcpb.Code_RESOURCE_EXHAUSTED,
				"hot row protection: too many queued transactions (%d >= %d)", t.globalSize, t.maxGlobalQueueSize)
		}
	}

	if q.size >= t.maxQueueSize {
		if t.dryRun {
			queueExceededDryRun.Add(table, 1)
			t.logQueueExceededDryRun.Warningf("Would have rejected BeginExecute RPC because there are too many queued transactions (%d >= %d) for the same row (table + WHERE clause: '%v')", q.size, t.maxQueueSize, key)
		} else {
			queueExceeded.Add(table, 1)
			return false, vterrors.Errorf(vtrpcpb.Code_RESOURCE_EXHAUSTED,
				"hot row protection: too many queued transactions (%d >= %d) for the same row (table + WHERE clause: '%v')", q.size, t.maxQueueSize, key)
		}
	}

	t.globalSize++
	q.size++
	q.count++
	if q.size == 2 && q.max == 1 {
		// Hot row detected: A second, concurrent transaction is seen for the first
		// time.

		// Include first transaction in the count at /debug/hotrows. (It was not
		// recorded on purpose because it did not wait.)
		t.Record(key)
	}
	if q.size > q.max {
		q.max = q.size
	}
	// Publish the number of waits at /debug/hotrows.
	t.Record(key)

	if t.dryRun {
		waitsDryRun.Add(table, 1)
		t.logWaitsDryRun.Warningf("Would have queued BeginExecute RPC for row (range): '%v' because another transaction to the same range is already in progress.", key)
		return false, nil
	}

	// Unlock before the wait and relock before returning because our caller
	// Wait() holds the lock and assumes it still has it.
	t.mu.Unlock()
	defer t.mu.Lock()

	// Non-blocking write attempt to get a slot.
	select {
	case q.availableSlots <- struct{}{}:
		// Return waited=false because a slot was immediately available.
		return false, nil
	default:
	}

	// Blocking wait for the next available slot.
	waits.Add(table, 1)
	select {
	case q.availableSlots <- struct{}{}:
		return true, nil
	case <-ctx.Done():
		return true, ctx.Err()
	}
}

func (t *TxSerializer) unlock(key string) {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.unlockLocked(key, true)
}

func (t *TxSerializer) unlockLocked(key string, returnSlot bool) {
	q := t.queues[key]
	q.size--
	t.globalSize--
	if q.size == 0 {
		// This is the last transaction in flight.
		delete(t.queues, key)

		if q.max > 1 {
			if t.dryRun {
				t.logDryRun.Infof("%v simultaneous transactions (%v in total) for the same row range (%v) would have been queued.", q.max, q.count, key)
			} else {
				t.log.Infof("%v simultaneous transactions (%v in total) for the same row range (%v) were queued.", q.max, q.count, key)
			}
		}

		// Return early because the queue "q" for this "key" will not be used any
		// more.
		return
	}

	// Give up slot by removing ourselves from the channel.
	// Wakes up the next queued transaction.
	if !t.dryRun && returnSlot {
		// This should never block.
		<-q.availableSlots
	}
}

// Pending returns the number of queued transactions (including the ones which
// are currently in flight.)
func (t *TxSerializer) Pending(key string) int {
	t.mu.Lock()
	defer t.mu.Unlock()

	q, ok := t.queues[key]
	if !ok {
		return 0
	}
	return q.size
}

// queue represents the local queue for a particular row (range).
//
// Note that we don't use a dedicated queue structure for all waiting
// transactions. Instead, we leverage that Go routines waiting for a channel
// are woken up in the order they are queued up. The "availableSlots" field is
// said channel which has n free slots (for the number of concurrent
// transactions which can access the tx pool). All queued transactions are
// competing for these slots and try to add themselves to the channel.
type queue struct {
	// NOTE: The following fields are guarded by TxSerializer.mu.
	// size counts how many transactions are currently queued/in flight (includes
	// the transactions which are not waiting.)
	size int
	// count is the same as "size", but never gets decremented.
	count int
	// max is the max of "size", i.e. the maximum number of transactions which
	// were simultaneously queued for the same row range.
	max int

	// availableSlots limits the number of concurrent transactions *per*
	// hot row (range). It holds one element for each allowed pending
	// transaction i.e. consumed tx pool slot. Consequently, if the channel
	// is full, subsequent transactions have to wait until they can place
	// their entry here.
	availableSlots chan struct{}
}

func newQueueForFirstTransaction(concurrentTransactions int) *queue {
	availableSlots := make(chan struct{}, concurrentTransactions)
	availableSlots <- struct{}{}
	return &queue{
		size:           1,
		count:          1,
		max:            1,
		availableSlots: availableSlots,
	}
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gtw_rtr_udp

import (
	"fmt"
	"github.com/thethingsnetwork/core"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/testing/mock_components"
	"github.com/thethingsnetwork/core/utils/log"
	. "github.com/thethingsnetwork/core/utils/testing"
	"net"
	"reflect"
	"testing"
	"time"
)

// ----- The adapter should be able to create a udp connection given a valid udp port
func TestListenOptions(t *testing.T) {
	tests := []listenOptionsTest{
		{uint(3000), nil},
		{uint(3000), core.ErrBadGatewayAddress}, // Already used now
		{int(14), core.ErrBadOptions},
		{"somethingElse", core.ErrBadOptions},
	}

	for _, test := range tests {
		test.run(t)
	}
}

type listenOptionsTest struct {
	options interface{}
	want    error
}

func (test listenOptionsTest) run(t *testing.T) {
	Desc(t, "Run Listen(router, %T %v)", test.options, test.options)
	adapter, router := generateAdapterAndRouter(t)
	got := adapter.Listen(router, test.options)
	test.check(t, got)
}

func (test listenOptionsTest) check(t *testing.T, got error) {
	// 1. Check if errors match
	if got != test.want {
		t.Errorf("expected {%v} to be {%v}\n", got, test.want)
		Ko(t)
		return
	}
	Ok(t)
}

// ----- The adapter should catch from the connection and forward valid semtech.Packet to the router
func TestPacketProcessing(t *testing.T) {
	tests := []packetProcessingTest{
		{generatePUSH_DATA(), 1, 3001},
		{[]byte{0x14, 0xff}, 0, 3003},
	}

	for _, test := range tests {
		test.run(t)
	}
}

type packetProcessingTest struct {
	in   interface{} // Could be raw []byte or plain semtech.Packet
	want uint        // 0 or 1 depending whether or not we expect a packet to has been transmitted
	port uint        // Probably temporary, just because goroutine and connection are still living between tests
}

func (test packetProcessingTest) run(t *testing.T) {
	Desc(t, "Simulate incoming datagram: %+v", test.in)
	adapter, router := generateAdapterAndRouter(t)
	conn, gateway := listen(adapter, router, test.port)
	send(conn, test.in)
	test.check(t, router, gateway) // Check whether or not packet has been forwarded to core router
}

func (test packetProcessingTest) check(t *testing.T, router core.Router, gateway core.GatewayAddress) {
	<-time.After(time.Millisecond * 50)
	mockRouter := router.(*mock_components.Router)

	// 1. Check if we expect a packet
	packets := mockRouter.Packets[gateway]
	if nb := len(packets); uint(nb) != test.want {
		t.Errorf("Received %d packets whereas expected %d", nb, test.want)
		Ko(t)
		return
	}

	// 2. If a packet was expected, check that it has been forwarded to the router
	if test.want > 0 {
		if !reflect.DeepEqual(packets[0], test.in) {
			t.Errorf("Expected %+v to match %+v", packets[0], test.in)
			Ko(t)
			return
		}
	}

	Ok(t)
}

// ----- Build Utilities
func generateAdapterAndRouter(t *testing.T) (Adapter, core.Router) {
	return Adapter{
		Logger: log.TestLogger{
			Tag: "Adapter",
			T:   t,
		},
	}, mock_components.NewRouter()
}

func generatePUSH_DATA() semtech.Packet {
	return semtech.Packet{
		Version:    semtech.VERSION,
		GatewayId:  []byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8},
		Token:      []byte{0x14, 0x42},
		Identifier: semtech.PUSH_DATA,
	}
}

// ----- Operate Utilities
func listen(adapter Adapter, router core.Router, port uint) (*net.UDPConn, core.GatewayAddress) {
	var err error

	// 1. Start the adapter watching procedure
	if err = adapter.Listen(router, port); err != nil {
		panic(err)
	}

	// 2. Create a UDP connection on the same port the adapter is listening
	var addr *net.UDPAddr
	var conn *net.UDPConn
	if addr, err = net.ResolveUDPAddr("udp", fmt.Sprintf("0.0.0.0:%d", port)); err != nil {
		panic(err)
	}
	if conn, err = net.DialUDP("udp", nil, addr); err != nil {
		panic(err)
	}

	// 3. Return the UDP connection and the corresponding simulated gateway address
	return conn, core.GatewayAddress(conn.LocalAddr().String())
}

func send(conn *net.UDPConn, data interface{}) {
	// 1. Send the packet or the raw sequence of bytes passed as argument
	var raw []byte
	var err error
	switch data.(type) {
	case []byte:
		raw = data.([]byte)
	case semtech.Packet:
		if raw, err = semtech.Marshal(data.(semtech.Packet)); err != nil {
			panic(err)
		}
	default:
		panic(fmt.Errorf("Unexpected data type to be send : %T", data))
	}
	if _, err = conn.Write(raw); err != nil {
		panic(err)
	}
}
<commit_msg>[router] Write test for Ack method<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gtw_rtr_udp

import (
	"fmt"
	"github.com/thethingsnetwork/core"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/testing/mock_components"
	"github.com/thethingsnetwork/core/utils/log"
	. "github.com/thethingsnetwork/core/utils/testing"
	"net"
	"reflect"
	"testing"
	"time"
)

// ----- The adapter should be able to create a udp connection given a valid udp port
func TestListenOptions(t *testing.T) {
	tests := []listenOptionsTest{
		{uint(3000), nil},
		{uint(3000), core.ErrBadGatewayAddress}, // Already used now
		{int(14), core.ErrBadOptions},
		{"somethingElse", core.ErrBadOptions},
	}

	for _, test := range tests {
		test.run(t)
	}
}

type listenOptionsTest struct {
	options interface{}
	want    error
}

func (test listenOptionsTest) run(t *testing.T) {
	Desc(t, "Run Listen(router, %T %v)", test.options, test.options)
	adapter, router := generateAdapterAndRouter(t)
	got := adapter.Listen(router, test.options)
	test.check(t, got)
}

func (test listenOptionsTest) check(t *testing.T, got error) {
	// 1. Check if errors match
	if got != test.want {
		t.Errorf("expected {%v} to be {%v}\n", got, test.want)
		Ko(t)
		return
	}
	Ok(t)
}

// ----- The adapter should catch from the connection and forward valid semtech.Packet to the router
func TestPacketProcessing(t *testing.T) {
	tests := []packetProcessingTest{
		{generatePUSH_DATA(), 1, 3001},
		{[]byte{0x14, 0xff}, 0, 3002},
	}

	for _, test := range tests {
		test.run(t)
	}
}

type packetProcessingTest struct {
	in   interface{} // Could be raw []byte or plain semtech.Packet
	want uint        // 0 or 1 depending whether or not we expect a packet to has been transmitted
	port uint        // Probably temporary, just because goroutine and connection are still living between tests
}

func (test packetProcessingTest) run(t *testing.T) {
	Desc(t, "Simulate incoming datagram: %+v", test.in)
	adapter, router := generateAdapterAndRouter(t)
	conn, gateway := createConnection(&adapter, router, test.port)
	sendDatagram(conn, test.in)
	test.check(t, router, gateway) // Check whether or not packet has been forwarded to core router
}

func (test packetProcessingTest) check(t *testing.T, router core.Router, gateway core.GatewayAddress) {
	<-time.After(time.Millisecond * 50)
	mockRouter := router.(*mock_components.Router)

	// 1. Check if we expect a packet
	packets := mockRouter.Packets[gateway]
	if nb := len(packets); uint(nb) != test.want {
		t.Errorf("Received %d packets whereas expected %d", nb, test.want)
		Ko(t)
		return
	}

	// 2. If a packet was expected, check that it has been forwarded to the router
	if test.want > 0 {
		if !reflect.DeepEqual(packets[0], test.in) {
			t.Errorf("Expected %+v to match %+v", packets[0], test.in)
			Ko(t)
			return
		}
	}

	Ok(t)
}

// ----- The adapter should send packet via back to an existing address through an opened connection
func TestSendAck(t *testing.T) {
	// 1. Initialize test data
	adapter, router := generateAdapterAndRouter(t)
	adapter2, router2 := generateAdapterAndRouter(t)
	conn, gateway := createConnection(&adapter, router, 3003)
	defer conn.Close()

	tests := []sendAckTest{
		{adapter, router, conn, gateway, generatePUSH_ACK(), nil},
		{adapter, router, conn, core.GatewayAddress("patate"), generatePUSH_ACK(), core.ErrBadGatewayAddress},
		{adapter, router, conn, gateway, semtech.Packet{}, core.ErrInvalidPacket},
		{adapter2, router2, nil, gateway, generatePUSH_ACK(), core.ErrMissingConnection},
	}

	// 2. Run tests
	for _, test := range tests {
		test.run(t)
	}
}

type sendAckTest struct {
	adapter Adapter
	router  core.Router
	conn    *net.UDPConn
	gateway core.GatewayAddress
	packet  semtech.Packet
	want    error
}

func (test sendAckTest) run(t *testing.T) {
	Desc(t, "Send ack packet %v to %v via %v", test.packet, test.conn, test.gateway)
	// Starts a goroutine that will redirect udp message to a dedicated channel
	cmsg := listenFromConnection(test.conn)
	defer close(cmsg)
	got := test.adapter.Ack(test.router, test.packet, test.gateway)
	test.check(t, cmsg, got) // Check the error or the packet if no error
}

func (test sendAckTest) check(t *testing.T, cmsg chan semtech.Packet, got error) {
	// 1. Check if an error was expected
	if test.want != nil {
		if got != test.want {
			t.Errorf("Expected %+v error but got %+v", test.want, got)
			Ko(t)
			return
		}
		Ok(t)
		return
	}

	// 2. Ensure the ack packet has been sent correctly
	packet := <-cmsg
	if !reflect.DeepEqual(test.packet, packet) {
		t.Errorf("Expected %+v to equal %+v", test.packet, packet)
		Ko(t)
		return
	}
	Ok(t)
}

// ----- Build Utilities
func generateAdapterAndRouter(t *testing.T) (Adapter, core.Router) {
	return Adapter{
		Logger: log.TestLogger{
			Tag: "Adapter",
			T:   t,
		},
	}, mock_components.NewRouter()
}

func generatePUSH_DATA() semtech.Packet {
	return semtech.Packet{
		Version:    semtech.VERSION,
		GatewayId:  []byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8},
		Token:      []byte{0x14, 0x42},
		Identifier: semtech.PUSH_DATA,
	}
}

func generatePUSH_ACK() semtech.Packet {
	return semtech.Packet{
		Version:    semtech.VERSION,
		Token:      []byte{0x14, 0x42},
		Identifier: semtech.PUSH_ACK,
	}
}

// ----- Operate Utilities
func createConnection(adapter *Adapter, router core.Router, port uint) (*net.UDPConn, core.GatewayAddress) {
	var err error

	// 1. Start the adapter watching procedure
	if err = adapter.Listen(router, port); err != nil {
		panic(err)
	}

	// 2. Create a UDP connection on the same port the adapter is listening
	var addr *net.UDPAddr
	var conn *net.UDPConn
	if addr, err = net.ResolveUDPAddr("udp", fmt.Sprintf("0.0.0.0:%d", port)); err != nil {
		panic(err)
	}
	if conn, err = net.DialUDP("udp", nil, addr); err != nil {
		panic(err)
	}

	// 3. Return the UDP connection and the corresponding simulated gateway address
	return conn, core.GatewayAddress(conn.LocalAddr().String())
}

func sendDatagram(conn *net.UDPConn, data interface{}) {
	// 1. Send the packet or the raw sequence of bytes passed as argument
	var raw []byte
	var err error
	switch data.(type) {
	case []byte:
		raw = data.([]byte)
	case semtech.Packet:
		if raw, err = semtech.Marshal(data.(semtech.Packet)); err != nil {
			panic(err)
		}
	default:
		panic(fmt.Errorf("Unexpected data type to be send : %T", data))
	}
	if _, err = conn.Write(raw); err != nil {
		panic(err)
	}
}

func listenFromConnection(conn *net.UDPConn) (cmsg chan semtech.Packet) {
	cmsg = make(chan semtech.Packet)

	// We won't listen on a nil connection
	if conn == nil {
		return
	}

	// Otherwise, wait for a packet
	go func() {
		for {
			buf := make([]byte, 128)
			n, err := conn.Read(buf)
			if err != nil {
				return
			}
			packet, err := semtech.Unmarshal(buf[:n])
			if err == nil {
				cmsg <- *packet
			}
		}
	}()

	return
}
<|endoftext|>


<commit_before>package parse

import (
	"bufio"
	"fmt"
	"os"
	"strings"
	"time"
)

func Solr(prefix *string) {
	timeLayout := "2006-01-02 15:04:05.000"

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		line := scanner.Text()
		isSelect := false
		var query string
		parts := strings.Split(line, " ")
		timeEnd := strings.Index(line, ",")
		t, _ := time.Parse(timeLayout, line[:timeEnd])

		for _, part := range parts {
			if part == "path=/select" {
				isSelect = true
			}

			if strings.HasPrefix(part, "params=") {
				qStart := strings.Index(part, "{")
				qEnd := strings.Index(part, "}")
				query = part[qStart+1 : qEnd]
			}
		}

		if isSelect {
			fmt.Printf("%f %s", float64(t.UnixNano())/1000000000, *prefix+"/select?"+query)
			fmt.Println()
		}
	}
}
<commit_msg>make solr log parsing more robust<commit_after>package parse

import (
	"bufio"
	"fmt"
	"net/url"
	"os"
	"strings"
	"time"
)

func Solr(prefix *string) {
	timeLayout := "2006-01-02 15:04:05.000"

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		line := scanner.Text()
		isSelect := false
		var query string
		parts := strings.Split(line, " ")
		splitByComma := strings.Split(line, ",")

		if !(len(splitByComma) > 2) {
			continue
		}

		if !(splitByComma[1] == "INFO") {
			continue
		}

		t, tErr := time.Parse(timeLayout, splitByComma[0])

		if tErr != nil {
			continue
		}

		for _, part := range parts {
			if part == "path=/select" {
				isSelect = true
			}

			if strings.HasPrefix(part, "params=") {
				qStart := strings.Index(part, "{")
				qEnd := strings.Index(part, "}")
				query = part[qStart+1 : qEnd]
			}
		}

		if isSelect {
			resultingUrl := *prefix + "/select?" + query
			_, err := url.Parse(resultingUrl)
			if err != nil {
				continue
			}
			fmt.Printf("%f %s", float64(t.UnixNano())/1000000000, resultingUrl)
			fmt.Println()
		}
	}
}
<|endoftext|>


<commit_before>package integration_test

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
	"time"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var bpDir string
var buildpackVersion string
var packagedBuildpack cutlass.VersionedBuildpackPackage

func init() {
	flag.StringVar(&buildpackVersion, "version", "", "version to use (builds if empty)")
	flag.BoolVar(&cutlass.Cached, "cached", true, "cached buildpack")
	flag.StringVar(&cutlass.DefaultMemory, "memory", "128M", "default memory for pushed apps")
	flag.StringVar(&cutlass.DefaultDisk, "disk", "256M", "default disk for pushed apps")
	flag.Parse()
}

var _ = SynchronizedBeforeSuite(func() []byte {
	// Run once
	if buildpackVersion == "" {
		packagedBuildpack, err := cutlass.PackageUniquelyVersionedBuildpack(os.Getenv("CF_STACK"), ApiHasStackAssociation())
		Expect(err).NotTo(HaveOccurred(), "failed to package buildpack")

		data, err := json.Marshal(packagedBuildpack)
		Expect(err).NotTo(HaveOccurred())
		return data
	}

	return []byte{}
}, func(data []byte) {
	// Run on all nodes
	var err error
	if len(data) > 0 {
		err = json.Unmarshal(data, &packagedBuildpack)
		Expect(err).NotTo(HaveOccurred())
		buildpackVersion = packagedBuildpack.Version
	}

	bpDir, err = cutlass.FindRoot()
	Expect(err).NotTo(HaveOccurred())

	Expect(cutlass.CopyCfHome()).To(Succeed())
	cutlass.SeedRandom()
	cutlass.DefaultStdoutStderr = GinkgoWriter
})

var _ = SynchronizedAfterSuite(func() {
	// Run on all nodes
}, func() {
	// Run once
	Expect(cutlass.RemovePackagedBuildpack(packagedBuildpack)).To(Succeed())
	Expect(cutlass.DeleteOrphanedRoutes()).To(Succeed())
})

func TestIntegration(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Integration Suite")
}

func PushAppAndConfirm(app *cutlass.App) {
	Expect(app.Push()).To(Succeed())
	Eventually(func() ([]string, error) { return app.InstanceStates() }, 20*time.Second).Should(Equal([]string{"RUNNING"}))
	Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())
}

func DestroyApp(app *cutlass.App) *cutlass.App {
	if app != nil {
		app.Destroy()
	}
	return nil
}

func ApiHasTask() bool {
	supported, err := cutlass.ApiGreaterThan("2.75.0")
	Expect(err).NotTo(HaveOccurred())
	return supported
}

func ApiHasMultiBuildpack() bool {
	supported, err := cutlass.ApiGreaterThan("2.90.0")
	Expect(err).NotTo(HaveOccurred(), "the targeted CF does not support multiple buildpacks")
	return supported
}

func ApiSupportsSymlinks() bool {
	supported, err := cutlass.ApiGreaterThan("2.103.0")
	Expect(err).NotTo(HaveOccurred(), "the targeted CF does not support symlinks")
	return supported
}

func ApiHasStackAssociation() bool {
	supported, err := cutlass.ApiGreaterThan("2.113.0")
	Expect(err).NotTo(HaveOccurred(), "the targeted CF does not support stack association")
	return supported
}

func AssertUsesProxyDuringStagingIfPresent(fixtureName string) {
	Context("with an uncached buildpack", func() {
		BeforeEach(func() {
			if cutlass.Cached {
				Skip("Running cached tests")
			}
		})

		It("uses a proxy during staging if present", func() {
			proxy, err := cutlass.NewProxy()
			Expect(err).To(BeNil())
			defer proxy.Close()

			bpFile := filepath.Join(bpDir, buildpackVersion+"tmp")
			cmd := exec.Command("cp", packagedBuildpack.File, bpFile)
			err = cmd.Run()
			Expect(err).To(BeNil())
			defer os.Remove(bpFile)

			traffic, _, err := cutlass.InternetTraffic(
				bpDir,
				filepath.Join("fixtures", fixtureName),
				bpFile,
				[]string{"HTTP_PROXY=" + proxy.URL, "HTTPS_PROXY=" + proxy.URL},
			)
			Expect(err).To(BeNil())
			// Expect(built).To(BeTrue())

			destUrl, err := url.Parse(proxy.URL)
			Expect(err).To(BeNil())

			Expect(cutlass.UniqueDestination(
				traffic, fmt.Sprintf("%s.%s", destUrl.Hostname(), destUrl.Port()),
			)).To(BeNil())
		})
	})
}

func AssertNoInternetTraffic(fixtureName string) {
	It("has no traffic", func() {
		if !cutlass.Cached {
			Skip("Running uncached tests")
		}

		bpFile := filepath.Join(bpDir, buildpackVersion+"tmp")
		cmd := exec.Command("cp", packagedBuildpack.File, bpFile)
		err := cmd.Run()
		Expect(err).To(BeNil())
		defer os.Remove(bpFile)

		traffic, _, err := cutlass.InternetTraffic(
			bpDir,
			filepath.Join("fixtures", fixtureName),
			bpFile,
			[]string{},
		)
		Expect(err).To(BeNil())
		// Expect(built).To(BeTrue())
		Expect(traffic).To(BeEmpty())
	})
}

func RunCF(args ...string) error {
	command := exec.Command("cf", args...)
	command.Stdout = GinkgoWriter
	command.Stderr = GinkgoWriter
	return command.Run()
}
<commit_msg>Fix for libbuildpack update to InternetTraffic<commit_after>package integration_test

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
	"time"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var bpDir string
var buildpackVersion string
var packagedBuildpack cutlass.VersionedBuildpackPackage

func init() {
	flag.StringVar(&buildpackVersion, "version", "", "version to use (builds if empty)")
	flag.BoolVar(&cutlass.Cached, "cached", true, "cached buildpack")
	flag.StringVar(&cutlass.DefaultMemory, "memory", "128M", "default memory for pushed apps")
	flag.StringVar(&cutlass.DefaultDisk, "disk", "256M", "default disk for pushed apps")
	flag.Parse()
}

var _ = SynchronizedBeforeSuite(func() []byte {
	// Run once
	if buildpackVersion == "" {
		packagedBuildpack, err := cutlass.PackageUniquelyVersionedBuildpack(os.Getenv("CF_STACK"), ApiHasStackAssociation())
		Expect(err).NotTo(HaveOccurred(), "failed to package buildpack")

		data, err := json.Marshal(packagedBuildpack)
		Expect(err).NotTo(HaveOccurred())
		return data
	}

	return []byte{}
}, func(data []byte) {
	// Run on all nodes
	var err error
	if len(data) > 0 {
		err = json.Unmarshal(data, &packagedBuildpack)
		Expect(err).NotTo(HaveOccurred())
		buildpackVersion = packagedBuildpack.Version
	}

	bpDir, err = cutlass.FindRoot()
	Expect(err).NotTo(HaveOccurred())

	Expect(cutlass.CopyCfHome()).To(Succeed())
	cutlass.SeedRandom()
	cutlass.DefaultStdoutStderr = GinkgoWriter
})

var _ = SynchronizedAfterSuite(func() {
	// Run on all nodes
}, func() {
	// Run once
	Expect(cutlass.RemovePackagedBuildpack(packagedBuildpack)).To(Succeed())
	Expect(cutlass.DeleteOrphanedRoutes()).To(Succeed())
})

func TestIntegration(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Integration Suite")
}

func PushAppAndConfirm(app *cutlass.App) {
	Expect(app.Push()).To(Succeed())
	Eventually(func() ([]string, error) { return app.InstanceStates() }, 20*time.Second).Should(Equal([]string{"RUNNING"}))
	Expect(app.ConfirmBuildpack(buildpackVersion)).To(Succeed())
}

func DestroyApp(app *cutlass.App) *cutlass.App {
	if app != nil {
		app.Destroy()
	}
	return nil
}

func ApiHasTask() bool {
	supported, err := cutlass.ApiGreaterThan("2.75.0")
	Expect(err).NotTo(HaveOccurred())
	return supported
}

func ApiHasMultiBuildpack() bool {
	supported, err := cutlass.ApiGreaterThan("2.90.0")
	Expect(err).NotTo(HaveOccurred(), "the targeted CF does not support multiple buildpacks")
	return supported
}

func ApiSupportsSymlinks() bool {
	supported, err := cutlass.ApiGreaterThan("2.103.0")
	Expect(err).NotTo(HaveOccurred(), "the targeted CF does not support symlinks")
	return supported
}

func ApiHasStackAssociation() bool {
	supported, err := cutlass.ApiGreaterThan("2.113.0")
	Expect(err).NotTo(HaveOccurred(), "the targeted CF does not support stack association")
	return supported
}

func AssertUsesProxyDuringStagingIfPresent(fixtureName string) {
	Context("with an uncached buildpack", func() {
		BeforeEach(func() {
			if cutlass.Cached {
				Skip("Running cached tests")
			}
		})

		It("uses a proxy during staging if present", func() {
			proxy, err := cutlass.NewProxy()
			Expect(err).To(BeNil())
			defer proxy.Close()

			bpFile := filepath.Join(bpDir, buildpackVersion+"tmp")
			cmd := exec.Command("cp", packagedBuildpack.File, bpFile)
			err = cmd.Run()
			Expect(err).To(BeNil())
			defer os.Remove(bpFile)

			traffic, _, _, err := cutlass.InternetTraffic(
				bpDir,
				filepath.Join("fixtures", fixtureName),
				bpFile,
				[]string{"HTTP_PROXY=" + proxy.URL, "HTTPS_PROXY=" + proxy.URL},
			)
			Expect(err).To(BeNil())
			// Expect(built).To(BeTrue())

			destUrl, err := url.Parse(proxy.URL)
			Expect(err).To(BeNil())

			Expect(cutlass.UniqueDestination(
				traffic, fmt.Sprintf("%s.%s", destUrl.Hostname(), destUrl.Port()),
			)).To(BeNil())
		})
	})
}

func AssertNoInternetTraffic(fixtureName string) {
	It("has no traffic", func() {
		if !cutlass.Cached {
			Skip("Running uncached tests")
		}

		bpFile := filepath.Join(bpDir, buildpackVersion+"tmp")
		cmd := exec.Command("cp", packagedBuildpack.File, bpFile)
		err := cmd.Run()
		Expect(err).To(BeNil())
		defer os.Remove(bpFile)

		traffic, _, _, err := cutlass.InternetTraffic(
			bpDir,
			filepath.Join("fixtures", fixtureName),
			bpFile,
			[]string{},
		)
		Expect(err).To(BeNil())
		// Expect(built).To(BeTrue())
		Expect(traffic).To(BeEmpty())
	})
}

func RunCF(args ...string) error {
	command := exec.Command("cf", args...)
	command.Stdout = GinkgoWriter
	command.Stderr = GinkgoWriter
	return command.Run()
}
<|endoftext|>


<commit_before>package sourcegraph

import (
	"encoding/json"
	"errors"
	"fmt"
	"path/filepath"
	"reflect"
	"strings"

	"sourcegraph.com/sourcegraph/srclib/unit"
	"sourcegraph.com/sourcegraph/vcsstore/vcsclient"
)

// A Token is the smallest indivisible component of a query, either a
// term or a "field:val" specifier (e.g., "repo:example.com/myrepo").
type Token interface {
	// String returns the string representation of the term.
	String() string
}

// A Term is a query term token. It is either a word or an arbitrary
// string (if quoted in the raw query).
type Term string

func (t Term) String() string {
	if strings.Contains(string(t), " ") {
		return `"` + string(t) + `"`
	}
	return string(t)
}

func (t Term) UnquotedString() string { return string(t) }

// An AnyToken is a token that has not yet been resolved into another
// token type. It resolves to Term if it can't be resolved to another
// token type.
type AnyToken string

func (u AnyToken) String() string { return string(u) }

// A RepoToken represents a repository, although it does not
// necessarily uniquely identify the repository. It consists of any
// number of slash-separated path components, such as "a/b" or
// "github.com/foo/bar".
type RepoToken struct {
	URI string

	Repo *Repo `json:",omitempty"`
}

func (t RepoToken) String() string { return t.URI }

func (t RepoToken) Spec() RepoSpec {
	var rid int
	if t.Repo != nil {
		rid = t.Repo.RID
	}
	return RepoSpec{URI: t.URI, RID: rid}
}

// A RevToken represents a specific revision (either a revspec or a
// commit ID) of a repository (which must be specified by a previous
// RepoToken in the query).
type RevToken struct {
	Rev string // Rev is either a revspec or commit ID

	Commit *Commit `json:",omitempty"`
}

func (t RevToken) String() string { return ":" + t.Rev }

// A UnitToken represents a source unit in a repository.
type UnitToken struct {
	// UnitType is the type of the source unit (e.g., GoPackage).
	UnitType string

	// Name is the name of the source unit (e.g., mypkg).
	Name string

	// Unit is the source unit object.
	Unit *unit.RepoSourceUnit
}

func (t UnitToken) String() string { return "~" + t.Name + "@" + t.UnitType }

type FileToken struct {
	Path string

	Entry *vcsclient.TreeEntry
}

func (t FileToken) String() string { return "/" + filepath.Clean(t.Path) }

// A UserToken represents a user or org, although it does not
// necessarily uniquely identify one. It consists of the string "@"
// followed by a full or partial user/org login.
type UserToken struct {
	Login string

	User *User `json:",omitempty"`
}

func (t UserToken) String() string { return "@" + t.Login }

// Tokens wraps a list of tokens and adds some helper methods. It also
// serializes to JSON with "Type" fields added to each token and
// deserializes that same JSON back into a typed list of tokens.
type Tokens []Token

func (d Tokens) MarshalJSON() ([]byte, error) {
	b, err := json.Marshal(([]Token)(d))
	if err != nil {
		return nil, err
	}
	var toks []interface{}
	if err := json.Unmarshal(b, &toks); err != nil {
		return nil, err
	}
	for i, tok := range toks {
		ttype := TokenType(d[i])
		switch tok := tok.(type) {
		case string:
			toks[i] = map[string]string{"Type": ttype, "String": tok}
		case map[string]interface{}:
			tok["Type"] = ttype
		}
	}
	return json.Marshal(toks)
}

func (d *Tokens) UnmarshalJSON(b []byte) error {
	var jtoks []jsonToken
	if err := json.Unmarshal(b, &jtoks); err != nil {
		return err
	}
	if jtoks == nil {
		*d = nil
	} else {
		*d = make(Tokens, len(jtoks))
		for i, jtok := range jtoks {
			(*d)[i] = jtok.Token
		}
	}
	return nil
}

func (d Tokens) RawQueryString() string { return Join(d).String }

type jsonToken struct {
	Token `json:",omitempty"`
}

func (t jsonToken) MarshalJSON() ([]byte, error) {
	b, err := json.Marshal(t.Token)
	if err != nil {
		return nil, err
	}

	var v interface{}
	if err := json.Unmarshal(b, &v); err != nil {
		return nil, err
	}

	if t.Token != nil {
		tokType := TokenType(t.Token)
		switch vv := v.(type) {
		case string:
			v = map[string]string{"Type": tokType, "String": vv}
		case map[string]interface{}:
			vv["Type"] = tokType
		}
	}
	return json.Marshal(v)
}

func (t *jsonToken) UnmarshalJSON(b []byte) error {
	var v map[string]interface{}
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}
	tok, err := toTypedToken(v)
	if err != nil {
		return err
	}
	*t = jsonToken{tok}
	return nil
}

func toTypedToken(tokJSON map[string]interface{}) (Token, error) {
	if tokJSON == nil {
		return nil, nil
	}
	typ, ok := tokJSON["Type"].(string)
	if !ok {
		return nil, errors.New("unmarshal Tokens: no 'Type' field in token")
	}
	delete(tokJSON, "Type")

	var tok interface{}
	switch typ {
	case "Term", "AnyToken":
		s, _ := tokJSON["String"].(string)
		switch typ {
		case "Term":
			tok = Term(s)
		case "AnyToken":
			tok = AnyToken(s)
		}
		return tok.(Token), nil

	case "RepoToken":
		tok = &RepoToken{}
	case "RevToken":
		tok = &RevToken{}
	case "UnitToken":
		tok = &UnitToken{}
	case "FileToken":
		tok = &FileToken{}
	case "UserToken":
		tok = &UserToken{}
	default:
		return nil, fmt.Errorf("unmarshal Tokens: unrecognized Type %q", typ)
	}
	tmpJSON, err := json.Marshal(tokJSON)
	if err != nil {
		return nil, err
	}
	if err := json.Unmarshal(tmpJSON, tok); err != nil {
		return nil, err
	}
	tok = reflect.ValueOf(tok).Elem().Interface() // deref
	return tok.(Token), nil
}

func TokenType(tok Token) string {
	return strings.Replace(strings.Replace(reflect.ValueOf(tok).Type().String(), "*", "", -1), "sourcegraph.", "", -1)
}
<commit_msg>only append UnitType if set<commit_after>package sourcegraph

import (
	"encoding/json"
	"errors"
	"fmt"
	"path/filepath"
	"reflect"
	"strings"

	"sourcegraph.com/sourcegraph/srclib/unit"
	"sourcegraph.com/sourcegraph/vcsstore/vcsclient"
)

// A Token is the smallest indivisible component of a query, either a
// term or a "field:val" specifier (e.g., "repo:example.com/myrepo").
type Token interface {
	// String returns the string representation of the term.
	String() string
}

// A Term is a query term token. It is either a word or an arbitrary
// string (if quoted in the raw query).
type Term string

func (t Term) String() string {
	if strings.Contains(string(t), " ") {
		return `"` + string(t) + `"`
	}
	return string(t)
}

func (t Term) UnquotedString() string { return string(t) }

// An AnyToken is a token that has not yet been resolved into another
// token type. It resolves to Term if it can't be resolved to another
// token type.
type AnyToken string

func (u AnyToken) String() string { return string(u) }

// A RepoToken represents a repository, although it does not
// necessarily uniquely identify the repository. It consists of any
// number of slash-separated path components, such as "a/b" or
// "github.com/foo/bar".
type RepoToken struct {
	URI string

	Repo *Repo `json:",omitempty"`
}

func (t RepoToken) String() string { return t.URI }

func (t RepoToken) Spec() RepoSpec {
	var rid int
	if t.Repo != nil {
		rid = t.Repo.RID
	}
	return RepoSpec{URI: t.URI, RID: rid}
}

// A RevToken represents a specific revision (either a revspec or a
// commit ID) of a repository (which must be specified by a previous
// RepoToken in the query).
type RevToken struct {
	Rev string // Rev is either a revspec or commit ID

	Commit *Commit `json:",omitempty"`
}

func (t RevToken) String() string { return ":" + t.Rev }

// A UnitToken represents a source unit in a repository.
type UnitToken struct {
	// UnitType is the type of the source unit (e.g., GoPackage).
	UnitType string

	// Name is the name of the source unit (e.g., mypkg).
	Name string

	// Unit is the source unit object.
	Unit *unit.RepoSourceUnit
}

func (t UnitToken) String() string {
	s := "~" + t.Name
	if t.UnitType != "" {
		s += "@" + t.UnitType
	}
	return s
}

type FileToken struct {
	Path string

	Entry *vcsclient.TreeEntry
}

func (t FileToken) String() string { return "/" + filepath.Clean(t.Path) }

// A UserToken represents a user or org, although it does not
// necessarily uniquely identify one. It consists of the string "@"
// followed by a full or partial user/org login.
type UserToken struct {
	Login string

	User *User `json:",omitempty"`
}

func (t UserToken) String() string { return "@" + t.Login }

// Tokens wraps a list of tokens and adds some helper methods. It also
// serializes to JSON with "Type" fields added to each token and
// deserializes that same JSON back into a typed list of tokens.
type Tokens []Token

func (d Tokens) MarshalJSON() ([]byte, error) {
	b, err := json.Marshal(([]Token)(d))
	if err != nil {
		return nil, err
	}
	var toks []interface{}
	if err := json.Unmarshal(b, &toks); err != nil {
		return nil, err
	}
	for i, tok := range toks {
		ttype := TokenType(d[i])
		switch tok := tok.(type) {
		case string:
			toks[i] = map[string]string{"Type": ttype, "String": tok}
		case map[string]interface{}:
			tok["Type"] = ttype
		}
	}
	return json.Marshal(toks)
}

func (d *Tokens) UnmarshalJSON(b []byte) error {
	var jtoks []jsonToken
	if err := json.Unmarshal(b, &jtoks); err != nil {
		return err
	}
	if jtoks == nil {
		*d = nil
	} else {
		*d = make(Tokens, len(jtoks))
		for i, jtok := range jtoks {
			(*d)[i] = jtok.Token
		}
	}
	return nil
}

func (d Tokens) RawQueryString() string { return Join(d).String }

type jsonToken struct {
	Token `json:",omitempty"`
}

func (t jsonToken) MarshalJSON() ([]byte, error) {
	b, err := json.Marshal(t.Token)
	if err != nil {
		return nil, err
	}

	var v interface{}
	if err := json.Unmarshal(b, &v); err != nil {
		return nil, err
	}

	if t.Token != nil {
		tokType := TokenType(t.Token)
		switch vv := v.(type) {
		case string:
			v = map[string]string{"Type": tokType, "String": vv}
		case map[string]interface{}:
			vv["Type"] = tokType
		}
	}
	return json.Marshal(v)
}

func (t *jsonToken) UnmarshalJSON(b []byte) error {
	var v map[string]interface{}
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}
	tok, err := toTypedToken(v)
	if err != nil {
		return err
	}
	*t = jsonToken{tok}
	return nil
}

func toTypedToken(tokJSON map[string]interface{}) (Token, error) {
	if tokJSON == nil {
		return nil, nil
	}
	typ, ok := tokJSON["Type"].(string)
	if !ok {
		return nil, errors.New("unmarshal Tokens: no 'Type' field in token")
	}
	delete(tokJSON, "Type")

	var tok interface{}
	switch typ {
	case "Term", "AnyToken":
		s, _ := tokJSON["String"].(string)
		switch typ {
		case "Term":
			tok = Term(s)
		case "AnyToken":
			tok = AnyToken(s)
		}
		return tok.(Token), nil

	case "RepoToken":
		tok = &RepoToken{}
	case "RevToken":
		tok = &RevToken{}
	case "UnitToken":
		tok = &UnitToken{}
	case "FileToken":
		tok = &FileToken{}
	case "UserToken":
		tok = &UserToken{}
	default:
		return nil, fmt.Errorf("unmarshal Tokens: unrecognized Type %q", typ)
	}
	tmpJSON, err := json.Marshal(tokJSON)
	if err != nil {
		return nil, err
	}
	if err := json.Unmarshal(tmpJSON, tok); err != nil {
		return nil, err
	}
	tok = reflect.ValueOf(tok).Elem().Interface() // deref
	return tok.(Token), nil
}

func TokenType(tok Token) string {
	return strings.Replace(strings.Replace(reflect.ValueOf(tok).Type().String(), "*", "", -1), "sourcegraph.", "", -1)
}
<|endoftext|>


<commit_before>package defaultdata_test

import (
	. "github.com/hiromaily/golibs/example/defaultdata"
	tu "github.com/hiromaily/golibs/testutil"
	"os"
	"testing"
)

//-----------------------------------------------------------------------------
// Test Framework
//-----------------------------------------------------------------------------
// Initialize
func init() {
	tu.InitializeTest("[DefaultData]")
}

func setup() {
}

func teardown() {
}

func TestMain(m *testing.M) {
	setup()

	code := m.Run()

	teardown()

	os.Exit(code)
}

//-----------------------------------------------------------------------------
// Test
//-----------------------------------------------------------------------------
func TestDefault(t *testing.T) {

	//cannot use nil as type int in argument to defaultdata.CheckInt
	/*
		CheckInt(nil)
	*/

	//cannot use nil as type string in argument to defaultdata.CheckString
	/*
		CheckString(nil)
	*/

	//cannot use nil as type bool in argument to defaultdata.CheckBool
	/*
		CheckBool(nil)
	*/
	CheckByte(nil)

	CheckError(nil)

	CheckSlice(nil)

	CheckMap(nil)

	CheckInterface(nil)

	CheckMultiInterface(nil)

	CheckMultiInterface(nil, nil, nil)

	//----------------------------------------------------
	//What's happened when sending slice data to interface
	//----------------------------------------------------
	data := []int{1, 2, 3, 4, 5}
	CheckInterfaceWhenSlice(data)

	var intData int = 1
	p := &intData
	CheckInterfaceWhenPointer(p)

	//----------------------------------------------------
	//Check givedvalue after calling func.
	//----------------------------------------------------
	strData := []string{"a", "b", "c", "d", "e"}
	ChangeValOnSlice(strData)
	//t.Logf("ChangeValOnSlice: %v", strData)
	if strData[0] == "a" {
		t.Errorf("ChangeValOnSlice value: %v", strData)
	}
	//changed!

	mapInt := map[string]int{"apple": 100, "lemon": 200, "banana": 300}
	ChangeValOnMap(mapInt)
	//t.Logf("ChangeValOnMap: %v", mapInt)
	if mapInt["apple"] == 100 {
		t.Errorf("ChangeValOnMap value: %v", mapInt)
	}
	//changed!

	strData2 := "before"
	ChangeValOnInterface(strData2)
	if strData2 != "before" {
		t.Errorf("ChangeValOnInterface value: %v", strData2)
	}
	//Not changed!

	//set address as pointer
	ChangeValOnPointer(&strData2)
	if strData2 == "before" {
		t.Errorf("ChangeValOnPointer value: %v", strData2)
	}
	//changed!

}
<commit_msg>added example<commit_after>package defaultdata_test

import (
	"fmt"
	. "github.com/hiromaily/golibs/example/defaultdata"
	tu "github.com/hiromaily/golibs/testutil"
	"os"
	"testing"
)

//-----------------------------------------------------------------------------
// Test Framework
//-----------------------------------------------------------------------------
// Initialize
func init() {
	tu.InitializeTest("[DefaultData]")
}

func setup() {
}

func teardown() {
}

func TestMain(m *testing.M) {
	setup()

	code := m.Run()

	teardown()

	os.Exit(code)
}

//-----------------------------------------------------------------------------
// Test
//-----------------------------------------------------------------------------
func TestDefault(t *testing.T) {

	//cannot use nil as type int in argument to defaultdata.CheckInt
	/*
		CheckInt(nil)
	*/

	//cannot use nil as type string in argument to defaultdata.CheckString
	/*
		CheckString(nil)
	*/

	//cannot use nil as type bool in argument to defaultdata.CheckBool
	/*
		CheckBool(nil)
	*/
	CheckByte(nil)

	CheckError(nil)

	CheckSlice(nil)

	CheckMap(nil)

	CheckInterface(nil)

	CheckMultiInterface(nil)

	CheckMultiInterface(nil, nil, nil)

	//----------------------------------------------------
	//What's happened when sending slice data to interface
	//----------------------------------------------------
	data := []int{1, 2, 3, 4, 5}
	CheckInterfaceWhenSlice(data)

	var intData int = 1
	p := &intData
	CheckInterfaceWhenPointer(p)

	//----------------------------------------------------
	//Check givedvalue after calling func.
	//----------------------------------------------------
	strData := []string{"a", "b", "c", "d", "e"}
	ChangeValOnSlice(strData)
	//t.Logf("ChangeValOnSlice: %v", strData)
	if strData[0] == "a" {
		t.Errorf("ChangeValOnSlice value: %v", strData)
	}
	//changed!

	mapInt := map[string]int{"apple": 100, "lemon": 200, "banana": 300}
	ChangeValOnMap(mapInt)
	//t.Logf("ChangeValOnMap: %v", mapInt)
	if mapInt["apple"] == 100 {
		t.Errorf("ChangeValOnMap value: %v", mapInt)
	}
	//changed!

	strData2 := "before"
	ChangeValOnInterface(strData2)
	if strData2 != "before" {
		t.Errorf("ChangeValOnInterface value: %v", strData2)
	}
	//Not changed!

	//set address as pointer
	ChangeValOnPointer(&strData2)
	if strData2 == "before" {
		t.Errorf("ChangeValOnPointer value: %v", strData2)
	}
	//changed!

}

//-----------------------------------------------------------------------------
// Example
//-----------------------------------------------------------------------------
func ExampleHello() {
	fmt.Println("hello")
	// Output: hello
}
<|endoftext|>


<commit_before>package weblinks

import "testing"
import "strings"

func TestGet(t *testing.T) {

	reader := strings.NewReader(`

    This webpage contains lot of links

`)

	links := Get(reader)

	if len(links) != 3 {
		t.Error("Incorrect number of URL returned")
	}

}
<commit_msg>Add benchmark test<commit_after>package weblinks

import (
	"strings"
	"testing"
)

var reader = strings.NewReader(`

  This webpage contains lot of links

`)

// go test -v
func TestGet(t *testing.T) {

	links := Get(reader)

	if len(links) != 3 {
		t.Error("Incorrect number of URL returned")
	}

}

// go test -test.bench=".*"
func BenchmarkGet(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Get(reader)
	}
}
<|endoftext|>