<commit_before>package mapreduce

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"sync"
	"time"

	"testing/iotest"

	"github.com/pachyderm-io/pfs/lib/btrfs"
	"github.com/pachyderm-io/pfs/lib/route"
	"github.com/samalba/dockerclient"
)

var retries int = 5

// StartContainer pulls image and starts a container from it with command. It
// returns the container id or an error.
func spinupContainer(image string, command []string) (string, error) {
	log.Print("spinupContainer", " ", image, " ", command)
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		log.Print(err)
		return "", err
	}
	if err := docker.PullImage(image, nil); err != nil {
		log.Print(err)
		return "", err
	}

	containerConfig := &dockerclient.ContainerConfig{Image: image, Cmd: command}

	containerId, err := docker.CreateContainer(containerConfig, "")
	if err != nil {
		log.Print(err)
		return "", nil
	}

	if err := docker.StartContainer(containerId, &dockerclient.HostConfig{}); err != nil {
		log.Print(err)
		return "", err
	}

	return containerId, nil
}

func ipAddr(containerId string) (string, error) {
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return "", err
	}
	containerInfo, err := docker.InspectContainer(containerId)
	if err != nil {
		return "", err
	}

	return containerInfo.NetworkSettings.IpAddress, nil
}

func retry(f func() error, retries int, pause time.Duration) error {
	var err error
	for i := 0; i < retries; i++ {
		err = f()
		if err == nil {
			break
		} else {
			time.Sleep(pause)
		}
	}
	return err
}

// contains checks if set contains val. It assums that set has already been
// sorted.
func contains(set []string, val string) bool {
	index := sort.SearchStrings(set, val)
	return index < len(set) && set[index] == val
}

type Job struct {
	Type    string   `json:"type"`
	Input   string   `json:"input"`
	Image   string   `json:"image"`
	Command []string `json:"command"`
}

type materializeInfo struct {
	In, Out, Branch, Commit string
}

func PrepJob(job Job, jobPath string, m materializeInfo) error {
	if err := btrfs.MkdirAll(path.Join(m.Out, m.Branch, jobPath)); err != nil {
		return err
	}
	return nil
}

func Map(job Job, jobPath string, m materializeInfo, host string) error {
	err := PrepJob(job, path.Base(jobPath), m)
	if err != nil {
		return err
	}

	if job.Type != "map" {
		return fmt.Errorf("runMap called on a job of type \"%s\". Should be \"map\".", job.Type)
	}

	inFiles, err := btrfs.ReadDir(path.Join(m.In, m.Commit, job.Input))
	if err != nil {
		return err
	}
	log.Print("In Map for ", jobPath, " len(inFiles) = ", len(inFiles))

	files := make(chan os.FileInfo, 20000)

	// spawn four worker goroutines
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for inF := range files {
				inFile, err := btrfs.Open(path.Join(m.In, m.Commit, job.Input, inF.Name()))
				if err != nil {
					log.Print(err)
					return
				}
				defer inFile.Close()

				var resp *http.Response
				err = retry(func() error {
					log.Print("Posting: ", inF.Name())
					resp, err = http.Post("http://"+path.Join(host, inF.Name()), "application/text", inFile)
					return err
				}, 5, 200*time.Millisecond)
				if err != nil {
					log.Print(err)
					return
				}
				defer resp.Body.Close()

				outFile, err := btrfs.Create(path.Join(m.Out, m.Branch, jobPath, inF.Name()))
				if err != nil {
					log.Print(err)
					return
				}
				defer outFile.Close()
				if _, err := io.Copy(outFile, resp.Body); err != nil {
					log.Print(err)
					return
				}
			}
		}()
	}

	for _, inF := range inFiles {
		files <- inF
	}

	close(files)

	return nil
}

func Reduce(job Job, jobPath string, m materializeInfo, host string, shard, modulos uint64) error {
	if (route.HashResource(path.Join("/job", jobPath)) % modulos) != shard {
		// This resource isn't supposed to be located on this machine.
		return nil
	}
	log.Print("Reduce: ", job, " ", jobPath, " ")
	if job.Type != "reduce" {
		return fmt.Errorf("Reduce called on a job of type \"%s\". Should be \"reduce\".", job.Type)
	}

	// Notice we're just passing "host" here. Multicast will fill in the host
	// field so we don't actually need to specify it.
	req, err := http.NewRequest("GET", "http://host/"+path.Join(job.Input, "file", "*")+"?commit="+m.Commit, nil)
	if err != nil {
		return err
	}
	_reader, err := route.Multicast(req, "/pfs/master")
	if err != nil {
		return err
	}
	defer _reader.Close()
	reader := iotest.NewReadLogger("Reduce", _reader)

	var resp *http.Response
	err = retry(func() error {
		resp, err = http.Post("http://"+path.Join(host, job.Input), "application/text", reader)
		return err
	}, 5, 200*time.Millisecond)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("Received error %s", resp.Status)
	}

	outFile, err := btrfs.Create(path.Join(m.Out, m.Branch, jobPath))
	if err != nil {
		return err
	}
	defer outFile.Close()
	if _, err := io.Copy(outFile, resp.Body); err != nil {
		return err
	}
	return nil
}

type jobCond struct {
	sync.Cond
	Done bool
}

var jobs map[string]*jobCond = make(map[string]*jobCond)
var jobsAccess sync.Mutex

// jobCond returns the name of the condition variable for job.
func condKey(in_repo, commit, job string) string {
	return path.Join(in_repo, commit, job)
}

func ensureCond(name string) {
	jobsAccess.Lock()
	defer jobsAccess.Unlock()
	if _, ok := jobs[name]; !ok {
		jobs[name] = &jobCond{sync.Cond{L: &sync.Mutex{}}, false}
	}
}

func broadcast(in_repo, commit, job string) {
	name := condKey(in_repo, commit, job)
	ensureCond(name)
	jobs[name].L.Lock()
	jobs[name].Done = true
	jobs[name].Broadcast()
	jobs[name].L.Unlock()
}

func WaitJob(in_repo, commit, job string) {
	name := condKey(in_repo, commit, job)
	ensureCond(name)
	jobs[name].L.Lock()
	for !jobs[name].Done {
		jobs[name].Wait()
	}
	jobs[name].L.Unlock()
}

// Materialize parses the jobs found in `in_repo`/`commit`/`jobDir` runs them
// with `in_repo/commit` as input, outputs the results to `out_repo`/`branch`
// and commits them as `out_repo`/`commit`
func Materialize(in_repo, branch, commit, out_repo, jobDir string, shard, modulos uint64) error {
	log.Printf("Materialize: %s %s %s %s %s.", in_repo, branch, commit, out_repo, jobDir)
	// We make sure that this function always commits so that we know the comp
	// repo stays in sync with the data repo.
	defer func() {
		if err := btrfs.Commit(out_repo, commit, branch); err != nil {
			log.Print("btrfs.Commit error in Materialize: ", err)
		}
	}()
	// First check if the jobs dir actually exists.
	exists, err := btrfs.FileExists(path.Join(in_repo, commit, jobDir))
	if err != nil {
		return err
	}
	if !exists {
		// Perfectly valid to have no jobs dir, it just means we have no work
		// to do.
		return nil
	}

	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return err
	}
	newFiles, err := btrfs.NewFiles(in_repo, commit)
	if err != nil {
		return err
	}
	sort.Strings(newFiles)

	jobsPath := path.Join(in_repo, commit, jobDir)
	jobs, err := btrfs.ReadDir(jobsPath)
	if err != nil {
		return err
	}
	var wg sync.WaitGroup
	defer wg.Wait()
	for _, jobInfo := range jobs {
		wg.Add(1)
		go func(jobInfo os.FileInfo) {
			defer wg.Done()
			defer broadcast(in_repo, commit, jobInfo.Name())
			jobFile, err := btrfs.Open(path.Join(jobsPath, jobInfo.Name()))
			if err != nil {
				log.Print(err)
				return
			}
			defer jobFile.Close()
			decoder := json.NewDecoder(jobFile)
			job := Job{}
			if err = decoder.Decode(&job); err != nil {
				log.Print(err)
				return
			}
			log.Print("Job: ", job)
			m := materializeInfo{in_repo, out_repo, branch, commit}

			containerId, err := spinupContainer(job.Image, job.Command)
			if err != nil {
				log.Print(err)
				return
			}
			defer docker.StopContainer(containerId, 5)

			containerAddr, err := ipAddr(containerId)
			if err != nil {
				log.Print(err)
				return
			}

			if job.Type == "map" {
				err := Map(job, jobInfo.Name(), m, containerAddr)
				if err != nil {
					log.Print(err)
					return
				}
			} else if job.Type == "reduce" {
				err := Reduce(job, jobInfo.Name(), m, containerAddr, shard, modulos)
				if err != nil {
					log.Print(err)
					return
				}
			} else {
				log.Printf("Job %s has unrecognized type: %s.", jobInfo.Name(), job.Type)
				return
			}
		}(jobInfo)
	}
	return nil
}
<commit_msg>Adds more logging.<commit_after>package mapreduce

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"sync"
	"time"

	"testing/iotest"

	"github.com/pachyderm-io/pfs/lib/btrfs"
	"github.com/pachyderm-io/pfs/lib/route"
	"github.com/samalba/dockerclient"
)

var retries int = 5

// StartContainer pulls image and starts a container from it with command. It
// returns the container id or an error.
func spinupContainer(image string, command []string) (string, error) {
	log.Print("spinupContainer", " ", image, " ", command)
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		log.Print(err)
		return "", err
	}
	if err := docker.PullImage(image, nil); err != nil {
		log.Print("Image: ", image, " Err: ", err)
		return "", err
	}

	containerConfig := &dockerclient.ContainerConfig{Image: image, Cmd: command}

	containerId, err := docker.CreateContainer(containerConfig, "")
	if err != nil {
		log.Print(err)
		return "", nil
	}

	if err := docker.StartContainer(containerId, &dockerclient.HostConfig{}); err != nil {
		log.Print(err)
		return "", err
	}

	return containerId, nil
}

func ipAddr(containerId string) (string, error) {
	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return "", err
	}
	containerInfo, err := docker.InspectContainer(containerId)
	if err != nil {
		return "", err
	}

	return containerInfo.NetworkSettings.IpAddress, nil
}

func retry(f func() error, retries int, pause time.Duration) error {
	var err error
	for i := 0; i < retries; i++ {
		err = f()
		if err == nil {
			break
		} else {
			time.Sleep(pause)
		}
	}
	return err
}

// contains checks if set contains val. It assums that set has already been
// sorted.
func contains(set []string, val string) bool {
	index := sort.SearchStrings(set, val)
	return index < len(set) && set[index] == val
}

type Job struct {
	Type    string   `json:"type"`
	Input   string   `json:"input"`
	Image   string   `json:"image"`
	Command []string `json:"command"`
}

type materializeInfo struct {
	In, Out, Branch, Commit string
}

func PrepJob(job Job, jobPath string, m materializeInfo) error {
	if err := btrfs.MkdirAll(path.Join(m.Out, m.Branch, jobPath)); err != nil {
		return err
	}
	return nil
}

func Map(job Job, jobPath string, m materializeInfo, host string) error {
	err := PrepJob(job, path.Base(jobPath), m)
	if err != nil {
		return err
	}

	if job.Type != "map" {
		return fmt.Errorf("runMap called on a job of type \"%s\". Should be \"map\".", job.Type)
	}

	inFiles, err := btrfs.ReadDir(path.Join(m.In, m.Commit, job.Input))
	if err != nil {
		return err
	}
	log.Print("In Map for ", jobPath, " len(inFiles) = ", len(inFiles))

	files := make(chan os.FileInfo, 20000)

	// spawn four worker goroutines
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for inF := range files {
				inFile, err := btrfs.Open(path.Join(m.In, m.Commit, job.Input, inF.Name()))
				if err != nil {
					log.Print(err)
					return
				}
				defer inFile.Close()

				var resp *http.Response
				err = retry(func() error {
					log.Print("Posting: ", inF.Name())
					resp, err = http.Post("http://"+path.Join(host, inF.Name()), "application/text", inFile)
					return err
				}, 5, 200*time.Millisecond)
				if err != nil {
					log.Print(err)
					return
				}
				defer resp.Body.Close()

				outFile, err := btrfs.Create(path.Join(m.Out, m.Branch, jobPath, inF.Name()))
				if err != nil {
					log.Print(err)
					return
				}
				defer outFile.Close()
				if _, err := io.Copy(outFile, resp.Body); err != nil {
					log.Print(err)
					return
				}
			}
		}()
	}

	for _, inF := range inFiles {
		files <- inF
	}

	close(files)

	return nil
}

func Reduce(job Job, jobPath string, m materializeInfo, host string, shard, modulos uint64) error {
	if (route.HashResource(path.Join("/job", jobPath)) % modulos) != shard {
		// This resource isn't supposed to be located on this machine.
		return nil
	}
	log.Print("Reduce: ", job, " ", jobPath, " ")
	if job.Type != "reduce" {
		return fmt.Errorf("Reduce called on a job of type \"%s\". Should be \"reduce\".", job.Type)
	}

	// Notice we're just passing "host" here. Multicast will fill in the host
	// field so we don't actually need to specify it.
	req, err := http.NewRequest("GET", "http://host/"+path.Join(job.Input, "file", "*")+"?commit="+m.Commit, nil)
	if err != nil {
		return err
	}
	_reader, err := route.Multicast(req, "/pfs/master")
	if err != nil {
		return err
	}
	defer _reader.Close()
	reader := iotest.NewReadLogger("Reduce", _reader)

	var resp *http.Response
	err = retry(func() error {
		resp, err = http.Post("http://"+path.Join(host, job.Input), "application/text", reader)
		return err
	}, 5, 200*time.Millisecond)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("Received error %s", resp.Status)
	}

	outFile, err := btrfs.Create(path.Join(m.Out, m.Branch, jobPath))
	if err != nil {
		return err
	}
	defer outFile.Close()
	if _, err := io.Copy(outFile, resp.Body); err != nil {
		return err
	}
	return nil
}

type jobCond struct {
	sync.Cond
	Done bool
}

var jobs map[string]*jobCond = make(map[string]*jobCond)
var jobsAccess sync.Mutex

// jobCond returns the name of the condition variable for job.
func condKey(in_repo, commit, job string) string {
	return path.Join(in_repo, commit, job)
}

func ensureCond(name string) {
	jobsAccess.Lock()
	defer jobsAccess.Unlock()
	if _, ok := jobs[name]; !ok {
		jobs[name] = &jobCond{sync.Cond{L: &sync.Mutex{}}, false}
	}
}

func broadcast(in_repo, commit, job string) {
	name := condKey(in_repo, commit, job)
	ensureCond(name)
	jobs[name].L.Lock()
	jobs[name].Done = true
	jobs[name].Broadcast()
	jobs[name].L.Unlock()
}

func WaitJob(in_repo, commit, job string) {
	name := condKey(in_repo, commit, job)
	ensureCond(name)
	jobs[name].L.Lock()
	for !jobs[name].Done {
		jobs[name].Wait()
	}
	jobs[name].L.Unlock()
}

// Materialize parses the jobs found in `in_repo`/`commit`/`jobDir` runs them
// with `in_repo/commit` as input, outputs the results to `out_repo`/`branch`
// and commits them as `out_repo`/`commit`
func Materialize(in_repo, branch, commit, out_repo, jobDir string, shard, modulos uint64) error {
	log.Printf("Materialize: %s %s %s %s %s.", in_repo, branch, commit, out_repo, jobDir)
	// We make sure that this function always commits so that we know the comp
	// repo stays in sync with the data repo.
	defer func() {
		if err := btrfs.Commit(out_repo, commit, branch); err != nil {
			log.Print("btrfs.Commit error in Materialize: ", err)
		}
	}()
	// First check if the jobs dir actually exists.
	exists, err := btrfs.FileExists(path.Join(in_repo, commit, jobDir))
	if err != nil {
		return err
	}
	if !exists {
		// Perfectly valid to have no jobs dir, it just means we have no work
		// to do.
		return nil
	}

	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		return err
	}
	newFiles, err := btrfs.NewFiles(in_repo, commit)
	if err != nil {
		return err
	}
	sort.Strings(newFiles)

	jobsPath := path.Join(in_repo, commit, jobDir)
	jobs, err := btrfs.ReadDir(jobsPath)
	if err != nil {
		return err
	}
	var wg sync.WaitGroup
	defer wg.Wait()
	for _, jobInfo := range jobs {
		wg.Add(1)
		go func(jobInfo os.FileInfo) {
			defer wg.Done()
			defer broadcast(in_repo, commit, jobInfo.Name())
			jobFile, err := btrfs.Open(path.Join(jobsPath, jobInfo.Name()))
			if err != nil {
				log.Print(err)
				return
			}
			defer jobFile.Close()
			decoder := json.NewDecoder(jobFile)
			job := Job{}
			if err = decoder.Decode(&job); err != nil {
				log.Print(err)
				return
			}
			log.Print("Job: ", job)
			m := materializeInfo{in_repo, out_repo, branch, commit}

			containerId, err := spinupContainer(job.Image, job.Command)
			if err != nil {
				log.Print(err)
				return
			}
			defer docker.StopContainer(containerId, 5)

			containerAddr, err := ipAddr(containerId)
			if err != nil {
				log.Print(err)
				return
			}

			if job.Type == "map" {
				err := Map(job, jobInfo.Name(), m, containerAddr)
				if err != nil {
					log.Print(err)
					return
				}
			} else if job.Type == "reduce" {
				err := Reduce(job, jobInfo.Name(), m, containerAddr, shard, modulos)
				if err != nil {
					log.Print(err)
					return
				}
			} else {
				log.Printf("Job %s has unrecognized type: %s.", jobInfo.Name(), job.Type)
				return
			}
		}(jobInfo)
	}
	return nil
}
<|endoftext|>


<commit_before>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * rpc/service_filter.go                                  *
 *                                                        *
 * hprose service filter for Go.                          *
 *                                                        *
 * LastModified: Oct 27, 2016                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package jsonrpc

import (
	"encoding/json"

	"github.com/hprose/hprose-golang/io"
	"github.com/hprose/hprose-golang/rpc"
)

// ServiceFilter is a JSONRPC Service Filter
type ServiceFilter struct{}

// InputFilter for JSONRPC Service
func (filter ServiceFilter) InputFilter(data []byte, context rpc.Context) []byte {
	if (len(data) > 0) && (data[0] == '[' || data[0] == '{') {
		var requests []map[string]interface{}
		if data[0] == '[' {
			if err := json.Unmarshal(data, &requests); err != nil {
				return data
			}
		} else {
			requests = make([]map[string]interface{}, 1)
			if err := json.Unmarshal(data, &requests[0]); err != nil {
				return data
			}
		}
		writer := io.NewWriter(true)
		n := len(requests)
		responses := make([]map[string]interface{}, n)
		for i, request := range requests {
			response := make(map[string]interface{})
			if id, ok := request["id"]; ok {
				response["id"] = id
			} else {
				response["id"] = nil
			}
			if version, ok := request["jsonrpc"]; ok {
				response["jsonrpc"] = version
			} else {
				if version, ok := request["version"]; ok {
					response["version"] = version
				}
				response["result"] = nil
				response["error"] = nil
			}
			responses[i] = response
			if method, ok := request["method"].(string); ok && method != "" {
				writer.WriteByte(io.TagCall)
				writer.WriteString(method)
				if params, ok := request["params"].([]interface{}); ok && params != nil && len(params) > 0 {
					writer.Serialize(params)
				}
			}
		}
		writer.WriteByte(io.TagEnd)
		data = writer.Bytes()
		context.SetInterface("jsonrpc", responses)
	}
	return data
}

// OutputFilter for JSONRPC Service
func (filter ServiceFilter) OutputFilter(data []byte, context rpc.Context) []byte {
	responses, ok := context.GetInterface("jsonrpc").([]map[string]interface{})
	if ok && responses != nil {
		reader := io.NewReader(data, false)
		reader.JSONCompatible = true
		tag, _ := reader.ReadByte()
		for _, response := range responses {
			if tag == io.TagResult {
				reader.Reset()
				var result interface{}
				reader.Unserialize(&result)
				response["result"] = result
				tag, _ = reader.ReadByte()
			} else if tag == io.TagError {
				reader.Reset()
				err := make(map[string]interface{})
				err["code"] = -1
				message := reader.ReadString()
				err["message"] = message
				tag, _ = reader.ReadByte()
				response["error"] = err
			}
			if tag == io.TagEnd {
				break
			}
		}
		if len(responses) == 1 {
			data, _ = json.Marshal(responses[0])
		} else {
			data, _ = json.Marshal(responses)
		}
	}
	return data
}
<commit_msg>Improved jsonrpc service filter<commit_after>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * rpc/service_filter.go                                  *
 *                                                        *
 * hprose service filter for Go.                          *
 *                                                        *
 * LastModified: Oct 29, 2016                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package jsonrpc

import (
	"encoding/json"

	"github.com/hprose/hprose-golang/io"
	"github.com/hprose/hprose-golang/rpc"
)

// ServiceFilter is a JSONRPC Service Filter
type ServiceFilter struct{}

func createResponse(request map[string]interface{}) (response map[string]interface{}) {
	response = make(map[string]interface{})
	if id, ok := request["id"]; ok {
		response["id"] = id
	} else {
		response["id"] = nil
	}
	if version, ok := request["jsonrpc"]; ok {
		response["jsonrpc"] = version
	} else {
		if version, ok := request["version"]; ok {
			response["version"] = version
		}
		response["result"] = nil
		response["error"] = nil
	}
	return
}

// InputFilter for JSONRPC Service
func (filter ServiceFilter) InputFilter(data []byte, context rpc.Context) []byte {
	if (len(data) > 0) && (data[0] == '[' || data[0] == '{') {
		var requests []map[string]interface{}
		if data[0] == '[' {
			if err := json.Unmarshal(data, &requests); err != nil {
				return data
			}
		} else {
			requests = make([]map[string]interface{}, 1)
			if err := json.Unmarshal(data, &requests[0]); err != nil {
				return data
			}
		}
		writer := io.NewWriter(true)
		n := len(requests)
		responses := make([]map[string]interface{}, n)
		for i, request := range requests {
			responses[i] = createResponse(request)
			if method, ok := request["method"].(string); ok && method != "" {
				writer.WriteByte(io.TagCall)
				writer.WriteString(method)
				if params, ok := request["params"].([]interface{}); ok && params != nil && len(params) > 0 {
					writer.Serialize(params)
				}
			}
		}
		writer.WriteByte(io.TagEnd)
		data = writer.Bytes()
		context.SetInterface("jsonrpc", responses)
	}
	return data
}

// OutputFilter for JSONRPC Service
func (filter ServiceFilter) OutputFilter(data []byte, context rpc.Context) []byte {
	responses, ok := context.GetInterface("jsonrpc").([]map[string]interface{})
	if ok && responses != nil {
		reader := io.NewReader(data, false)
		reader.JSONCompatible = true
		tag, _ := reader.ReadByte()
		for _, response := range responses {
			if tag == io.TagResult {
				reader.Reset()
				var result interface{}
				reader.Unserialize(&result)
				response["result"] = result
				tag, _ = reader.ReadByte()
			} else if tag == io.TagError {
				reader.Reset()
				err := make(map[string]interface{})
				err["code"] = -1
				message := reader.ReadString()
				err["message"] = message
				tag, _ = reader.ReadByte()
				response["error"] = err
			}
			if tag == io.TagEnd {
				break
			}
		}
		if len(responses) == 1 {
			data, _ = json.Marshal(responses[0])
		} else {
			data, _ = json.Marshal(responses)
		}
	}
	return data
}
<|endoftext|>


<commit_before>package rpc

import (
	"fmt"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"xd/lib/bittorrent/swarm"
	"xd/lib/config"
	"xd/lib/log"
	"xd/lib/rpc"
	"xd/lib/util"
)

var formatRate = util.FormatRate

func Run() {
	var args []string
	cmd := "list"
	fname := "torrents.ini"
	if len(os.Args) > 1 {
		cmd = os.Args[1]
		args = os.Args[2:]
	}
	cfg := new(config.Config)
	err := cfg.Load(fname)
	if err != nil {
		log.Errorf("error: %s", err)
		return
	}
	log.SetLevel(cfg.Log.Level)
	u := url.URL{
		Scheme: "http",
		Host:   cfg.RPC.Bind,
		Path:   rpc.RPCPath,
	}
	c := rpc.NewClient(u.String())

	switch strings.ToLower(cmd) {
	case "list":
		listTorrents(c)
	case "add":
		addTorrents(c, args...)
	case "set-piece-window":
		setPieceWindow(c, args[0])
	}
}

func setPieceWindow(c *rpc.Client, str string) {
	n, err := strconv.Atoi(str)
	if err != nil {
		log.Fatalf("error: %s", err.Error())
	}
	c.SetPieceWindow(n)
}

func addTorrents(c *rpc.Client, urls ...string) {
	for idx := range urls {
		fmt.Printf("fetch %s", urls[idx])
		c.AddTorrent(urls[idx])
	}
}

func listTorrents(c *rpc.Client) {
	var err error
	var list swarm.TorrentsList
	list, err = c.ListTorrents()
	if err != nil {
		log.Errorf("rpc error: %s", err)
		return
	}
	var globalTx, globalRx float64

	var torrents swarm.TorrentStatusList
	sort.Stable(&list.Infohashes)

	for _, ih := range list.Infohashes {
		var status swarm.TorrentStatus
		status, err = c.SwarmStatus(ih)

		if err != nil {
			log.Errorf("rpc error: %s", err)
			return
		}

		torrents = append(torrents, status)

	}
	sort.Stable(&torrents)
	for _, status := range torrents {
		var tx, rx float64
		fmt.Printf("%s [%s] %s\n", status.Name, status.Infohash, status.Bitfield.Percent())
		fmt.Println("peers:")
		sort.Stable(&status.Peers)
		for _, peer := range status.Peers {
			pad := peer.ID

			for len(pad) < 65 {
				pad += " "
			}
			fmt.Printf("\t%stx=%s rx=%s\n", pad, formatRate(peer.TX), formatRate(peer.RX))
			tx += peer.TX
			rx += peer.RX
		}
		fmt.Printf("%s tx=%s rx=%s\n", status.State, formatRate(tx), formatRate(rx))
		fmt.Println("files:")
		for idx, f := range status.Files {
			fmt.Printf("\t[%d] %s (%s)\n", idx, f.FileInfo.Path.FilePath(), f.Progress.Percent())
		}
		fmt.Println()
		globalRx += rx
		globalTx += tx
	}
	fmt.Println()
	fmt.Printf("%d torrents: tx=%s rx=%s\n", list.Infohashes.Len(), formatRate(globalTx), formatRate(globalRx))
}
<commit_msg>leftpad rate in rpc tool<commit_after>package rpc

import (
	"fmt"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"xd/lib/bittorrent/swarm"
	"xd/lib/config"
	"xd/lib/log"
	"xd/lib/rpc"
	"xd/lib/util"
)

func formatRate(r float64) string {
	str := util.FormatRate(r)
	for len(str) < 12 {
		str += " "
	}
	return str
}

func Run() {
	var args []string
	cmd := "list"
	fname := "torrents.ini"
	if len(os.Args) > 1 {
		cmd = os.Args[1]
		args = os.Args[2:]
	}
	cfg := new(config.Config)
	err := cfg.Load(fname)
	if err != nil {
		log.Errorf("error: %s", err)
		return
	}
	log.SetLevel(cfg.Log.Level)
	u := url.URL{
		Scheme: "http",
		Host:   cfg.RPC.Bind,
		Path:   rpc.RPCPath,
	}
	c := rpc.NewClient(u.String())

	switch strings.ToLower(cmd) {
	case "list":
		listTorrents(c)
	case "add":
		addTorrents(c, args...)
	case "set-piece-window":
		setPieceWindow(c, args[0])
	}
}

func setPieceWindow(c *rpc.Client, str string) {
	n, err := strconv.Atoi(str)
	if err != nil {
		log.Fatalf("error: %s", err.Error())
	}
	c.SetPieceWindow(n)
}

func addTorrents(c *rpc.Client, urls ...string) {
	for idx := range urls {
		fmt.Printf("fetch %s", urls[idx])
		c.AddTorrent(urls[idx])
	}
}

func listTorrents(c *rpc.Client) {
	var err error
	var list swarm.TorrentsList
	list, err = c.ListTorrents()
	if err != nil {
		log.Errorf("rpc error: %s", err)
		return
	}
	var globalTx, globalRx float64

	var torrents swarm.TorrentStatusList
	sort.Stable(&list.Infohashes)

	for _, ih := range list.Infohashes {
		var status swarm.TorrentStatus
		status, err = c.SwarmStatus(ih)

		if err != nil {
			log.Errorf("rpc error: %s", err)
			return
		}

		torrents = append(torrents, status)

	}
	sort.Stable(&torrents)
	for _, status := range torrents {
		var tx, rx float64
		fmt.Printf("%s [%s] %s\n", status.Name, status.Infohash, status.Bitfield.Percent())
		fmt.Println("peers:")
		sort.Stable(&status.Peers)
		for _, peer := range status.Peers {
			pad := peer.ID

			for len(pad) < 65 {
				pad += " "
			}
			fmt.Printf("\t%stx=%s rx=%s\n", pad, formatRate(peer.TX), formatRate(peer.RX))
			tx += peer.TX
			rx += peer.RX
		}
		fmt.Printf("%s tx=%s rx=%s\n", status.State, formatRate(tx), formatRate(rx))
		fmt.Println("files:")
		for idx, f := range status.Files {
			fmt.Printf("\t[%d] %s (%s)\n", idx, f.FileInfo.Path.FilePath(), f.Progress.Percent())
		}
		fmt.Println()
		globalRx += rx
		globalTx += tx
	}
	fmt.Println()
	fmt.Printf("%d torrents: tx=%s rx=%s\n", list.Infohashes.Len(), formatRate(globalTx), formatRate(globalRx))
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net/http"

	"github.com/Xe/middleware"
	"github.com/codegangsta/negroni"
	"github.com/drone/routes"
	"github.com/goincremental/negroni-sessions"
	"stevenbooru.cf/csrf"
	"stevenbooru.cf/eye"
	. "stevenbooru.cf/globals"
	"stevenbooru.cf/models"
)

func main() {
	mux := routes.New()

	mux.Get("/", func(rw http.ResponseWriter, r *http.Request) {
		eye.DoTemplate("index", rw, r, nil)
	})

	mux.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
		tok := csrf.SetToken(r)
		eye.DoTemplate("users/login", rw, r, tok)
	})

	mux.Get("/register", func(rw http.ResponseWriter, r *http.Request) {
		tok := csrf.SetToken(r)
		eye.DoTemplate("users/register", rw, r, tok)
	})

	mux.Post("/register", func(rw http.ResponseWriter, r *http.Request) {
		err := r.ParseForm()

		if err != nil {
			panic(err)
		}

		tok := r.PostForm.Get("token")
		if !csrf.CheckToken(tok, r) {
			eye.HandleError(rw, r, errors.New("Invalid CSRF token"))
			return
		}

		_, err = models.NewUser(r.PostForm)
		if err != nil {
			eye.HandleError(rw, r, err)
		}
	})

	n := negroni.Classic()

	n.Use(sessions.Sessions("stevenbooru", CookieStore))
	middleware.Inject(n)
	n.UseHandler(mux)

	n.Run(fmt.Sprintf("%s:%s", Config.HTTP.Bindhost, Config.HTTP.Port))
}
<commit_msg>stevenbooru: set the active user UID on register<commit_after>package main

import (
	"errors"
	"fmt"
	"net/http"

	"github.com/Xe/middleware"
	"github.com/codegangsta/negroni"
	"github.com/drone/routes"
	"github.com/goincremental/negroni-sessions"
	"stevenbooru.cf/csrf"
	"stevenbooru.cf/eye"
	. "stevenbooru.cf/globals"
	"stevenbooru.cf/models"
)

func main() {
	mux := routes.New()

	mux.Get("/", func(rw http.ResponseWriter, r *http.Request) {
		eye.DoTemplate("index", rw, r, nil)
	})

	mux.Get("/login", func(rw http.ResponseWriter, r *http.Request) {
		tok := csrf.SetToken(r)
		eye.DoTemplate("users/login", rw, r, tok)
	})

	mux.Get("/register", func(rw http.ResponseWriter, r *http.Request) {
		tok := csrf.SetToken(r)
		eye.DoTemplate("users/register", rw, r, tok)
	})

	mux.Post("/register", func(rw http.ResponseWriter, r *http.Request) {
		err := r.ParseForm()

		if err != nil {
			panic(err)
		}

		sess := sessions.GetSession(r)

		tok := r.PostForm.Get("token")
		if !csrf.CheckToken(tok, r) {
			eye.HandleError(rw, r, errors.New("Invalid CSRF token"))
			return
		}

		u, err := models.NewUser(r.PostForm)
		if err != nil {
			eye.HandleError(rw, r, err)
		}

		sess.Set("uid", u.UUID)
	})

	n := negroni.Classic()

	n.Use(sessions.Sessions("stevenbooru", CookieStore))
	middleware.Inject(n)
	n.UseHandler(mux)

	n.Run(fmt.Sprintf("%s:%s", Config.HTTP.Bindhost, Config.HTTP.Port))
}
<|endoftext|>


<commit_before>package plugin

import (
	"zvr/server"
	"fmt"
	"zvr/utils"
	"strings"
	log "github.com/Sirupsen/logrus"
)

const (
	VR_CREATE_EIP = "/createeip"
	VR_REMOVE_EIP = "/removeeip"
	VR_SYNC_EIP = "/synceip"
)

type eipInfo struct {
	VipIp string `json:"vipIp"`
	PrivateMac string `json:"privateMac"`
	GuestIp string `json:"guestIp"`
	PublicMac string `json:"publicMac"`
	SnatInboundTraffic bool `json:"snatInboundTraffic"`
}

type setEipCmd struct {
	Eip eipInfo `json:"eip"`
}

type removeEipCmd struct {
	Eip eipInfo `json:"eip"`
}

type syncEipCmd struct {
	Eips []eipInfo `json:"eips"`
}

var EIP_SNAT_START_RULE_NUM = 5000

func makeEipDescription(info eipInfo) string {
	return fmt.Sprintf("EIP-%v-%v-%v", info.VipIp, info.GuestIp, info.PrivateMac)
}

func makeEipDescriptionReg(info eipInfo) string {
	return fmt.Sprintf("^EIP-%v-", info.VipIp)
}

func makeEipDescriptionForPrivateMac(info eipInfo) string {
	return fmt.Sprintf("EIP-%v-%v-%v-private", info.VipIp, info.GuestIp, info.PrivateMac)
}

func cleanupOldEip(tree *server.VyosConfigTree, eip eipInfo) {
	desReg := makeEipDescriptionReg(eip)
	for i := 0; i < 1; {
		if r := tree.FindSnatRuleDescriptionRegex(desReg, utils.StringRegCompareFn); r != nil {
			r.Delete()
		} else {
			break
		}
	}
	for i := 0; i < 1; {
		if r := tree.FindDnatRuleDescriptionRegex(desReg, utils.StringRegCompareFn); r != nil {
			r.Delete()
		} else {
			break
		}
	}
	if nics, nicErr := utils.GetAllNics(); nicErr == nil {
		for _, val := range nics {
			for i := 0; i < 1; {
				if r := tree.FindFirewallRuleByDescriptionRegex(val.Name, "in", desReg, utils.StringRegCompareFn); r != nil {
					r.Delete()
				} else {
					break
				}
			}
		}
	}
}

func setEip(tree *server.VyosConfigTree, eip eipInfo) {
	des := makeEipDescription(eip)
	priDes := makeEipDescriptionForPrivateMac(eip)
	nicname, err := utils.GetNicNameByIp(eip.VipIp)
	if err != nil && eip.PublicMac != "" {
		var nicname string
		err = utils.Retry(func() error {
			var e error
			nicname, e = utils.GetNicNameByMac(eip.PublicMac)
			if e != nil {
				return e
			} else {
				return nil
			}
		}, 5, 1)
	}
	utils.PanicOnError(err)

	prinicname, err := utils.GetNicNameByMac(eip.PrivateMac); utils.PanicOnError(err)

	/* delete old rule in case deleted failed when delete EIP */
	cleanupOldEip(tree, eip)

	if r := tree.FindSnatRuleDescription(des); r == nil {
		tree.SetSnat(
			fmt.Sprintf("description %v", des),
			fmt.Sprintf("outbound-interface %v", nicname),
			fmt.Sprintf("source address %v", eip.GuestIp),
			fmt.Sprintf("translation address %v", eip.VipIp),
		)
	}

	if r := tree.FindSnatRuleDescription(priDes); r == nil {
		tree.SetSnat(
			fmt.Sprintf("description %v", priDes),
			fmt.Sprintf("outbound-interface %v", prinicname),
			fmt.Sprintf("source address %v", eip.GuestIp),
			fmt.Sprintf("translation address %v", eip.VipIp),
		)
	}

	if r := tree.FindDnatRuleDescription(des); r == nil {
		tree.SetDnat(
			fmt.Sprintf("description %v", des),
			fmt.Sprintf("inbound-interface any"),
			fmt.Sprintf("destination address %v", eip.VipIp),
			fmt.Sprintf("translation address %v", eip.GuestIp),
		)
	}

	if r := tree.FindFirewallRuleByDescription(nicname, "in", des); r == nil {
		tree.SetFirewallOnInterface(nicname, "in",
			fmt.Sprintf("description %v", des),
			fmt.Sprintf("destination address %v", eip.GuestIp),
			"state new enable",
			"state established enable",
			"state related enable",
			"action accept",
		)

		tree.AttachFirewallToInterface(nicname, "in")
	}

	if r := tree.FindFirewallRuleByDescription(prinicname, "in", des); r == nil {
		tree.SetFirewallOnInterface(prinicname, "in",
			fmt.Sprintf("description %v", des),
			fmt.Sprintf("source address %v", eip.GuestIp),
			"state new enable",
			"state established enable",
			"state related enable",
			"action accept",
		)

		tree.AttachFirewallToInterface(prinicname, "in")
	}
}

func checkEipExists(eip eipInfo) error {
	tree := server.NewParserFromShowConfiguration().Tree
	des := makeEipDescription(eip)
	priDes := makeEipDescriptionForPrivateMac(eip)

	if r := tree.FindSnatRuleDescription(des); r != nil {
		return fmt.Errorf("%s snat deletion fail", des)
	}

	if r := tree.FindSnatRuleDescription(priDes); r != nil {
		return fmt.Errorf("%s snat deletion fail", priDes)
	}

	if r := tree.FindDnatRuleDescription(des); r != nil {
		return fmt.Errorf("%s dnat deletion fail", des)
	}

	log.Debugf("checkEipExists %v des %s priDes %s successfuuly deleted", eip, des, priDes)

	return nil
}

func deleteEip(tree *server.VyosConfigTree, eip eipInfo) {
	des := makeEipDescription(eip)
	priDes := makeEipDescriptionForPrivateMac(eip)
	nicname, err := utils.GetNicNameByIp(eip.VipIp)
	if err != nil && eip.PublicMac != "" {
		var nicname string
		err = utils.Retry(func() error {
			var e error
			nicname, e = utils.GetNicNameByMac(eip.PublicMac)
			if e != nil {
				return e
			} else {
				return nil
			}
		}, 5, 1)
	}
	utils.PanicOnError(err)

	if r := tree.FindSnatRuleDescription(des); r != nil {
		r.Delete()
	}

	if r := tree.FindSnatRuleDescription(priDes); r != nil {
		r.Delete()
	}

	if r := tree.FindDnatRuleDescription(des); r != nil {
		r.Delete()
	}

	if r := tree.FindFirewallRuleByDescription(nicname, "in", des); r != nil {
		r.Delete()
	}

	prinicname, err := utils.GetNicNameByMac(eip.PrivateMac); utils.PanicOnError(err)
	if r := tree.FindFirewallRuleByDescription(prinicname, "in", des); r != nil {
		r.Delete()
	}
}

func createEip(ctx *server.CommandContext) interface{} {
	cmd := &setEipCmd{}
	ctx.GetCommand(cmd)
	eip := cmd.Eip

	tree := server.NewParserFromShowConfiguration().Tree
	setEip(tree, eip)
	tree.Apply(false)

	return nil
}

func removeEip(ctx *server.CommandContext) interface{} {
	cmd := &removeEipCmd{}
	ctx.GetCommand(cmd)
	eip := cmd.Eip

	err := utils.Retry(func() error {
                tree := server.NewParserFromShowConfiguration().Tree
                deleteEip(tree, eip)
                tree.Apply(false)

                return checkEipExists(eip);
        }, 3, 1); utils.LogError(err)
	return nil
}

func syncEip(ctx *server.CommandContext) interface{} {
	cmd := &syncEipCmd{}
	ctx.GetCommand(cmd)

	tree := server.NewParserFromShowConfiguration().Tree

	// delete all EIP related rules
	if rs := tree.Get("nat destination rule"); rs != nil {
		for _, r := range rs.Children() {
			if d := r.Get("description"); d != nil && strings.HasPrefix(d.Value(), "EIP") {
				r.Delete()
			}
		}
	}

	if rs := tree.Getf("nat source rule"); rs != nil {
		for _, r := range rs.Children() {
			if d := r.Get("description"); d != nil && strings.HasPrefix(d.Value(), "EIP") {
				r.Delete()
			}
		}
	}

	if rs := tree.Getf("firewall name"); rs != nil {
		for _, r := range rs.Children() {
			if rss := r.Get("rule"); rss != nil {
				for _, rr := range rss.Children() {
					if d := rr.Get("description"); d != nil && strings.HasPrefix(d.Value(), "EIP") {
						rr.Delete()
					}
				}
			}
		}
	}

	for _, eip := range cmd.Eips {
		setEip(tree, eip)
	}

	tree.Apply(false)

	return nil
}

func EipEntryPoint() {
	server.RegisterAsyncCommandHandler(VR_CREATE_EIP, server.VyosLock(createEip))
	server.RegisterAsyncCommandHandler(VR_REMOVE_EIP, server.VyosLock(removeEip))
	server.RegisterAsyncCommandHandler(VR_SYNC_EIP, server.VyosLock(syncEip))
}
<commit_msg>Avoid empty nic name in set eip<commit_after>package plugin

import (
	"zvr/server"
	"fmt"
	"zvr/utils"
	"strings"
	log "github.com/Sirupsen/logrus"
)

const (
	VR_CREATE_EIP = "/createeip"
	VR_REMOVE_EIP = "/removeeip"
	VR_SYNC_EIP = "/synceip"
)

type eipInfo struct {
	VipIp string `json:"vipIp"`
	PrivateMac string `json:"privateMac"`
	GuestIp string `json:"guestIp"`
	PublicMac string `json:"publicMac"`
	SnatInboundTraffic bool `json:"snatInboundTraffic"`
}

type setEipCmd struct {
	Eip eipInfo `json:"eip"`
}

type removeEipCmd struct {
	Eip eipInfo `json:"eip"`
}

type syncEipCmd struct {
	Eips []eipInfo `json:"eips"`
}

var EIP_SNAT_START_RULE_NUM = 5000

func makeEipDescription(info eipInfo) string {
	return fmt.Sprintf("EIP-%v-%v-%v", info.VipIp, info.GuestIp, info.PrivateMac)
}

func makeEipDescriptionReg(info eipInfo) string {
	return fmt.Sprintf("^EIP-%v-", info.VipIp)
}

func makeEipDescriptionForPrivateMac(info eipInfo) string {
	return fmt.Sprintf("EIP-%v-%v-%v-private", info.VipIp, info.GuestIp, info.PrivateMac)
}

func cleanupOldEip(tree *server.VyosConfigTree, eip eipInfo) {
	desReg := makeEipDescriptionReg(eip)
	for i := 0; i < 1; {
		if r := tree.FindSnatRuleDescriptionRegex(desReg, utils.StringRegCompareFn); r != nil {
			r.Delete()
		} else {
			break
		}
	}
	for i := 0; i < 1; {
		if r := tree.FindDnatRuleDescriptionRegex(desReg, utils.StringRegCompareFn); r != nil {
			r.Delete()
		} else {
			break
		}
	}
	if nics, nicErr := utils.GetAllNics(); nicErr == nil {
		for _, val := range nics {
			for i := 0; i < 1; {
				if r := tree.FindFirewallRuleByDescriptionRegex(val.Name, "in", desReg, utils.StringRegCompareFn); r != nil {
					r.Delete()
				} else {
					break
				}
			}
		}
	}
}

func setEip(tree *server.VyosConfigTree, eip eipInfo) {
	des := makeEipDescription(eip)
	priDes := makeEipDescriptionForPrivateMac(eip)
	nicname, err := utils.GetNicNameByIp(eip.VipIp)
	if (nicname == "" || err != nil) && eip.PublicMac != "" {
		var nicname string
		err = utils.Retry(func() error {
			var e error
			nicname, e = utils.GetNicNameByMac(eip.PublicMac)
			if e != nil {
				return e
			} else if nicname == "" {
				return fmt.Errorf("empty nic name found for mac[%s]", eip.PublicMac)
			} else {
				return nil
			}
		}, 5, 1)
	}
	utils.PanicOnError(err)

	prinicname, err := utils.GetNicNameByMac(eip.PrivateMac); utils.PanicOnError(err)

	/* delete old rule in case deleted failed when delete EIP */
	cleanupOldEip(tree, eip)

	if r := tree.FindSnatRuleDescription(des); r == nil {
		tree.SetSnat(
			fmt.Sprintf("description %v", des),
			fmt.Sprintf("outbound-interface %v", nicname),
			fmt.Sprintf("source address %v", eip.GuestIp),
			fmt.Sprintf("translation address %v", eip.VipIp),
		)
	}

	if r := tree.FindSnatRuleDescription(priDes); r == nil {
		tree.SetSnat(
			fmt.Sprintf("description %v", priDes),
			fmt.Sprintf("outbound-interface %v", prinicname),
			fmt.Sprintf("source address %v", eip.GuestIp),
			fmt.Sprintf("translation address %v", eip.VipIp),
		)
	}

	if r := tree.FindDnatRuleDescription(des); r == nil {
		tree.SetDnat(
			fmt.Sprintf("description %v", des),
			fmt.Sprintf("inbound-interface any"),
			fmt.Sprintf("destination address %v", eip.VipIp),
			fmt.Sprintf("translation address %v", eip.GuestIp),
		)
	}

	if r := tree.FindFirewallRuleByDescription(nicname, "in", des); r == nil {
		tree.SetFirewallOnInterface(nicname, "in",
			fmt.Sprintf("description %v", des),
			fmt.Sprintf("destination address %v", eip.GuestIp),
			"state new enable",
			"state established enable",
			"state related enable",
			"action accept",
		)

		tree.AttachFirewallToInterface(nicname, "in")
	}

	if r := tree.FindFirewallRuleByDescription(prinicname, "in", des); r == nil {
		tree.SetFirewallOnInterface(prinicname, "in",
			fmt.Sprintf("description %v", des),
			fmt.Sprintf("source address %v", eip.GuestIp),
			"state new enable",
			"state established enable",
			"state related enable",
			"action accept",
		)

		tree.AttachFirewallToInterface(prinicname, "in")
	}
}

func checkEipExists(eip eipInfo) error {
	tree := server.NewParserFromShowConfiguration().Tree
	des := makeEipDescription(eip)
	priDes := makeEipDescriptionForPrivateMac(eip)

	if r := tree.FindSnatRuleDescription(des); r != nil {
		return fmt.Errorf("%s snat deletion fail", des)
	}

	if r := tree.FindSnatRuleDescription(priDes); r != nil {
		return fmt.Errorf("%s snat deletion fail", priDes)
	}

	if r := tree.FindDnatRuleDescription(des); r != nil {
		return fmt.Errorf("%s dnat deletion fail", des)
	}

	log.Debugf("checkEipExists %v des %s priDes %s successfuuly deleted", eip, des, priDes)

	return nil
}

func deleteEip(tree *server.VyosConfigTree, eip eipInfo) {
	des := makeEipDescription(eip)
	priDes := makeEipDescriptionForPrivateMac(eip)
	nicname, err := utils.GetNicNameByIp(eip.VipIp)
	if err != nil && eip.PublicMac != "" {
		var nicname string
		err = utils.Retry(func() error {
			var e error
			nicname, e = utils.GetNicNameByMac(eip.PublicMac)
			if e != nil {
				return e
			} else {
				return nil
			}
		}, 5, 1)
	}
	utils.PanicOnError(err)

	if r := tree.FindSnatRuleDescription(des); r != nil {
		r.Delete()
	}

	if r := tree.FindSnatRuleDescription(priDes); r != nil {
		r.Delete()
	}

	if r := tree.FindDnatRuleDescription(des); r != nil {
		r.Delete()
	}

	if r := tree.FindFirewallRuleByDescription(nicname, "in", des); r != nil {
		r.Delete()
	}

	prinicname, err := utils.GetNicNameByMac(eip.PrivateMac); utils.PanicOnError(err)
	if r := tree.FindFirewallRuleByDescription(prinicname, "in", des); r != nil {
		r.Delete()
	}
}

func createEip(ctx *server.CommandContext) interface{} {
	cmd := &setEipCmd{}
	ctx.GetCommand(cmd)
	eip := cmd.Eip

	tree := server.NewParserFromShowConfiguration().Tree
	setEip(tree, eip)
	tree.Apply(false)

	return nil
}

func removeEip(ctx *server.CommandContext) interface{} {
	cmd := &removeEipCmd{}
	ctx.GetCommand(cmd)
	eip := cmd.Eip

	err := utils.Retry(func() error {
                tree := server.NewParserFromShowConfiguration().Tree
                deleteEip(tree, eip)
                tree.Apply(false)

                return checkEipExists(eip);
        }, 3, 1); utils.LogError(err)
	return nil
}

func syncEip(ctx *server.CommandContext) interface{} {
	cmd := &syncEipCmd{}
	ctx.GetCommand(cmd)

	tree := server.NewParserFromShowConfiguration().Tree

	// delete all EIP related rules
	if rs := tree.Get("nat destination rule"); rs != nil {
		for _, r := range rs.Children() {
			if d := r.Get("description"); d != nil && strings.HasPrefix(d.Value(), "EIP") {
				r.Delete()
			}
		}
	}

	if rs := tree.Getf("nat source rule"); rs != nil {
		for _, r := range rs.Children() {
			if d := r.Get("description"); d != nil && strings.HasPrefix(d.Value(), "EIP") {
				r.Delete()
			}
		}
	}

	if rs := tree.Getf("firewall name"); rs != nil {
		for _, r := range rs.Children() {
			if rss := r.Get("rule"); rss != nil {
				for _, rr := range rss.Children() {
					if d := rr.Get("description"); d != nil && strings.HasPrefix(d.Value(), "EIP") {
						rr.Delete()
					}
				}
			}
		}
	}

	for _, eip := range cmd.Eips {
		setEip(tree, eip)
	}

	tree.Apply(false)

	return nil
}

func EipEntryPoint() {
	server.RegisterAsyncCommandHandler(VR_CREATE_EIP, server.VyosLock(createEip))
	server.RegisterAsyncCommandHandler(VR_REMOVE_EIP, server.VyosLock(removeEip))
	server.RegisterAsyncCommandHandler(VR_SYNC_EIP, server.VyosLock(syncEip))
}
<|endoftext|>


<commit_before>package starbound

import (
	"bytes"
	"encoding/binary"
	"io"
)

type IndexEntry struct {
	Offset, Size int64
}

func NewSBAsset6(r io.ReaderAt) (a *SBAsset6, err error) {
	buf := make([]byte, 16)
	// Read the initial header which points at a metadata section.
	_, err = r.ReadAt(buf, 0)
	if err != nil {
		return
	}
	if !bytes.Equal(buf[:8], []byte("SBAsset6")) {
		return nil, ErrInvalidData
	}
	a = &SBAsset6{r: r}
	a.metadata = int64(binary.BigEndian.Uint64(buf[8:]))
	// Read the metadata section of the asset file.
	buf5 := buf[:5]
	_, err = r.ReadAt(buf5, a.metadata)
	if err != nil {
		return
	}
	if !bytes.Equal(buf5, []byte("INDEX")) {
		return nil, ErrInvalidData
	}
	rr := &readerAtReader{r: r, off: a.metadata + 5}
	a.Metadata, err = ReadMap(rr)
	if err != nil {
		return
	}
	c, err := ReadVaruint(rr)
	if err != nil {
		return
	}
	a.FileCount = int(c)
	a.index = rr.off
	return
}

type SBAsset6 struct {
	FileCount int
	Index     map[string]IndexEntry
	Metadata  map[string]interface{}

	r        io.ReaderAt
	metadata int64
	index    int64
}

func (a *SBAsset6) GetReader(path string) (r io.Reader, err error) {
	entry, ok := a.Index[path]
	if !ok {
		return nil, ErrKeyNotFound
	}
	return io.NewSectionReader(a.r, entry.Offset, entry.Size), nil
}

func (a *SBAsset6) ReadIndex() error {
	a.Index = make(map[string]IndexEntry, a.FileCount)
	buf := make([]byte, 255)
	r := &readerAtReader{r: a.r, off: a.index}
	for i := 0; i < a.FileCount; i++ {
		// Read the path, which will be used as the index key.
		key, err := ReadString(r)
		if err != nil {
			return err
		}
		// Read the offset and size of the file.
		_, err = r.Read(buf[:16])
		if err != nil {
			return err
		}
		var entry IndexEntry
		entry.Offset = int64(binary.BigEndian.Uint64(buf))
		entry.Size = int64(binary.BigEndian.Uint64(buf[8:]))
		// Update the map.
		a.Index[key] = entry
	}
	return nil
}
<commit_msg>Only allocate 16 bytes in buffer for offset and size<commit_after>package starbound

import (
	"bytes"
	"encoding/binary"
	"io"
)

type IndexEntry struct {
	Offset, Size int64
}

func NewSBAsset6(r io.ReaderAt) (a *SBAsset6, err error) {
	buf := make([]byte, 16)
	// Read the initial header which points at a metadata section.
	_, err = r.ReadAt(buf, 0)
	if err != nil {
		return
	}
	if !bytes.Equal(buf[:8], []byte("SBAsset6")) {
		return nil, ErrInvalidData
	}
	a = &SBAsset6{r: r}
	a.metadata = int64(binary.BigEndian.Uint64(buf[8:]))
	// Read the metadata section of the asset file.
	buf5 := buf[:5]
	_, err = r.ReadAt(buf5, a.metadata)
	if err != nil {
		return
	}
	if !bytes.Equal(buf5, []byte("INDEX")) {
		return nil, ErrInvalidData
	}
	rr := &readerAtReader{r: r, off: a.metadata + 5}
	a.Metadata, err = ReadMap(rr)
	if err != nil {
		return
	}
	c, err := ReadVaruint(rr)
	if err != nil {
		return
	}
	a.FileCount = int(c)
	a.index = rr.off
	return
}

type SBAsset6 struct {
	FileCount int
	Index     map[string]IndexEntry
	Metadata  map[string]interface{}

	r        io.ReaderAt
	metadata int64
	index    int64
}

func (a *SBAsset6) GetReader(path string) (r io.Reader, err error) {
	entry, ok := a.Index[path]
	if !ok {
		return nil, ErrKeyNotFound
	}
	return io.NewSectionReader(a.r, entry.Offset, entry.Size), nil
}

func (a *SBAsset6) ReadIndex() error {
	a.Index = make(map[string]IndexEntry, a.FileCount)
	buf := make([]byte, 16)
	r := &readerAtReader{r: a.r, off: a.index}
	for i := 0; i < a.FileCount; i++ {
		// Read the path, which will be used as the index key.
		key, err := ReadString(r)
		if err != nil {
			return err
		}
		// Read the offset and size of the file.
		_, err = r.Read(buf)
		if err != nil {
			return err
		}
		var entry IndexEntry
		entry.Offset = int64(binary.BigEndian.Uint64(buf))
		entry.Size = int64(binary.BigEndian.Uint64(buf[8:]))
		// Update the map.
		a.Index[key] = entry
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/42wim/tail"
	"github.com/Sirupsen/logrus"
	"github.com/hashicorp/golang-lru"
	"github.com/oschwald/geoip2-golang"
	_ "github.com/pkg/profile"
	"github.com/pquerna/ffjson/ffjson"
	"net"
	"os"
	"runtime"
	"time"
)

var flagPrimary, flagQueue, flagBackup, flagTailFile string
var geoipDB *geoip2.Reader
var geoipCache *lru.Cache
var cfg *Config
var nrCPU = runtime.GOMAXPROCS(-1)
var log = logrus.New()

const nfLayout = "2006-01-02T15:04:05.999999999"

var myLocation *time.Location

func parseLine(line *[]byte) {
	var f nf
	var t time.Time
	var realRegionName, regionName string
	var record *geoip2.City

	err := ffjson.Unmarshal(*line, &f)
	if err != nil {
		*line = []byte("")
		log.Error(err, "couldn't unmarshal ", *line)
		return
	}

	// use LRU cache
	if val, ok := geoipCache.Get(*f.Srcip); ok {
		record = val.(*geoip2.City)
	} else {
		ip := net.ParseIP(*f.Srcip)
		record, _ = geoipDB.City(ip)
		geoipCache.Add(*f.Srcip, record)
	}

	// add @timestamp with zulu (ISO8601 time)
	t, _ = time.ParseInLocation(nfLayout, *f.Timestamp, myLocation)
	f.Ltimestamp = t.UTC().Format(time.RFC3339Nano)

	if record.Location.Longitude != 0 && record.Location.Latitude != 0 {
		mylen := len(record.Subdivisions)
		if mylen > 0 {
			mylen--
			realRegionName = record.Subdivisions[mylen].Names["en"]
			regionName = record.Subdivisions[mylen].IsoCode
		}
		f.GeoIP.Longitude = &record.Location.Longitude
		f.GeoIP.Latitude = &record.Location.Latitude
		f.GeoIP.CountryName = record.Country.Names["en"]
		f.GeoIP.Timezone = &record.Location.TimeZone
		f.GeoIP.ContinentCode = &record.Continent.Code
		f.GeoIP.CityName = record.City.Names["en"]
		f.GeoIP.CountryCode2 = &record.Country.IsoCode
		f.GeoIP.RealRegionName = &realRegionName
		f.GeoIP.RegionName = &regionName
		f.GeoIP.IP = f.Srcip
		f.GeoIP.Location = &esGeoIPLocation{f.GeoIP.Longitude, f.GeoIP.Latitude}
		f.GeoIP.Coordinates = f.GeoIP.Location
	}

	*line, _ = ffjson.Marshal(f)
}

func parseLineWorker(ctx *Context) {
	bt0 := time.Now()
	show := true
	for {
		select {
		case line := <-ctx.lines:
			myline := []byte(*line)
			parseLine(&myline)
			// primary buffer full, send to backup
			if len(ctx.parsedLines) > ctx.cfg.General.Buffer*90/100 {
				ctx.buffering = true
				if show {
					log.Info("primary isn't fast enough, buffer full, buffering to backup")
					show = false
				}
				// only log message every 5 seconds
				if time.Since(bt0).Seconds() > 5 {
					show = true
					bt0 = time.Now()
				}
				ctx.backupLines <- &myline
			} else {
				ctx.buffering = false
				ctx.parsedLines <- &myline
			}
		case <-time.After(time.Second * 3):
			ctx.buffering = false
		}
	}
}

func doTask(ctx *Context) {
	switch flagPrimary {
	case "redis", "rabbit":
		go doPrimaryTask(ctx, flagPrimary)
	case "es": // es is pretty slow, start multiple
		log.Info("backend:starting ", ctx.cfg.Backend["es"].Workers, " ES backends")
		for i := 0; i < ctx.cfg.Backend["es"].Workers; i++ {
			go doPrimaryTask(ctx, "es")
		}
	}
	if flagBackup != "" {
		go doBackupTask(ctx, flagBackup)
	}
}

func rateLogger(ctx *Context) {
	parsedRatecount := 0
	backupRatecount := 0
	parsedTotalcount := 0
	backupTotalcount := 0
	pt0 := time.Now()
	pt1 := time.Now()

	bt0 := time.Now()
	bt1 := time.Now()
	ticker := time.NewTicker(5 * time.Second)
	for {
		select {
		case <-ctx.parsedRate:
			parsedRatecount++
			parsedTotalcount++
			if time.Since(pt0).Seconds() > 5 {
				log.Info("primary: total: ", parsedTotalcount,
					" rate: ", int(float64(parsedRatecount)/float64(time.Since(pt0).Seconds())), "/s",
					" avg rate: ", int(float64(parsedTotalcount)/float64(time.Since(pt1).Seconds())), "/s",
					" buffer: ", len(ctx.parsedLines))
				pt0 = time.Now()
				parsedRatecount = 0
			}
		case <-ctx.backupRate:
			backupRatecount++
			backupTotalcount++
			if time.Since(bt0).Seconds() > 5 {
				log.Info("backup: total: ", backupTotalcount,
					" rate: ", int(float64(backupRatecount)/float64(time.Since(bt0).Seconds())), "/s",
					" avg rate: ", int(float64(backupTotalcount)/float64(time.Since(bt1).Seconds())), "/s",
					" buffer: ", len(ctx.parsedLines))
				bt0 = time.Now()
				backupRatecount = 0
			}
		case <-ticker.C:
			//log.Debug(backupRatecount, ":", ctx.backupRateInt)
			if backupRatecount == ctx.backupRateInt {
				//log.Debug("ratelogger: sending restoreStart")
				ctx.restoreStart <- "ratelogger"
			} else {
				ctx.backupRateInt = backupRatecount
			}
		}
	}
}

func tailUlog(ctx *Context) {
	logfile := flagTailFile
	t, err := tail.TailFile(logfile, tail.Config{Poll: true, Follow: true, ReOpen: true, Pipe: true})
	if err != nil {
		log.Error(err)
	}

	// create the workers. line goes in, parsed line goes out
	for i := 0; i < nrCPU/2; i++ {
		go parseLineWorker(ctx)
	}
	// do primary and backup tasks
	go doTask(ctx)

	//show some stats
	go rateLogger(ctx)

	for line := range t.Lines {
		ctx.lines <- &line.Text
	}
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func init() {
	var flagDebug bool
	var flagConfig string
	flag.StringVar(&flagPrimary, "primary", "", "name of primary backend (es/redis/rabbit)")
	flag.StringVar(&flagBackup, "backup", "", "name of backup backend (disk/redis/rabbit)")
	flag.StringVar(&flagConfig, "conf", "ulog2queue.cfg", "config file")
	flag.StringVar(&flagTailFile, "tail", "", "file to tail")
	flag.BoolVar(&flagDebug, "debug", false, "enable debug")
	hostname, _ := os.Hostname()
	flag.StringVar(&flagQueue, "queue", hostname, "name of queue")
	log.Level = logrus.InfoLevel
	flag.Parse()
	if flagDebug {
		log.Println("enabling debug")
		log.Level = logrus.DebugLevel
	}
	cfg = NewConfig(flagConfig)
	if flagPrimary == "" {
		flagPrimary = cfg.General.Primary
	}
	if flagBackup == "" {
		flagBackup = cfg.General.Backup
	}
	if flagTailFile == "" {
		flagTailFile = cfg.General.TailFile
	}
	myLocation, _ = time.LoadLocation("Local")
	geoipCache, _ = lru.New(10000)
}

func main() {
	var err error
	if nrCPU == 1 { // no GOMAXPROCS set
		nrCPU = runtime.NumCPU() / 2
		if nrCPU > 10 {
			nrCPU = 10
		}
		runtime.GOMAXPROCS(nrCPU)
	}
	context := &Context{make(chan *string, 10000),
		make(chan *[]byte, cfg.General.Buffer),
		make(chan *[]byte, 10000),
		make(chan int),
		make(chan int),
		make(chan string),
		make(chan bool),
		make(chan string),
		0,
		false,
		cfg}
	geoipDB, err = geoip2.Open(context.cfg.General.Geoip2db)
	failOnError(err, "can't open geoip db")
	redisPool = NewPool(context)
	tailUlog(context)
}
<commit_msg>Fix nil pointer panic<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/42wim/tail"
	"github.com/Sirupsen/logrus"
	"github.com/hashicorp/golang-lru"
	"github.com/oschwald/geoip2-golang"
	_ "github.com/pkg/profile"
	"github.com/pquerna/ffjson/ffjson"
	"net"
	"os"
	"runtime"
	"time"
)

var flagPrimary, flagQueue, flagBackup, flagTailFile string
var geoipDB *geoip2.Reader
var geoipCache *lru.Cache
var cfg *Config
var nrCPU = runtime.GOMAXPROCS(-1)
var log = logrus.New()

const nfLayout = "2006-01-02T15:04:05.999999999"

var myLocation *time.Location

func parseLine(line *[]byte) {
	var f nf
	var t time.Time
	var realRegionName, regionName string
	var record *geoip2.City

	err := ffjson.Unmarshal(*line, &f)
	if err != nil {
		*line = []byte("")
		log.Error(err, "couldn't unmarshal ", *line)
		return
	}

	if f.Srcip == nil {
		log.Error("Packet without source ip, shouldn't happen: ", *line)
		return
	}

	// use LRU cache
	if val, ok := geoipCache.Get(*f.Srcip); ok {
		record = val.(*geoip2.City)
	} else {
		ip := net.ParseIP(*f.Srcip)
		record, _ = geoipDB.City(ip)
		geoipCache.Add(*f.Srcip, record)
	}

	// add @timestamp with zulu (ISO8601 time)
	t, _ = time.ParseInLocation(nfLayout, *f.Timestamp, myLocation)
	f.Ltimestamp = t.UTC().Format(time.RFC3339Nano)

	if record.Location.Longitude != 0 && record.Location.Latitude != 0 {
		mylen := len(record.Subdivisions)
		if mylen > 0 {
			mylen--
			realRegionName = record.Subdivisions[mylen].Names["en"]
			regionName = record.Subdivisions[mylen].IsoCode
		}
		f.GeoIP.Longitude = &record.Location.Longitude
		f.GeoIP.Latitude = &record.Location.Latitude
		f.GeoIP.CountryName = record.Country.Names["en"]
		f.GeoIP.Timezone = &record.Location.TimeZone
		f.GeoIP.ContinentCode = &record.Continent.Code
		f.GeoIP.CityName = record.City.Names["en"]
		f.GeoIP.CountryCode2 = &record.Country.IsoCode
		f.GeoIP.RealRegionName = &realRegionName
		f.GeoIP.RegionName = &regionName
		f.GeoIP.IP = f.Srcip
		f.GeoIP.Location = &esGeoIPLocation{f.GeoIP.Longitude, f.GeoIP.Latitude}
		f.GeoIP.Coordinates = f.GeoIP.Location
	}

	*line, _ = ffjson.Marshal(f)
}

func parseLineWorker(ctx *Context) {
	bt0 := time.Now()
	show := true
	for {
		select {
		case line := <-ctx.lines:
			myline := []byte(*line)
			parseLine(&myline)
			// primary buffer full, send to backup
			if len(ctx.parsedLines) > ctx.cfg.General.Buffer*90/100 {
				ctx.buffering = true
				if show {
					log.Info("primary isn't fast enough, buffer full, buffering to backup")
					show = false
				}
				// only log message every 5 seconds
				if time.Since(bt0).Seconds() > 5 {
					show = true
					bt0 = time.Now()
				}
				ctx.backupLines <- &myline
			} else {
				ctx.buffering = false
				ctx.parsedLines <- &myline
			}
		case <-time.After(time.Second * 3):
			ctx.buffering = false
		}
	}
}

func doTask(ctx *Context) {
	switch flagPrimary {
	case "redis", "rabbit":
		go doPrimaryTask(ctx, flagPrimary)
	case "es": // es is pretty slow, start multiple
		log.Info("backend:starting ", ctx.cfg.Backend["es"].Workers, " ES backends")
		for i := 0; i < ctx.cfg.Backend["es"].Workers; i++ {
			go doPrimaryTask(ctx, "es")
		}
	}
	if flagBackup != "" {
		go doBackupTask(ctx, flagBackup)
	}
}

func rateLogger(ctx *Context) {
	parsedRatecount := 0
	backupRatecount := 0
	parsedTotalcount := 0
	backupTotalcount := 0
	pt0 := time.Now()
	pt1 := time.Now()

	bt0 := time.Now()
	bt1 := time.Now()
	ticker := time.NewTicker(5 * time.Second)
	for {
		select {
		case <-ctx.parsedRate:
			parsedRatecount++
			parsedTotalcount++
			if time.Since(pt0).Seconds() > 5 {
				log.Info("primary: total: ", parsedTotalcount,
					" rate: ", int(float64(parsedRatecount)/float64(time.Since(pt0).Seconds())), "/s",
					" avg rate: ", int(float64(parsedTotalcount)/float64(time.Since(pt1).Seconds())), "/s",
					" buffer: ", len(ctx.parsedLines))
				pt0 = time.Now()
				parsedRatecount = 0
			}
		case <-ctx.backupRate:
			backupRatecount++
			backupTotalcount++
			if time.Since(bt0).Seconds() > 5 {
				log.Info("backup: total: ", backupTotalcount,
					" rate: ", int(float64(backupRatecount)/float64(time.Since(bt0).Seconds())), "/s",
					" avg rate: ", int(float64(backupTotalcount)/float64(time.Since(bt1).Seconds())), "/s",
					" buffer: ", len(ctx.parsedLines))
				bt0 = time.Now()
				backupRatecount = 0
			}
		case <-ticker.C:
			//log.Debug(backupRatecount, ":", ctx.backupRateInt)
			if backupRatecount == ctx.backupRateInt {
				//log.Debug("ratelogger: sending restoreStart")
				ctx.restoreStart <- "ratelogger"
			} else {
				ctx.backupRateInt = backupRatecount
			}
		}
	}
}

func tailUlog(ctx *Context) {
	logfile := flagTailFile
	t, err := tail.TailFile(logfile, tail.Config{Poll: true, Follow: true, ReOpen: true, Pipe: true})
	if err != nil {
		log.Error(err)
	}

	// create the workers. line goes in, parsed line goes out
	for i := 0; i < nrCPU/2; i++ {
		go parseLineWorker(ctx)
	}
	// do primary and backup tasks
	go doTask(ctx)

	//show some stats
	go rateLogger(ctx)

	for line := range t.Lines {
		ctx.lines <- &line.Text
	}
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func init() {
	var flagDebug bool
	var flagConfig string
	flag.StringVar(&flagPrimary, "primary", "", "name of primary backend (es/redis/rabbit)")
	flag.StringVar(&flagBackup, "backup", "", "name of backup backend (disk/redis/rabbit)")
	flag.StringVar(&flagConfig, "conf", "ulog2queue.cfg", "config file")
	flag.StringVar(&flagTailFile, "tail", "", "file to tail")
	flag.BoolVar(&flagDebug, "debug", false, "enable debug")
	hostname, _ := os.Hostname()
	flag.StringVar(&flagQueue, "queue", hostname, "name of queue")
	log.Level = logrus.InfoLevel
	flag.Parse()
	if flagDebug {
		log.Println("enabling debug")
		log.Level = logrus.DebugLevel
	}
	cfg = NewConfig(flagConfig)
	if flagPrimary == "" {
		flagPrimary = cfg.General.Primary
	}
	if flagBackup == "" {
		flagBackup = cfg.General.Backup
	}
	if flagTailFile == "" {
		flagTailFile = cfg.General.TailFile
	}
	myLocation, _ = time.LoadLocation("Local")
	geoipCache, _ = lru.New(10000)
}

func main() {
	var err error
	if nrCPU == 1 { // no GOMAXPROCS set
		nrCPU = runtime.NumCPU() / 2
		if nrCPU > 10 {
			nrCPU = 10
		}
		runtime.GOMAXPROCS(nrCPU)
	}
	context := &Context{make(chan *string, 10000),
		make(chan *[]byte, cfg.General.Buffer),
		make(chan *[]byte, 10000),
		make(chan int),
		make(chan int),
		make(chan string),
		make(chan bool),
		make(chan string),
		0,
		false,
		cfg}
	geoipDB, err = geoip2.Open(context.cfg.General.Geoip2db)
	failOnError(err, "can't open geoip db")
	redisPool = NewPool(context)
	tailUlog(context)
}
<|endoftext|>


<commit_before>package class

import (
    "strings"
)

type MemberDescriptorParser struct {
    descriptor  string
    offset      int
    md          *MethodDescriptor
}

func newMemberDescriptorParser(descriptor string) (*MemberDescriptorParser) {
    return &MemberDescriptorParser{descriptor: descriptor}
}

func (self *MemberDescriptorParser) parse() (*MethodDescriptor) {
    self.md = &MethodDescriptor{}
    self.startParams()
    self.parseParameterTypes()
    self.endParams()
    self.parseReturnType()
    self.finish()
    return self.md
}

func (self *MemberDescriptorParser) startParams() {
    b := self.readUint8()
    if b != '(' {
        self.causePanic()
    }
}
func (self *MemberDescriptorParser) endParams() {
    b := self.readUint8()
    if b != ')' {
        self.causePanic()
    }
}
func (self *MemberDescriptorParser) finish() {
    if self.offset != len(self.descriptor) {
        self.causePanic()
    }
}

func (self *MemberDescriptorParser) parseParameterTypes() {
    for {
        t := self.readFieldType()
        if t != nil {
            self.md.addParameterType(t)
        } else {
            break
        }
    }
}
func (self *MemberDescriptorParser) parseReturnType() {
    t := self.readFieldType()
    if t != nil {
        self.md.returnType = t
    } else {
        self.causePanic()
    }
}

func (self *MemberDescriptorParser) readUint8() uint8 {
    b := self.descriptor[self.offset]
    self.offset++
    return b
}
func (self *MemberDescriptorParser) unreadUint8() {
    self.offset--
}

func (self *MemberDescriptorParser) readFieldType() (*FieldType) {
    switch self.readUint8() {
    case 'B': return baseTypeB
    case 'C': return baseTypeC
    case 'D': return baseTypeD
    case 'F': return baseTypeF
    case 'I': return baseTypeI
    case 'J': return baseTypeJ
    case 'S': return baseTypeS
    case 'Z': return baseTypeZ
    case 'V': return baseTypeV
    case 'L': return self.readObjectType()
    case '[': return self.readArrayType()
    default:
        self.unreadUint8()
        return nil
    }
}
func (self *MemberDescriptorParser) readObjectType() (*FieldType) {
    unread := self.descriptor[self.offset:]
    semicolonIndex := strings.IndexRune(unread, ';')
    if semicolonIndex == -1 {
        self.causePanic()
        return nil
    } else {
        objStart := self.offset - 1
        objEnd := self.offset + semicolonIndex + 1
        self.offset = objEnd
        descriptor := self.descriptor[objStart: objEnd]
        return &FieldType{descriptor}
    }
}
func (self *MemberDescriptorParser) readArrayType() (*FieldType) {
    arrStart := self.offset - 1
    self.readFieldType()
    arrEnd := self.offset
    descriptor := self.descriptor[arrStart: arrEnd]
    return &FieldType{descriptor}
}

func (self *MemberDescriptorParser) causePanic() {
    panic("BAD descriptor: " + self.descriptor)
}
<commit_msg>code refactor<commit_after>package class

import (
    "strings"
)

type MemberDescriptorParser struct {
    descriptor  string
    offset      int
    md          *MethodDescriptor
}

func newMemberDescriptorParser(descriptor string) (*MemberDescriptorParser) {
    return &MemberDescriptorParser{descriptor: descriptor}
}

func (self *MemberDescriptorParser) parse() (*MethodDescriptor) {
    self.md = &MethodDescriptor{}
    self.startParams()
    self.parseParameterTypes()
    self.endParams()
    self.parseReturnType()
    self.finish()
    return self.md
}

func (self *MemberDescriptorParser) startParams() {
    if self.readUint8() != '(' {
        self.causePanic()
    }
}
func (self *MemberDescriptorParser) endParams() {
    if self.readUint8() != ')' {
        self.causePanic()
    }
}
func (self *MemberDescriptorParser) finish() {
    if self.offset != len(self.descriptor) {
        self.causePanic()
    }
}

func (self *MemberDescriptorParser) parseParameterTypes() {
    for {
        t := self.readFieldType()
        if t != nil {
            self.md.addParameterType(t)
        } else {
            break
        }
    }
}
func (self *MemberDescriptorParser) parseReturnType() {
    t := self.readFieldType()
    if t != nil {
        self.md.returnType = t
    } else {
        self.causePanic()
    }
}

func (self *MemberDescriptorParser) readUint8() uint8 {
    b := self.descriptor[self.offset]
    self.offset++
    return b
}
func (self *MemberDescriptorParser) unreadUint8() {
    self.offset--
}

func (self *MemberDescriptorParser) readFieldType() (*FieldType) {
    switch self.readUint8() {
    case 'B': return baseTypeB
    case 'C': return baseTypeC
    case 'D': return baseTypeD
    case 'F': return baseTypeF
    case 'I': return baseTypeI
    case 'J': return baseTypeJ
    case 'S': return baseTypeS
    case 'Z': return baseTypeZ
    case 'V': return baseTypeV
    case 'L': return self.readObjectType()
    case '[': return self.readArrayType()
    default:
        self.unreadUint8()
        return nil
    }
}
func (self *MemberDescriptorParser) readObjectType() (*FieldType) {
    unread := self.descriptor[self.offset:]
    semicolonIndex := strings.IndexRune(unread, ';')
    if semicolonIndex == -1 {
        self.causePanic()
        return nil
    } else {
        objStart := self.offset - 1
        objEnd := self.offset + semicolonIndex + 1
        self.offset = objEnd
        descriptor := self.descriptor[objStart: objEnd]
        return &FieldType{descriptor}
    }
}
func (self *MemberDescriptorParser) readArrayType() (*FieldType) {
    arrStart := self.offset - 1
    self.readFieldType()
    arrEnd := self.offset
    descriptor := self.descriptor[arrStart: arrEnd]
    return &FieldType{descriptor}
}

func (self *MemberDescriptorParser) causePanic() {
    panic("BAD descriptor: " + self.descriptor)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/printer"
	"go/token"
	"os"
	"strings"
)

// godefs returns the output for -godefs mode.
func (p *Package) godefs(f *File, srcfile string) string {
	var buf bytes.Buffer

	fmt.Fprintf(&buf, "// Created by cgo -godefs - DO NOT EDIT\n")
	fmt.Fprintf(&buf, "// %s\n", strings.Join(os.Args, " "))
	fmt.Fprintf(&buf, "\n")

	override := make(map[string]string)

	// Allow source file to specify override mappings.
	// For example, the socket data structures refer
	// to in_addr and in_addr6 structs but we want to be
	// able to treat them as byte arrays, so the godefs
	// inputs in package syscall say
	//
	//	// +godefs map struct_in_addr [4]byte
	//	// +godefs map struct_in_addr6 [16]byte
	//
	for _, g := range f.Comments {
		for _, c := range g.List {
			i := strings.Index(c.Text, "+godefs map")
			if i < 0 {
				continue
			}
			s := strings.TrimSpace(c.Text[i+len("+godefs map"):])
			i = strings.Index(s, " ")
			if i < 0 {
				fmt.Fprintf(os.Stderr, "invalid +godefs map comment: %s\n", c.Text)
				continue
			}
			override["_Ctype_"+strings.TrimSpace(s[:i])] = strings.TrimSpace(s[i:])
		}
	}
	for _, n := range f.Name {
		if s := override[n.Go]; s != "" {
			override[n.Mangle] = s
		}
	}

	// Otherwise, if the source file says type T C.whatever,
	// use "T" as the mangling of C.whatever,
	// except in the definition (handled at end of function).
	refName := make(map[*ast.Expr]*Name)
	for _, r := range f.Ref {
		refName[r.Expr] = r.Name
	}
	for _, d := range f.AST.Decls {
		d, ok := d.(*ast.GenDecl)
		if !ok || d.Tok != token.TYPE {
			continue
		}
		for _, s := range d.Specs {
			s := s.(*ast.TypeSpec)
			n := refName[&s.Type]
			if n != nil && n.Mangle != "" {
				override[n.Mangle] = s.Name.Name
			}
		}
	}

	// Extend overrides using typedefs:
	// If we know that C.xxx should format as T
	// and xxx is a typedef for yyy, make C.yyy format as T.
	for typ, def := range typedef {
		if new := override[typ]; new != "" {
			if id, ok := def.(*ast.Ident); ok {
				override[id.Name] = new
			}
		}
	}

	// Apply overrides.
	for old, new := range override {
		if id := goIdent[old]; id != nil {
			id.Name = new
		}
	}

	// Any names still using the _C syntax are not going to compile,
	// although in general we don't know whether they all made it
	// into the file, so we can't warn here.
	//
	// The most common case is union types, which begin with
	// _Ctype_union and for which typedef[name] is a Go byte
	// array of the appropriate size (such as [4]byte).
	// Substitute those union types with byte arrays.
	for name, id := range goIdent {
		if id.Name == name && strings.Contains(name, "_Ctype_union") {
			if def := typedef[name]; def != nil {
				id.Name = gofmt(def)
			}
		}
	}

	printer.Fprint(&buf, fset, f.AST)

	return buf.String()
}

// cdefs returns the output for -cdefs mode.
// The easiest way to do this is to translate the godefs Go to C.
func (p *Package) cdefs(f *File, srcfile string) string {
	godefsOutput := p.godefs(f, srcfile)

	lines := strings.Split(godefsOutput, "\n")
	lines[0] = "// Created by cgo -cdefs - DO NOT EDIT"

	for i, line := range lines {
		lines[i] = strings.TrimSpace(line)
	}

	var out bytes.Buffer
	printf := func(format string, args ...interface{}) { fmt.Fprintf(&out, format, args...) }

	didTypedef := false
	for i := 0; i < len(lines); i++ {
		line := lines[i]

		// Delete
		//	package x
		if strings.HasPrefix(line, "package ") {
			continue
		}

		// Convert
		//	const (
		//		A = 1
		//		B = 2
		//	)
		//
		// to
		//
		//	enum {
		//		A = 1,
		//		B = 2,
		//	};
		if line == "const (" {
			printf("enum {\n")
			for i++; i < len(lines) && lines[i] != ")"; i++ {
				line = lines[i]
				if line != "" {
					printf("\t%s,", line)
				}
				printf("\n")
			}
			printf("};\n")
			continue
		}

		// Convert
		//	const A = 1
		// to
		//	enum { A = 1 };
		if strings.HasPrefix(line, "const ") {
			printf("enum { %s };\n", line[len("const "):])
			continue
		}

		// On first type definition, typedef all the structs
		// in case there are dependencies between them.
		if !didTypedef && strings.HasPrefix(line, "type ") {
			didTypedef = true
			for _, line := range lines {
				line = strings.TrimSpace(line)
				if strings.HasPrefix(line, "type ") && strings.HasSuffix(line, " struct {") {
					s := line[len("type ") : len(line)-len(" struct {")]
					printf("typedef struct %s %s;\n", s, s)
				}
			}
			printf("\n")
			printf("#pragma pack on\n")
			printf("\n")
		}

		// Convert
		//	type T struct {
		//		X int64
		//		Y *int32
		//		Z [4]byte
		//	}
		//
		// to
		//
		//	struct T {
		//		int64 X;
		//		int32 *Y;
		//		byte Z[4];
		//	}
		if strings.HasPrefix(line, "type ") && strings.HasSuffix(line, " struct {") {
			s := line[len("type ") : len(line)-len(" struct {")]
			printf("struct %s {\n", s)
			for i++; i < len(lines) && lines[i] != "}"; i++ {
				line := lines[i]
				if line != "" {
					f := strings.Fields(line)
					if len(f) != 2 {
						fmt.Fprintf(os.Stderr, "cgo: cannot parse struct field: %s\n", line)
						nerrors++
						continue
					}
					printf("\t%s;", cdecl(f[0], f[1]))
				}
				printf("\n")
			}
			printf("};\n")
			continue
		}

		// Convert
		//	type T int
		// to
		//	typedef int T;
		if strings.HasPrefix(line, "type ") {
			f := strings.Fields(line[len("type "):])
			if len(f) != 2 {
				fmt.Fprintf(os.Stderr, "cgo: cannot parse type definition: %s\n", line)
				nerrors++
				continue
			}
			printf("typedef\t%s;\n", cdecl(f[0], f[1]))
			continue
		}

		printf("%s\n", line)
	}

	if didTypedef {
		printf("\n")
		printf("#pragma pack off\n")
	}

	return out.String()
}

// cdecl returns the C declaration for the given Go name and type.
// It only handles the specific cases necessary for converting godefs output.
func cdecl(name, typ string) string {
	// X *[0]byte -> X *void
	if strings.HasPrefix(typ, "*[0]") {
		typ = "*void"
	}
	// X *byte -> *X byte
	if strings.HasPrefix(typ, "*") {
		name = "*" + name
		typ = typ[1:]
	}
	// X [4]byte -> X[4] byte
	if strings.HasPrefix(typ, "[") {
		i := strings.Index(typ, "]") + 1
		name = name + typ[:i]
		typ = typ[i:]
	}
	// X T -> T X
	return typ + "\t" + name
}

var gofmtBuf bytes.Buffer

// gofmt returns the gofmt-formatted string for an AST node.
func gofmt(n interface{}) string {
	gofmtBuf.Reset()
	err := printer.Fprint(&gofmtBuf, fset, n)
	if err != nil {
		return "<" + err.Error() + ">"
	}
	return gofmtBuf.String()
}
<commit_msg>cgo: -cdefs should translate unsafe.Pointer to void *         Fixes #2454.<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/printer"
	"go/token"
	"os"
	"strings"
)

// godefs returns the output for -godefs mode.
func (p *Package) godefs(f *File, srcfile string) string {
	var buf bytes.Buffer

	fmt.Fprintf(&buf, "// Created by cgo -godefs - DO NOT EDIT\n")
	fmt.Fprintf(&buf, "// %s\n", strings.Join(os.Args, " "))
	fmt.Fprintf(&buf, "\n")

	override := make(map[string]string)

	// Allow source file to specify override mappings.
	// For example, the socket data structures refer
	// to in_addr and in_addr6 structs but we want to be
	// able to treat them as byte arrays, so the godefs
	// inputs in package syscall say
	//
	//	// +godefs map struct_in_addr [4]byte
	//	// +godefs map struct_in_addr6 [16]byte
	//
	for _, g := range f.Comments {
		for _, c := range g.List {
			i := strings.Index(c.Text, "+godefs map")
			if i < 0 {
				continue
			}
			s := strings.TrimSpace(c.Text[i+len("+godefs map"):])
			i = strings.Index(s, " ")
			if i < 0 {
				fmt.Fprintf(os.Stderr, "invalid +godefs map comment: %s\n", c.Text)
				continue
			}
			override["_Ctype_"+strings.TrimSpace(s[:i])] = strings.TrimSpace(s[i:])
		}
	}
	for _, n := range f.Name {
		if s := override[n.Go]; s != "" {
			override[n.Mangle] = s
		}
	}

	// Otherwise, if the source file says type T C.whatever,
	// use "T" as the mangling of C.whatever,
	// except in the definition (handled at end of function).
	refName := make(map[*ast.Expr]*Name)
	for _, r := range f.Ref {
		refName[r.Expr] = r.Name
	}
	for _, d := range f.AST.Decls {
		d, ok := d.(*ast.GenDecl)
		if !ok || d.Tok != token.TYPE {
			continue
		}
		for _, s := range d.Specs {
			s := s.(*ast.TypeSpec)
			n := refName[&s.Type]
			if n != nil && n.Mangle != "" {
				override[n.Mangle] = s.Name.Name
			}
		}
	}

	// Extend overrides using typedefs:
	// If we know that C.xxx should format as T
	// and xxx is a typedef for yyy, make C.yyy format as T.
	for typ, def := range typedef {
		if new := override[typ]; new != "" {
			if id, ok := def.(*ast.Ident); ok {
				override[id.Name] = new
			}
		}
	}

	// Apply overrides.
	for old, new := range override {
		if id := goIdent[old]; id != nil {
			id.Name = new
		}
	}

	// Any names still using the _C syntax are not going to compile,
	// although in general we don't know whether they all made it
	// into the file, so we can't warn here.
	//
	// The most common case is union types, which begin with
	// _Ctype_union and for which typedef[name] is a Go byte
	// array of the appropriate size (such as [4]byte).
	// Substitute those union types with byte arrays.
	for name, id := range goIdent {
		if id.Name == name && strings.Contains(name, "_Ctype_union") {
			if def := typedef[name]; def != nil {
				id.Name = gofmt(def)
			}
		}
	}

	printer.Fprint(&buf, fset, f.AST)

	return buf.String()
}

// cdefs returns the output for -cdefs mode.
// The easiest way to do this is to translate the godefs Go to C.
func (p *Package) cdefs(f *File, srcfile string) string {
	godefsOutput := p.godefs(f, srcfile)

	lines := strings.Split(godefsOutput, "\n")
	lines[0] = "// Created by cgo -cdefs - DO NOT EDIT"

	for i, line := range lines {
		lines[i] = strings.TrimSpace(line)
	}

	var out bytes.Buffer
	printf := func(format string, args ...interface{}) { fmt.Fprintf(&out, format, args...) }

	didTypedef := false
	for i := 0; i < len(lines); i++ {
		line := lines[i]

		// Delete
		//	package x
		if strings.HasPrefix(line, "package ") {
			continue
		}

		// Convert
		//	const (
		//		A = 1
		//		B = 2
		//	)
		//
		// to
		//
		//	enum {
		//		A = 1,
		//		B = 2,
		//	};
		if line == "const (" {
			printf("enum {\n")
			for i++; i < len(lines) && lines[i] != ")"; i++ {
				line = lines[i]
				if line != "" {
					printf("\t%s,", line)
				}
				printf("\n")
			}
			printf("};\n")
			continue
		}

		// Convert
		//	const A = 1
		// to
		//	enum { A = 1 };
		if strings.HasPrefix(line, "const ") {
			printf("enum { %s };\n", line[len("const "):])
			continue
		}

		// On first type definition, typedef all the structs
		// in case there are dependencies between them.
		if !didTypedef && strings.HasPrefix(line, "type ") {
			didTypedef = true
			for _, line := range lines {
				line = strings.TrimSpace(line)
				if strings.HasPrefix(line, "type ") && strings.HasSuffix(line, " struct {") {
					s := line[len("type ") : len(line)-len(" struct {")]
					printf("typedef struct %s %s;\n", s, s)
				}
			}
			printf("\n")
			printf("#pragma pack on\n")
			printf("\n")
		}

		// Convert
		//	type T struct {
		//		X int64
		//		Y *int32
		//		Z [4]byte
		//	}
		//
		// to
		//
		//	struct T {
		//		int64 X;
		//		int32 *Y;
		//		byte Z[4];
		//	}
		if strings.HasPrefix(line, "type ") && strings.HasSuffix(line, " struct {") {
			s := line[len("type ") : len(line)-len(" struct {")]
			printf("struct %s {\n", s)
			for i++; i < len(lines) && lines[i] != "}"; i++ {
				line := lines[i]
				if line != "" {
					f := strings.Fields(line)
					if len(f) != 2 {
						fmt.Fprintf(os.Stderr, "cgo: cannot parse struct field: %s\n", line)
						nerrors++
						continue
					}
					printf("\t%s;", cdecl(f[0], f[1]))
				}
				printf("\n")
			}
			printf("};\n")
			continue
		}

		// Convert
		//	type T int
		// to
		//	typedef int T;
		if strings.HasPrefix(line, "type ") {
			f := strings.Fields(line[len("type "):])
			if len(f) != 2 {
				fmt.Fprintf(os.Stderr, "cgo: cannot parse type definition: %s\n", line)
				nerrors++
				continue
			}
			printf("typedef\t%s;\n", cdecl(f[0], f[1]))
			continue
		}

		printf("%s\n", line)
	}

	if didTypedef {
		printf("\n")
		printf("#pragma pack off\n")
	}

	return out.String()
}

// cdecl returns the C declaration for the given Go name and type.
// It only handles the specific cases necessary for converting godefs output.
func cdecl(name, typ string) string {
	// X *[0]byte -> X *void
	if strings.HasPrefix(typ, "*[0]") {
		typ = "*void"
	}
	// X *byte -> *X byte
	if strings.HasPrefix(typ, "*") {
		name = "*" + name
		typ = typ[1:]
	}
	// X [4]byte -> X[4] byte
	if strings.HasPrefix(typ, "[") {
		i := strings.Index(typ, "]") + 1
		name = name + typ[:i]
		typ = typ[i:]
	}
	// X T -> T X
	// Handle the special case: 'unsafe.Pointer' is 'void *'
	if typ == "unsafe.Pointer" {
		typ = "void"
		name = "*" + name
	}
	return typ + "\t" + name
}

var gofmtBuf bytes.Buffer

// gofmt returns the gofmt-formatted string for an AST node.
func gofmt(n interface{}) string {
	gofmtBuf.Reset()
	err := printer.Fprint(&gofmtBuf, fset, n)
	if err != nil {
		return "<" + err.Error() + ">"
	}
	return gofmtBuf.String()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	// TODO "github.com/tianon/dtodo/src/dnew"
	"dnew"

	"pault.ag/go/debian/changelog"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/resolver"
)

func main() {
	log.SetFlags(log.Lshortfile)

	// TODO configurable path?  perhaps allow for an optional *.dsc instead?
	con, err := control.ParseControlFile("debian/control")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	chg, err := changelog.ParseFileOne("debian/changelog")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	// TODO configurable or something to avoid guesswork
	targetSuite := chg.Target
	if targetSuite == "UNRELEASED" {
		// check for "Upload to XYZ." in changelog
		re := regexp.MustCompile(`^\s*\*?\s*Upload\s+to\s+(\S+?)\.?(\s+|$)`)
		matches := re.FindStringSubmatch(chg.Changelog)
		if matches != nil {
			targetSuite = matches[1]
		} else {
			targetSuite = "unstable"
		}
	}

	// TODO configurable
	arch := "amd64"

	fmt.Printf("Target: %s\n", targetSuite)
	fmt.Printf("Architecture: %s\n", arch)
	fmt.Printf("Source: %s\n", con.Source.Source)
	fmt.Printf("Version: %s\n", chg.Version)
	fmt.Printf("\n")

	index, err := resolver.GetBinaryIndex(
		"http://httpredir.debian.org/debian",
		targetSuite,
		"main",
		arch,
	)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	// TODO use target suite to include more suites if necessary (ie, "experimental" needs "sid" too)

	incoming, err := resolver.GetBinaryIndex(
		"http://incoming.debian.org/debian-buildd",
		"buildd-"+targetSuite,
		"main",
		arch,
	)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	newQueue, err := dnew.ParseNewUrl(dnew.New822)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	newBinaries := map[string]dnew.NewEntry{}
	for _, newPkg := range newQueue {
		for _, newBin := range newPkg.Binary {
			newBinaries[newBin] = newPkg
		}
	}

	allDeps := dependency.Dependency{}

	binRelation := dependency.Relation{}
	for _, bin := range con.Binaries {
		binRelation.Possibilities = append(binRelation.Possibilities, dependency.Possibility{
			Name: bin.Package,
			Version: &dependency.VersionRelation{
				Operator: "=",
				Number:   chg.Version.String(),
			},
		})
	}
	allDeps.Relations = append(allDeps.Relations, binRelation)

	allDeps.Relations = append(allDeps.Relations, con.Source.BuildDepends.Relations...)
	allDeps.Relations = append(allDeps.Relations, con.Source.BuildDependsIndep.Relations...)

	for _, bin := range con.Binaries {
		allDeps.Relations = append(allDeps.Relations, bin.Depends.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Recommends.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Suggests.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Enhances.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.PreDepends.Relations...)
	}

	depArch, err := dependency.ParseArch("any")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	seenRelations := map[string]bool{}
	for _, relation := range allDeps.Relations {
		relationString := relation.String()
		if seenRelations[relationString] {
			continue
		}
		seenRelations[relationString] = true

		notes := []string{}
		for _, possi := range relation.Possibilities {
			if possi.Substvar {
				//fmt.Printf("ignoring substvar %s\n", possi)
				continue
			}
			can, why, _ := index.ExplainSatisfies(*depArch, possi)
			if !can {
				inCan, _, _ := incoming.ExplainSatisfies(*depArch, possi)
				if !inCan {
					if newPkg, ok := newBinaries[possi.Name]; ok {
						newUrl := fmt.Sprintf("https://ftp-master.debian.org/new/%s_%s.html", newPkg.Source, newPkg.Version[0])
						notes = append(notes, fmt.Sprintf("NEW (%s): %s", possi.Name, newUrl))
					} else {
						notes = append(notes, why)
					}
				} else {
					notes = append(notes, fmt.Sprintf("%s is in incoming", possi.Name))
				}
			}
		}
		if len(notes) > 0 {
			fmt.Printf("Relation: %s\n", relation)
			if len(notes) > 1 {
				fmt.Printf("Notes:\n %s\n", strings.Join(notes, "\n "))
			} else {
				fmt.Printf("Notes: %s\n", notes[0])
			}
			fmt.Printf("\n")
		}
	}
}
<commit_msg>Refactor so that we can have the URL to incoming packages<commit_after>package main

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	// TODO "github.com/tianon/dtodo/src/dnew"
	"dnew"

	"pault.ag/go/debian/changelog"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/resolver"
)

type Target struct {
	Mirror     string
	Suites     []string
	Components []string
	Arches     []string

	resolver.Candidates
}

func NewTarget(mirror string, suites, components, arches []string) (*Target, error) {
	target := Target{
		Mirror:     mirror,
		Suites:     suites,
		Components: components,
		Arches:     arches,

		Candidates: resolver.Candidates{},
	}
	for _, suite := range suites {
		for _, component := range components {
			for _, arch := range arches {
				err := resolver.AppendBinaryIndex(&target.Candidates, mirror, suite, component, arch)
				if err != nil {
					return nil, err
				}
			}
		}
	}
	return &target, nil
}

func (target Target) UrlTo(bin control.BinaryIndex) string {
	return target.Mirror + "/" + bin.Filename
}

func main() {
	log.SetFlags(log.Lshortfile)

	// TODO configurable path?  perhaps allow for an optional *.dsc instead?
	con, err := control.ParseControlFile("debian/control")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	chg, err := changelog.ParseFileOne("debian/changelog")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	// TODO configurable or something to avoid guesswork
	targetSuite := chg.Target
	if targetSuite == "UNRELEASED" {
		// check for "Upload to XYZ." in changelog
		re := regexp.MustCompile(`^\s*\*?\s*Upload\s+to\s+(\S+?)\.?(\s+|$)`)
		matches := re.FindStringSubmatch(chg.Changelog)
		if matches != nil {
			targetSuite = matches[1]
		} else {
			targetSuite = "unstable"
		}
	}

	// TODO configurable
	arches := []string{"amd64", "i386"}
	components := []string{"main", "contrib", "non-free"}

	fmt.Printf("Target: %s\n", targetSuite)
	fmt.Printf("Architectures: %s\n", arches)
	fmt.Printf("Components: %s\n", components)
	fmt.Printf("Source: %s\n", con.Source.Source)
	fmt.Printf("Version: %s\n", chg.Version)
	fmt.Printf("\n")

	index, err := NewTarget(
		"http://httpredir.debian.org/debian",
		[]string{targetSuite},
		components,
		arches,
	)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	// TODO use target suite to include more suites if necessary (ie, "experimental" needs "sid" too)

	incoming, err := NewTarget(
		"http://incoming.debian.org/debian-buildd",
		[]string{"buildd-" + targetSuite},
		components,
		arches,
	)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	newQueue, err := dnew.ParseNewUrl(dnew.New822)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	newBinaries := map[string]dnew.NewEntry{}
	for _, newPkg := range newQueue {
		for _, newBin := range newPkg.Binary {
			newBinaries[newBin] = newPkg
		}
	}

	allDeps := dependency.Dependency{}

	binRelation := dependency.Relation{}
	for _, bin := range con.Binaries {
		binRelation.Possibilities = append(binRelation.Possibilities, dependency.Possibility{
			Name: bin.Package,
			Version: &dependency.VersionRelation{
				Operator: "=",
				Number:   chg.Version.String(),
			},
		})
	}
	allDeps.Relations = append(allDeps.Relations, binRelation)

	allDeps.Relations = append(allDeps.Relations, con.Source.BuildDepends.Relations...)
	allDeps.Relations = append(allDeps.Relations, con.Source.BuildDependsIndep.Relations...)

	for _, bin := range con.Binaries {
		allDeps.Relations = append(allDeps.Relations, bin.Depends.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Recommends.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Suggests.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Enhances.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.PreDepends.Relations...)
	}

	depArch, err := dependency.ParseArch("any")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	seenRelations := map[string]bool{}
	for _, relation := range allDeps.Relations {
		relationString := relation.String()
		if seenRelations[relationString] {
			continue
		}
		seenRelations[relationString] = true

		notes := []string{}
		for _, possi := range relation.Possibilities {
			if possi.Substvar {
				//fmt.Printf("ignoring substvar %s\n", possi)
				continue
			}
			can, why, _ := index.ExplainSatisfies(*depArch, possi)
			if !can {
				inCan, _, incomingBins := incoming.ExplainSatisfies(*depArch, possi)
				if !inCan {
					if newPkg, ok := newBinaries[possi.Name]; ok {
						newUrl := fmt.Sprintf("https://ftp-master.debian.org/new/%s_%s.html", newPkg.Source, newPkg.Version[0])
						notes = append(notes, fmt.Sprintf("NEW (%s): %s", possi.Name, newUrl))
					} else {
						notes = append(notes, why)
					}
				} else {
					notes = append(notes, fmt.Sprintf("incoming (%s): %s", possi.Name, incoming.UrlTo(incomingBins[0])))
				}
			}
		}
		if len(notes) > 0 {
			fmt.Printf("Relation: %s\n", relation)
			if len(notes) > 1 {
				fmt.Printf("Notes:\n %s\n", strings.Join(notes, "\n "))
			} else {
				fmt.Printf("Notes: %s\n", notes[0])
			}
			fmt.Printf("\n")
		}
	}
}
<|endoftext|>