<commit_before>package ristretto255

import (
	"bytes"
	"encoding/hex"
	"testing"

	"github.com/gtank/ristretto255/internal/edwards25519"
	"github.com/gtank/ristretto255/internal/radix51"
)

func assertFeEqual(value, expect *radix51.FieldElement) {
	if value.Equal(expect) == 1 {
		return
	} else {
		panic("failed equality assertion")
	}
}

type sqrtRatioTest struct {
	u, v     *radix51.FieldElement
	sqrt     *radix51.FieldElement
	choice   int
	negative int
}

// These tests can be found in curve25519-dalek's 'field.rs'
func TestSqrtRatioM1(t *testing.T) {
	var (
		zero, one = radix51.Zero, radix51.One

		// Two is nonsquare in our field, 4 is square
		two  = new(radix51.FieldElement).Add(one, one)
		four = new(radix51.FieldElement).Add(two, two)

		// 2*i
		twoTimesSqrtM1 = new(radix51.FieldElement).Mul(two, sqrtM1)

		sqrt2i = fieldElementFromDecimal(
			"38214883241950591754978413199355411911188925816896391856984770930832735035196")

		invSqrt4 = fieldElementFromDecimal(
			"28948022309329048855892746252171976963317496166410141009864396001978282409974")
	)

	// Check the construction of those magic numbers.
	assertFeEqual(new(radix51.FieldElement).Mul(sqrt2i, sqrt2i), twoTimesSqrtM1)
	assertFeEqual(new(radix51.FieldElement).Mul(new(radix51.FieldElement).Square(invSqrt4), four), one)

	var tests = []sqrtRatioTest{
		{u: zero, v: zero, sqrt: zero, choice: 1, negative: 0},    // 0
		{u: one, v: zero, sqrt: zero, choice: 0, negative: 0},     // 1
		{u: two, v: one, sqrt: sqrt2i, choice: 0, negative: 0},    // 2
		{u: four, v: one, sqrt: two, choice: 1, negative: 0},      // 3
		{u: one, v: four, sqrt: invSqrt4, choice: 1, negative: 0}, // 4
	}

	for idx, tt := range tests {
		sqrt := new(radix51.FieldElement)
		choice := feSqrtRatio(sqrt, tt.u, tt.v)
		if choice != tt.choice || sqrt.Equal(tt.sqrt) != 1 || sqrt.IsNegative() != tt.negative {
			t.Errorf("Failed test %d", idx)
			t.Logf("Got {u: %v, v: %v, sqrt: %v, choice: %d, neg: %d}", tt.u, tt.v, sqrt, choice, sqrt.IsNegative())
		}
	}
}

var (
	// The encoding of Ristretto element that can be represented internally by the Curve25519 base point.
	compressedRistrettoBasepoint, _ = hex.DecodeString("e2f2ae0a6abc4e71a884a961c500515f58e30b6aa582dd8db6a65945e08d2d76")

	// The representative Ristretto basepoint in extended coordinates.
	ristrettoBasepoint = Element{r: edwards25519.ExtendedGroupElement{
		X: radix51.FieldElement([5]uint64{426475514619346, 2063872706840040, 14628272888959, 107677749330612, 288339085807592}),
		Y: radix51.FieldElement([5]uint64{1934594822876571, 2049809580636559, 1991994783322914, 1758681962032007, 380046701118659}),
		Z: radix51.FieldElement([5]uint64{1, 0, 0, 0, 0}),
		T: radix51.FieldElement([5]uint64{410445769351754, 2235400917701188, 1495825632738689, 1351628537510093, 430502003771208}),
	}}
)

func TestRistrettoBasepointRoundTrip(t *testing.T) {
	decodedBasepoint := &Element{}
	err := decodedBasepoint.Decode(compressedRistrettoBasepoint)
	if err != nil {
		t.Fatal(err)
	}

	if decodedBasepoint.Equal(&ristrettoBasepoint) != 1 {
		t.Error("decode succeeded, but got wrong point")
	}

	roundtripBasepoint := decodedBasepoint.Encode(nil)
	if !bytes.Equal(compressedRistrettoBasepoint, roundtripBasepoint) {
		t.Error("decode<>encode roundtrip produced different results")
	}

	encodedBasepoint := ristrettoBasepoint.Encode(nil)
	if !bytes.Equal(compressedRistrettoBasepoint, encodedBasepoint) {
		t.Error("point encode produced different results")
	}
}

func TestRistrettoRandomRoundtrip(t *testing.T) {
	// TODO quickcheck
}

func TestRistrettoSmallMultiplesTestVectors(t *testing.T) {
	var testVectors = [16]string{
		// This is the identity point
		"0000000000000000000000000000000000000000000000000000000000000000",
		// This is the basepoint
		"e2f2ae0a6abc4e71a884a961c500515f58e30b6aa582dd8db6a65945e08d2d76",
		// These are small multiples of the basepoint
		"6a493210f7499cd17fecb510ae0cea23a110e8d5b901f8acadd3095c73a3b919",
		"94741f5d5d52755ece4f23f044ee27d5d1ea1e2bd196b462166b16152a9d0259",
		"da80862773358b466ffadfe0b3293ab3d9fd53c5ea6c955358f568322daf6a57",
		"e882b131016b52c1d3337080187cf768423efccbb517bb495ab812c4160ff44e",
		"f64746d3c92b13050ed8d80236a7f0007c3b3f962f5ba793d19a601ebb1df403",
		"44f53520926ec81fbd5a387845beb7df85a96a24ece18738bdcfa6a7822a176d",
		"903293d8f2287ebe10e2374dc1a53e0bc887e592699f02d077d5263cdd55601c",
		"02622ace8f7303a31cafc63f8fc48fdc16e1c8c8d234b2f0d6685282a9076031",
		"20706fd788b2720a1ed2a5dad4952b01f413bcf0e7564de8cdc816689e2db95f",
		"bce83f8ba5dd2fa572864c24ba1810f9522bc6004afe95877ac73241cafdab42",
		"e4549ee16b9aa03099ca208c67adafcafa4c3f3e4e5303de6026e3ca8ff84460",
		"aa52e000df2e16f55fb1032fc33bc42742dad6bd5a8fc0be0167436c5948501f",
		"46376b80f409b29dc2b5f6f0c52591990896e5716f41477cd30085ab7f10301e",
		"e0c418f7c8d9c4cdd7395b93ea124f3ad99021bb681dfc3302a9d99a2e53e64e",
	}

	basepointMultiple := Element{}
	basepointMultiple.Zero()

	for i := range testVectors {
		// Grab the bytes of the encoding
		encoding, err := hex.DecodeString(testVectors[i])
		if err != nil {
			t.Fatalf("#%d: bad hex encoding in test vector: %v", i, err)
		}

		// Decode the test vector to a ristretto255 element
		decodedPoint := Element{}
		err = decodedPoint.Decode(encoding)
		if err != nil {
			t.Fatalf("#%d: could not decode test vector: %v", i, err)
		}
		// Re-encode and check round trips
		roundtripEncoding := decodedPoint.Encode(nil)
		if !bytes.Equal(encoding, roundtripEncoding) {
			t.Errorf("#%d: decode<>encode roundtrip failed", i)
		}

		// Check that the test vector encodes i * B
		if basepointMultiple.Equal(&decodedPoint) != 1 {
			t.Errorf("decoded small multiple %d * B is not %d * B", i, i)
		}
		// Ensure basepointMultiple = i * B in the next iteration
		basepointMultiple.Add(&basepointMultiple, &ristrettoBasepoint)
	}
}
<commit_msg>Add test that the computed value encodes correctly<commit_after>package ristretto255

import (
	"bytes"
	"encoding/hex"
	"testing"

	"github.com/gtank/ristretto255/internal/edwards25519"
	"github.com/gtank/ristretto255/internal/radix51"
)

func assertFeEqual(value, expect *radix51.FieldElement) {
	if value.Equal(expect) == 1 {
		return
	} else {
		panic("failed equality assertion")
	}
}

type sqrtRatioTest struct {
	u, v     *radix51.FieldElement
	sqrt     *radix51.FieldElement
	choice   int
	negative int
}

// These tests can be found in curve25519-dalek's 'field.rs'
func TestSqrtRatioM1(t *testing.T) {
	var (
		zero, one = radix51.Zero, radix51.One

		// Two is nonsquare in our field, 4 is square
		two  = new(radix51.FieldElement).Add(one, one)
		four = new(radix51.FieldElement).Add(two, two)

		// 2*i
		twoTimesSqrtM1 = new(radix51.FieldElement).Mul(two, sqrtM1)

		sqrt2i = fieldElementFromDecimal(
			"38214883241950591754978413199355411911188925816896391856984770930832735035196")

		invSqrt4 = fieldElementFromDecimal(
			"28948022309329048855892746252171976963317496166410141009864396001978282409974")
	)

	// Check the construction of those magic numbers.
	assertFeEqual(new(radix51.FieldElement).Mul(sqrt2i, sqrt2i), twoTimesSqrtM1)
	assertFeEqual(new(radix51.FieldElement).Mul(new(radix51.FieldElement).Square(invSqrt4), four), one)

	var tests = []sqrtRatioTest{
		{u: zero, v: zero, sqrt: zero, choice: 1, negative: 0},    // 0
		{u: one, v: zero, sqrt: zero, choice: 0, negative: 0},     // 1
		{u: two, v: one, sqrt: sqrt2i, choice: 0, negative: 0},    // 2
		{u: four, v: one, sqrt: two, choice: 1, negative: 0},      // 3
		{u: one, v: four, sqrt: invSqrt4, choice: 1, negative: 0}, // 4
	}

	for idx, tt := range tests {
		sqrt := new(radix51.FieldElement)
		choice := feSqrtRatio(sqrt, tt.u, tt.v)
		if choice != tt.choice || sqrt.Equal(tt.sqrt) != 1 || sqrt.IsNegative() != tt.negative {
			t.Errorf("Failed test %d", idx)
			t.Logf("Got {u: %v, v: %v, sqrt: %v, choice: %d, neg: %d}", tt.u, tt.v, sqrt, choice, sqrt.IsNegative())
		}
	}
}

var (
	// The encoding of Ristretto element that can be represented internally by the Curve25519 base point.
	compressedRistrettoBasepoint, _ = hex.DecodeString("e2f2ae0a6abc4e71a884a961c500515f58e30b6aa582dd8db6a65945e08d2d76")

	// The representative Ristretto basepoint in extended coordinates.
	ristrettoBasepoint = Element{r: edwards25519.ExtendedGroupElement{
		X: radix51.FieldElement([5]uint64{426475514619346, 2063872706840040, 14628272888959, 107677749330612, 288339085807592}),
		Y: radix51.FieldElement([5]uint64{1934594822876571, 2049809580636559, 1991994783322914, 1758681962032007, 380046701118659}),
		Z: radix51.FieldElement([5]uint64{1, 0, 0, 0, 0}),
		T: radix51.FieldElement([5]uint64{410445769351754, 2235400917701188, 1495825632738689, 1351628537510093, 430502003771208}),
	}}
)

func TestRistrettoBasepointRoundTrip(t *testing.T) {
	decodedBasepoint := &Element{}
	err := decodedBasepoint.Decode(compressedRistrettoBasepoint)
	if err != nil {
		t.Fatal(err)
	}

	if decodedBasepoint.Equal(&ristrettoBasepoint) != 1 {
		t.Error("decode succeeded, but got wrong point")
	}

	roundtripBasepoint := decodedBasepoint.Encode(nil)
	if !bytes.Equal(compressedRistrettoBasepoint, roundtripBasepoint) {
		t.Error("decode<>encode roundtrip produced different results")
	}

	encodedBasepoint := ristrettoBasepoint.Encode(nil)
	if !bytes.Equal(compressedRistrettoBasepoint, encodedBasepoint) {
		t.Error("point encode produced different results")
	}
}

func TestRistrettoRandomRoundtrip(t *testing.T) {
	// TODO quickcheck
}

func TestRistrettoSmallMultiplesTestVectors(t *testing.T) {
	var testVectors = [16]string{
		// This is the identity point
		"0000000000000000000000000000000000000000000000000000000000000000",
		// This is the basepoint
		"e2f2ae0a6abc4e71a884a961c500515f58e30b6aa582dd8db6a65945e08d2d76",
		// These are small multiples of the basepoint
		"6a493210f7499cd17fecb510ae0cea23a110e8d5b901f8acadd3095c73a3b919",
		"94741f5d5d52755ece4f23f044ee27d5d1ea1e2bd196b462166b16152a9d0259",
		"da80862773358b466ffadfe0b3293ab3d9fd53c5ea6c955358f568322daf6a57",
		"e882b131016b52c1d3337080187cf768423efccbb517bb495ab812c4160ff44e",
		"f64746d3c92b13050ed8d80236a7f0007c3b3f962f5ba793d19a601ebb1df403",
		"44f53520926ec81fbd5a387845beb7df85a96a24ece18738bdcfa6a7822a176d",
		"903293d8f2287ebe10e2374dc1a53e0bc887e592699f02d077d5263cdd55601c",
		"02622ace8f7303a31cafc63f8fc48fdc16e1c8c8d234b2f0d6685282a9076031",
		"20706fd788b2720a1ed2a5dad4952b01f413bcf0e7564de8cdc816689e2db95f",
		"bce83f8ba5dd2fa572864c24ba1810f9522bc6004afe95877ac73241cafdab42",
		"e4549ee16b9aa03099ca208c67adafcafa4c3f3e4e5303de6026e3ca8ff84460",
		"aa52e000df2e16f55fb1032fc33bc42742dad6bd5a8fc0be0167436c5948501f",
		"46376b80f409b29dc2b5f6f0c52591990896e5716f41477cd30085ab7f10301e",
		"e0c418f7c8d9c4cdd7395b93ea124f3ad99021bb681dfc3302a9d99a2e53e64e",
	}

	basepointMultiple := Element{}
	basepointMultiple.Zero()

	for i := range testVectors {
		// Grab the bytes of the encoding
		encoding, err := hex.DecodeString(testVectors[i])
		if err != nil {
			t.Fatalf("#%d: bad hex encoding in test vector: %v", i, err)
		}

		// Decode the test vector to a ristretto255 element
		decodedPoint := Element{}
		err = decodedPoint.Decode(encoding)
		if err != nil {
			t.Fatalf("#%d: could not decode test vector: %v", i, err)
		}
		// Re-encode and check round trips
		roundtripEncoding := decodedPoint.Encode(nil)
		if !bytes.Equal(encoding, roundtripEncoding) {
			t.Errorf("#%d: decode<>encode roundtrip failed", i)
		}

		// Check that the test vector encodes i * B
		if basepointMultiple.Equal(&decodedPoint) != 1 {
			t.Errorf("decoded small multiple %d * B is not %d * B", i, i)
		}
		computedEncoding := basepointMultiple.Encode(nil)
		if !bytes.Equal(encoding, computedEncoding) {
			t.Errorf("#%d: encoding computed value did not match", i)
		}

		// Ensure basepointMultiple = i * B in the next iteration
		basepointMultiple.Add(&basepointMultiple, &ristrettoBasepoint)
	}
}
<|endoftext|>


<commit_before>/*
Author: Aosen
Data: 2016-01-15
QQ: 316052486
Desc: 爬取www.79xs.com的所有小说内容，一键式智能爬取及更新
将spider.conf配置文件放在可执行文件的同层
*/

package main

import (
	"database/sql"
	"log"

	"github.com/PuerkitoBio/goquery"
	"github.com/aosen/mlog"
	"github.com/aosen/robot"
	"github.com/aosen/robot/downloader"
	"github.com/aosen/robot/resource"
	"github.com/aosen/robot/scheduler"
	"github.com/aosen/utils"
	_ "github.com/go-sql-driver/mysql"
)

const (
	baseurl string = "http://www.79xs.com"
)

func loadconf(path string) (settings map[string]string) {
	//生成配置文件对象,加载配置文件
	config := utils.NewConfig().Load(path)
	return config.GlobalContent()
}

func initrequest(url string, meta map[string]string, cb func(*robot.Page)) *robot.Request {
	return &robot.Request{
		Url:      url,
		RespType: "html",
		Meta:     meta,
		CallBack: cb,
	}
}

//页面处理类
type Www79xsComProcessor struct {
}

func NewWww79xsComProcessor() *Www79xsComProcessor {
	return &Www79xsComProcessor{}
}

func (self *Www79xsComProcessor) Process(p *robot.Page) {
	//判断页面是否抓取成功
	if !p.IsSucc() {
		mlog.LogInst().LogError(p.Errormsg())
		return
	}

	//如果callback为空，则说明是入口页面，否则直接执行对应callback
	callback := p.GetRequest().GetCallBack()
	if callback == nil {
		self.mainParse(p)
	} else {
		callback(p)
	}
}

//主页解析
func (self *Www79xsComProcessor) mainParse(p *robot.Page) {
	//开始解析页面
	query := p.GetHtmlParser()
	query.Find(".subnav ul li a").Each(func(i int, s *goquery.Selection) {
		addr, _ := s.Attr("href")
		p.AddTargetRequest(initrequest(baseurl+addr, map[string]string{}, self.urlListParse))
	})
}

//分类列表解析
func (self *Www79xsComProcessor) urlListParse(p *robot.Page) {
	//开始解析页面
	//query := p.GetHtmlParser()
	log.Println("1111111111111111111111")
}

func (self *Www79xsComProcessor) Finish() {
}

//mysql pipeline
type PipelineMySQL struct {
	DB *sql.DB
}

func NewPipelineMySQL(dbinfo string) *PipelineMySQL {
	db, _ := sql.Open("mysql", dbinfo)
	db.SetMaxOpenConns(30)
	db.SetMaxIdleConns(30)
	if db.Ping() != nil {
		log.Fatalf("connect mysql fail\n")
	}
	return &PipelineMySQL{
		DB: db,
	}
}

func (self *PipelineMySQL) Process(pageitems *robot.PageItems, task robot.Task) {
}

func main() {
	start_url := baseurl
	//加载配置文件
	settings := loadconf("spider.conf")
	//获取数据库连接信息
	dbinfo, ok := settings["DBINFO"]
	if !ok {
		log.Fatalf("please insert dbinfo in spider.conf")
	}

	//爬虫初始化
	options := robot.SpiderOptions{
		TaskName:      "79xs",
		PageProcesser: NewWww79xsComProcessor(),
		Downloader:    downloader.NewHttpDownloader(),
		Scheduler:     scheduler.NewQueueScheduler(false),
		Pipelines:     []robot.Pipeline{NewPipelineMySQL(dbinfo)},
		//设置资源管理器，资源池容量为10
		ResourceManage: resource.NewSpidersPool(10, nil),
	}

	sp := robot.NewSpider(options)
	//增加根url
	sp.AddRequest(initrequest(start_url, nil, nil))
	sp.Run()
}
<commit_msg>明天搞定小说爬虫<commit_after>/*
Author: Aosen
Data: 2016-01-15
QQ: 316052486
Desc:
爬取www.79xs.com的所有小说内容，一键式智能爬取及更新
将spider.conf配置文件放在可执行文件的同层
爬虫先从主页www.79xs.com爬取, 提取分类目录并设置回调
抓取分类目录中的小说目录，提取二级分类，小说标题，并设置小说简介回调
抓取小说简介并设置章节回调
抓取章节并设置小说内容回调
抓取小说内容
如果抓到与数据库中大量相同的url 则退出, 阀值 100
*/

package main

import (
	"database/sql"
	"log"

	"github.com/PuerkitoBio/goquery"
	"github.com/aosen/mlog"
	"github.com/aosen/robot"
	"github.com/aosen/robot/downloader"
	"github.com/aosen/robot/resource"
	"github.com/aosen/robot/scheduler"
	"github.com/aosen/utils"
	_ "github.com/go-sql-driver/mysql"
)

const (
	baseurl  string = "http://www.79xs.com"
	girl_url string = "/book/LC/165.aspx"
	GIRL     string = "女生"
	BOY      string = "男生"
)

//停止标志
var stop chan bool = make(chan bool)

func loadconf(path string) (settings map[string]string) {
	//生成配置文件对象,加载配置文件
	config := utils.NewConfig().Load(path)
	return config.GlobalContent()
}

func initrequest(url string, meta map[string]string, cb func(*robot.Page)) *robot.Request {
	return &robot.Request{
		Url:      url,
		RespType: "html",
		Meta:     meta,
		CallBack: cb,
	}
}

func stopspider() {
	close(stop)
}

//页面处理类
type Www79xsComProcessor struct {
}

func NewWww79xsComProcessor() *Www79xsComProcessor {
	return &Www79xsComProcessor{}
}

func (self *Www79xsComProcessor) Process(p *robot.Page) {
	//判断页面是否抓取成功
	if !p.IsSucc() {
		mlog.LogInst().LogError(p.Errormsg())
		return
	}

	//如果callback为空，则说明是入口页面，否则直接执行对应callback
	callback := p.GetRequest().GetCallBack()
	if callback == nil {
		self.mainParse(p)
	} else {
		callback(p)
	}
}

//主页解析
func (self *Www79xsComProcessor) mainParse(p *robot.Page) {
	//开始解析页面
	query := p.GetHtmlParser()
	query.Find(".subnav ul li a").Each(func(i int, s *goquery.Selection) {
		addr, _ := s.Attr("href")
		if addr == girl_url {
			p.AddTargetRequest(initrequest(baseurl+addr, map[string]string{"first": GIRL}, self.urlListParse))
		} else {
			p.AddTargetRequest(initrequest(baseurl+addr, map[string]string{"first": BOY}, self.urlListParse))
		}
	})
}

//分类列表解析
func (self *Www79xsComProcessor) urlListParse(p *robot.Page) {
	//开始解析页面
	//query := p.GetHtmlParser()
	meta := p.GetRequest().GetMeta()
	p.AddField("code", "0")
	p.AddField("first", meta.(map[string]string)["first"])
}

func (self *Www79xsComProcessor) Finish() {
}

//mysql pipeline
type PipelineMySQL struct {
	DB *sql.DB
}

func NewPipelineMySQL(dbinfo string) *PipelineMySQL {
	db, _ := sql.Open("mysql", dbinfo)
	db.SetMaxOpenConns(30)
	db.SetMaxIdleConns(30)
	if db.Ping() != nil {
		log.Fatalf("connect mysql fail\n")
	}
	return &PipelineMySQL{
		DB: db,
	}
}

func (self *PipelineMySQL) Process(pageitems *robot.PageItems, task robot.Task) {
	//如果code＝“0” 则调用处理一级分类函数
	if code, ok := pageitems.GetItem("code"); ok {
		switch code {
		case "0":
			self.firstProcess(pageitems, task)
		}
	}
}

func (self *PipelineMySQL) firstProcess(pageitems *robot.PageItems, task robot.Task) {
	//接下来存储一级分类和二级分类
}

func main() {
	start_url := baseurl
	//加载配置文件
	settings := loadconf("spider.conf")
	//获取数据库连接信息
	dbinfo, ok := settings["DBINFO"]
	if !ok {
		log.Fatalf("please insert dbinfo in spider.conf")
	}

	//爬虫初始化
	options := robot.SpiderOptions{
		TaskName:      "79xs",
		PageProcesser: NewWww79xsComProcessor(),
		Downloader:    downloader.NewHttpDownloader(),
		Scheduler:     scheduler.NewQueueScheduler(false),
		Pipelines:     []robot.Pipeline{NewPipelineMySQL(dbinfo)},
		//设置资源管理器，资源池容量为10
		ResourceManage: resource.NewSpidersPool(10, nil),
	}

	sp := robot.NewSpider(options)
	//增加根url
	sp.AddRequest(initrequest(start_url, nil, nil))
	go sp.Run()
	<-stop
	sp.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/chrissnell/syslog"
)

type handler struct {
	// To simplify implementation of our handler we embed helper
	// syslog.BaseHandler struct.
	*syslog.BaseHandler
}

type LogentriesHostEntity struct {
	Response  string `json:"response"`
	Host      Host   `json:"host"`
	Host_key  string `json:"host_key"`
	Worker    string `json:"worker"`
	Agent_key string `json:"agent_key"`
}

type Host struct {
	C        float64 `json:"c"`
	Name     string  `json:"name"`
	Distver  string  `json:"distver"`
	Hostname string  `json:"hostname"`
	Object   string  `json:"object"`
	Distname string  `json:"distname"`
	Key      string  `json:"key"`
}

type LogentriesLogEntity struct {
	Response string `json:"response"`
	Log_key  string `json:"log_key"`
	Log      Log    `json:"log"`
}

type Log struct {
	Token     string  `json:"token"`
	Created   float64 `json:"created"`
	Name      string  `json:"name`
	Retention float64 `json:"retention"`
	Filename  string  `json:"filename"`
	Object    string  `json:"object"`
	Type      string  `json:"type"`
	Key       string  `json:"key"`
	Follow    string  `json:"folow"`
}

type LogLine struct {
	Line  syslog.Message
	Token string
}

var (
	logconsumerPtr        *string
	logentriesAPIKeyPtr   *string
	listenAddrPtr         *string
	logentities           = make(map[string]LogentriesLogEntity)
	hostentities          = make(map[string]LogentriesHostEntity)
	tokenchan             = make(chan string)
	logentities_filename  = "logentries-logentities.gob"
	hostentities_filename = "logentries-hostentities.gob"
)

// Simple fiter for named/bind messages which can be used with BaseHandler
func filter(m *syslog.Message) bool {
	return m.Tag == "named" || m.Tag == "bind"
}

func newHandler() *handler {
	msg := make(chan syslog.Message)
	// Filter function name set to nil to disable filtering
	h := handler{syslog.NewBaseHandler(5, nil, false)}
	go h.mainLoop(msg)
	go ProcessLogMessage(msg)
	return &h
}

// mainLoop reads from BaseHandler queue using h.Get and logs messages to stdout
func (h *handler) mainLoop(msg chan syslog.Message) {
	for {
		m := h.Get()
		if m == nil {
			break
		}
		msg <- *m
	}
	fmt.Println("Exit handler")
	h.End()
}

func ProcessLogMessage(msg chan syslog.Message) {
	tokenfetchdone := make(chan bool, 1)
	logentrieschan := make(chan LogLine)
	lh := make(chan struct{ host, log string })

	var logline LogLine

	for m := range msg {
		if m.Hostname == "" {
			m.Hostname = "NONE"
		}
		go GetTokenForLog(tokenfetchdone, lh)
		lh <- struct{ host, log string }{m.Hostname, m.Tag}
		token := <-tokenchan
		<-tokenfetchdone

		logline.Token = token
		logline.Line = m

		go SendLogMessages(logentrieschan)
		logentrieschan <- logline

	}
}

func GetTokenForLog(tokenfetchdone chan bool, lh chan struct{ host, log string }) {
	select {
	case lht, msg_ok := <-lh:
		if !msg_ok {
			fmt.Println("msg channel closed")
		} else {

			var hostentity LogentriesHostEntity
			var logentity LogentriesLogEntity

			l := strings.Join([]string{lht.host, lht.log}, "::")

			hostentity = hostentities[lht.host]
			if hostentity.Host.Key == "" {
				hostentity = RegisterNewHost(lht.host)

				// Store our new host token in our map and sync it to disk
				hostentities[lht.host] = hostentity
				err := SyncHostEntitiesToDisk()
				if err != nil {
					log.Fatal(err)
				}
			}

			logentity = logentities[l]
			if logentity.Log.Token == "" {
				logentity := RegisterNewLog(hostentity, l)
				logentities[l] = logentity
				tokenchan <- logentity.Log.Token
				tokenfetchdone <- true
				err := SyncLogEntitiesToDisk()
				if err != nil {
					log.Fatal(err)
				}
			} else {
				tokenchan <- logentity.Log.Token
				tokenfetchdone <- true
			}
		}
	}
}

func DialLogEntries() (err error, conn net.Conn) {
	conn, err = net.Dial("tcp", *logconsumerPtr)
	if err != nil {
		fmt.Println("Could not connect to LogEntries log endpoint ", err.Error())
	}
	return err, conn
}

func SendLogMessages(msg chan LogLine) {
	err, conn := DialLogEntries()
	if err != nil {
		fmt.Println("Could not connect to LogEntries log endpoint ", err.Error())
	}

	select {
	case logline, msg_ok := <-msg:
		if !msg_ok {
			fmt.Println("msg channel closed")
		} else {
			t := logline.Line.Time
			line := fmt.Sprintf("%v %v %v %v\n", logline.Token, t.Format(time.RFC3339), logline.Line.Hostname, logline.Line.Content)
			_, err = conn.Write([]byte(line))
			if err != nil {
				log.Print("Send to Logentries endpoint failed.")
				break
			}
			// fmt.Printf("Sending line: %v", line)
		}
	}
}

func SyncLogEntitiesToDisk() (err error) {
	m := new(bytes.Buffer)
	enc := gob.NewEncoder(m)
	enc.Encode(logentities)
	err = ioutil.WriteFile("logentries-logentities.gob", m.Bytes(), 0600)
	return (err)
}

func SyncHostEntitiesToDisk() (err error) {
	m := new(bytes.Buffer)
	enc := gob.NewEncoder(m)
	enc.Encode(hostentities)
	err = ioutil.WriteFile("logentries-hostentities.gob", m.Bytes(), 0600)
	return (err)
}

func LoadLogEntitiesFromDisk() (err error) {
	n, err := ioutil.ReadFile("logentries-logentities.gob")
	if err != nil {
		return (err)
	}
	p := bytes.NewBuffer(n)
	dec := gob.NewDecoder(p)
	err = dec.Decode(&logentities)
	return (err)
}

func LoadHostEntitiesFromDisk() (err error) {
	n, err := ioutil.ReadFile("logentries-hostentities.gob")
	if err != nil {
		return (err)
	}
	p := bytes.NewBuffer(n)
	dec := gob.NewDecoder(p)
	err = dec.Decode(&hostentities)
	return (err)
}

func RegisterNewHost(h string) (he LogentriesHostEntity) {
	v := url.Values{}
	v.Set("request", "register")
	v.Set("user_key", *logentriesAPIKeyPtr)
	v.Set("name", h)
	v.Set("hostname", h)
	v.Set("distver", "")
	v.Set("system", "")
	v.Set("distname", "")
	res, err := http.PostForm("http://api.logentries.com/", v)
	if err != nil {
		log.Fatal(err)
	}
	body, err := ioutil.ReadAll(res.Body)
	err = json.Unmarshal(body, &he)
	return (he)
}

func RegisterNewLog(e LogentriesHostEntity, n string) (logentity LogentriesLogEntity) {
	v := url.Values{}
	v.Set("request", "new_log")
	v.Set("user_key", *logentriesAPIKeyPtr)
	v.Set("host_key", e.Host.Key)
	v.Set("name", n)
	v.Set("filename", "")
	v.Set("retention", "-1")
	v.Set("source", "token")
	res, err := http.PostForm("http://api.logentries.com/", v)
	if err != nil {
		log.Fatal(err)
	}
	body, err := ioutil.ReadAll(res.Body)
	err = json.Unmarshal(body, &logentity)
	return (logentity)
}

func main() {

	logconsumerPtr = flag.String("consumer", "api.logentries.com:10000", "Logentries log consumer endpoint <host:port> (Default: api.logentries.com:10000)")
	logentriesAPIKeyPtr = flag.String("apikey", "", "Logentries API key")
	listenAddrPtr = flag.String("listen", "0.0.0.0:1987", "Host/port to listen for syslog messages <host:port> (Default: 0.0.0.0:1987)")

	flag.Parse()

	if *logentriesAPIKeyPtr == "" {
		log.Fatal("Must pass a Logentries API key. Use -h for help.")
	}

	if _, err := os.Stat(logentities_filename); err == nil {
		err = LoadLogEntitiesFromDisk()
		if err != nil {
			log.Fatal(err)
		}
	}

	if _, err := os.Stat(hostentities_filename); err == nil {
		err = LoadHostEntitiesFromDisk()
		if err != nil {
			log.Fatal(err)
		}
	}

	// Create a server with one handler and run one listen gorutine
	s := syslog.NewServer()
	s.AddAllowedRunes("-._")
	s.AddHandler(newHandler())
	s.Listen(*listenAddrPtr)

	// Wait for terminating signal
	sc := make(chan os.Signal, 2)
	signal.Notify(sc, syscall.SIGTERM, syscall.SIGINT)
	<-sc

	// Shutdown the server
	fmt.Println("Shutdown the server...")
	s.Shutdown()
	fmt.Println("Server is down")
}
<commit_msg>Code cleanup<commit_after>package main

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/chrissnell/syslog"
)

type handler struct {
	// To simplify implementation of our handler we embed helper
	// syslog.BaseHandler struct.
	*syslog.BaseHandler
}

type LogentriesHostEntity struct {
	Response  string `json:"response"`
	Host      Host   `json:"host"`
	Host_key  string `json:"host_key"`
	Worker    string `json:"worker"`
	Agent_key string `json:"agent_key"`
}

type Host struct {
	C        float64 `json:"c"`
	Name     string  `json:"name"`
	Distver  string  `json:"distver"`
	Hostname string  `json:"hostname"`
	Object   string  `json:"object"`
	Distname string  `json:"distname"`
	Key      string  `json:"key"`
}

type LogentriesLogEntity struct {
	Response string `json:"response"`
	Log_key  string `json:"log_key"`
	Log      Log    `json:"log"`
}

type Log struct {
	Token     string  `json:"token"`
	Created   float64 `json:"created"`
	Name      string  `json:"name`
	Retention float64 `json:"retention"`
	Filename  string  `json:"filename"`
	Object    string  `json:"object"`
	Type      string  `json:"type"`
	Key       string  `json:"key"`
	Follow    string  `json:"folow"`
}

type LogLine struct {
	Line  syslog.Message
	Token string
}

var (
	logconsumerPtr        *string
	logentriesAPIKeyPtr   *string
	listenAddrPtr         *string
	logentities           = make(map[string]LogentriesLogEntity)
	hostentities          = make(map[string]LogentriesHostEntity)
	tokenchan             = make(chan string)
	logentities_filename  = "logentries-logentities.gob"
	hostentities_filename = "logentries-hostentities.gob"
)

func newHandler() *handler {
	msg := make(chan syslog.Message)
	// Filter function name set to nil to disable filtering
	h := handler{syslog.NewBaseHandler(5, nil, false)}
	go h.mainLoop(msg)
	go ProcessLogMessage(msg)
	return &h
}

func (h *handler) mainLoop(msg chan syslog.Message) {
	for {
		m := h.Get()
		if m == nil {
			break
		}
		msg <- *m
	}
	fmt.Println("Exit handler")
	h.End()
}

func ProcessLogMessage(msg chan syslog.Message) {
	tokenfetchdone := make(chan bool, 1)
	logentrieschan := make(chan LogLine)
	lh := make(chan struct{ host, log string })

	var logline LogLine

	for m := range msg {
		if m.Hostname == "" {
			m.Hostname = "NONE"
		}
		go GetTokenForLog(tokenfetchdone, lh)
		lh <- struct{ host, log string }{m.Hostname, m.Tag}
		token := <-tokenchan
		<-tokenfetchdone

		logline.Token = token
		logline.Line = m

		go SendLogMessages(logentrieschan)
		logentrieschan <- logline

	}
}

func GetTokenForLog(tokenfetchdone chan bool, lh chan struct{ host, log string }) {
	select {
	case lht, msg_ok := <-lh:
		if !msg_ok {
			fmt.Println("msg channel closed")
		} else {

			var hostentity LogentriesHostEntity
			var logentity LogentriesLogEntity

			l := strings.Join([]string{lht.host, lht.log}, "::")

			hostentity = hostentities[lht.host]
			if hostentity.Host.Key == "" {
				hostentity = RegisterNewHost(lht.host)

				// Store our new host token in our map and sync it to disk
				hostentities[lht.host] = hostentity
				err := SyncHostEntitiesToDisk()
				if err != nil {
					log.Fatal(err)
				}
			}

			logentity = logentities[l]
			if logentity.Log.Token == "" {
				logentity := RegisterNewLog(hostentity, l)
				logentities[l] = logentity
				tokenchan <- logentity.Log.Token
				tokenfetchdone <- true
				err := SyncLogEntitiesToDisk()
				if err != nil {
					log.Fatal(err)
				}
			} else {
				tokenchan <- logentity.Log.Token
				tokenfetchdone <- true
			}
		}
	}
}

func DialLogEntries() (err error, conn net.Conn) {
	conn, err = net.Dial("tcp", *logconsumerPtr)
	if err != nil {
		fmt.Println("Could not connect to LogEntries log endpoint ", err.Error())
	}
	return err, conn
}

func SendLogMessages(msg chan LogLine) {
	err, conn := DialLogEntries()
	if err != nil {
		fmt.Println("Could not connect to LogEntries log endpoint ", err.Error())
	}

	select {
	case logline, msg_ok := <-msg:
		if !msg_ok {
			fmt.Println("msg channel closed")
		} else {
			t := logline.Line.Time
			line := fmt.Sprintf("%v %v %v %v\n", logline.Token, t.Format(time.RFC3339), logline.Line.Hostname, logline.Line.Content)
			_, err = conn.Write([]byte(line))
			if err != nil {
				log.Print("Send to Logentries endpoint failed.")
				break
			}
			// fmt.Printf("Sending line: %v", line)
		}
	}
}

func SyncLogEntitiesToDisk() (err error) {
	m := new(bytes.Buffer)
	enc := gob.NewEncoder(m)
	enc.Encode(logentities)
	err = ioutil.WriteFile("logentries-logentities.gob", m.Bytes(), 0600)
	return (err)
}

func SyncHostEntitiesToDisk() (err error) {
	m := new(bytes.Buffer)
	enc := gob.NewEncoder(m)
	enc.Encode(hostentities)
	err = ioutil.WriteFile("logentries-hostentities.gob", m.Bytes(), 0600)
	return (err)
}

func LoadLogEntitiesFromDisk() (err error) {
	n, err := ioutil.ReadFile("logentries-logentities.gob")
	if err != nil {
		return (err)
	}
	p := bytes.NewBuffer(n)
	dec := gob.NewDecoder(p)
	err = dec.Decode(&logentities)
	return (err)
}

func LoadHostEntitiesFromDisk() (err error) {
	n, err := ioutil.ReadFile("logentries-hostentities.gob")
	if err != nil {
		return (err)
	}
	p := bytes.NewBuffer(n)
	dec := gob.NewDecoder(p)
	err = dec.Decode(&hostentities)
	return (err)
}

func RegisterNewHost(h string) (he LogentriesHostEntity) {
	v := url.Values{}
	v.Set("request", "register")
	v.Set("user_key", *logentriesAPIKeyPtr)
	v.Set("name", h)
	v.Set("hostname", h)
	v.Set("distver", "")
	v.Set("system", "")
	v.Set("distname", "")
	res, err := http.PostForm("http://api.logentries.com/", v)
	if err != nil {
		log.Fatal(err)
	}
	body, err := ioutil.ReadAll(res.Body)
	err = json.Unmarshal(body, &he)
	return (he)
}

func RegisterNewLog(e LogentriesHostEntity, n string) (logentity LogentriesLogEntity) {
	v := url.Values{}
	v.Set("request", "new_log")
	v.Set("user_key", *logentriesAPIKeyPtr)
	v.Set("host_key", e.Host.Key)
	v.Set("name", n)
	v.Set("filename", "")
	v.Set("retention", "-1")
	v.Set("source", "token")
	res, err := http.PostForm("http://api.logentries.com/", v)
	if err != nil {
		log.Fatal(err)
	}
	body, err := ioutil.ReadAll(res.Body)
	err = json.Unmarshal(body, &logentity)
	return (logentity)
}

func main() {

	logconsumerPtr = flag.String("consumer", "api.logentries.com:10000", "Logentries log consumer endpoint <host:port> (Default: api.logentries.com:10000)")
	logentriesAPIKeyPtr = flag.String("apikey", "", "Logentries API key")
	listenAddrPtr = flag.String("listen", "0.0.0.0:1987", "Host/port to listen for syslog messages <host:port> (Default: 0.0.0.0:1987)")

	flag.Parse()

	if *logentriesAPIKeyPtr == "" {
		log.Fatal("Must pass a Logentries API key. Use -h for help.")
	}

	if _, err := os.Stat(logentities_filename); err == nil {
		err = LoadLogEntitiesFromDisk()
		if err != nil {
			log.Fatal(err)
		}
	}

	if _, err := os.Stat(hostentities_filename); err == nil {
		err = LoadHostEntitiesFromDisk()
		if err != nil {
			log.Fatal(err)
		}
	}

	// Create a server with one handler and run one listen gorutine
	s := syslog.NewServer()
	s.AddAllowedRunes("-._")
	s.AddHandler(newHandler())
	s.Listen(*listenAddrPtr)

	// Wait for terminating signal
	sc := make(chan os.Signal, 2)
	signal.Notify(sc, syscall.SIGTERM, syscall.SIGINT)
	<-sc

	// Shutdown the server
	fmt.Println("Shutdown the server...")
	s.Shutdown()
	fmt.Println("Server is down")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"time"

	"text/template"

	"github.com/dustin/go-nmea"
)

const kmlHeader = `<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2"
          xmlns:gx="http://www.google.com/kml/ext/2.2">

<Document>
<name>%s</name>
`

const kmlPoint = `<Placemark>
    <name>{{.TS}}</name>
    <TimeStamp>{{.TS}}</TimeStamp>
    <Point><coordinates>{{.Lon}},{{.Lat}},0.0</coordinates></Point>
</Placemark>
`

const kmlFooter = `</Document></kml>`

const tsFormat = "2006-01-02T15:04:05Z"

var (
	minDist = flag.Int("minDist", 1000, "minimum distance (meters) between points")
	minTime = flag.Duration("minTime", 1*time.Minute, "minimum time between points")
	tilt    = flag.Float64("tilt", 85, "viewing angle")
	rng     = flag.Float64("range", 800, "viewing range")
	alt     = flag.Float64("alt", 20, "altitude")
	flyDur  = flag.Duration("flyDur", time.Second, "fly-to duration")
	waitDur = flag.Duration("waitDur", 0, "wait duration")
	title   = flag.String("title", "Road Trip", "KML title")

	tmpl = template.Must(template.New("").Parse(kmlPoint))
)

type errRememberer struct {
	w   io.WriteCloser
	err error
}

func (e errRememberer) Write(b []byte) (int, error) {
	if e.err != nil {
		return 0, e.err
	}

	var n int
	n, e.err = e.w.Write(b)

	return n, e.err
}

func (e errRememberer) Close() error {
	if e.err != nil {
		e.w.Close()
		return e.err
	}
	return e.w.Close()
}

type kmlWriter struct {
	w          errRememberer
	plat, plon float64
	pts        time.Time
}

func (k *kmlWriter) render(m nmea.RMC, Δλ float64) {
	tmpl.Execute(k.w, struct {
		Lon, Lat float64
		TS       string
		D        float64
		H        float64
		Tilt     float64
		Range    float64
		Altitude float64
		FlyDur   float64
		WaitDur  float64
	}{m.Longitude, m.Latitude, m.Timestamp.Format(tsFormat), Δλ, m.Angle,
		*tilt, *rng, *alt,
		flyDur.Seconds(), waitDur.Seconds()})
}

func (k *kmlWriter) HandleRMC(m nmea.RMC) {
	if k.plat == 0 {
		k.render(m, 0)
		k.plat = m.Latitude
		k.plon = m.Longitude
		k.pts = m.Timestamp
		return
	}
	Δλ := distance(m.Longitude, m.Latitude, k.plon, k.plat)
	Δt := m.Timestamp.Sub(k.pts)
	if Δλ < float64(*minDist) && Δt > *minTime {
		log.Printf("Δλ = %v, Δt = %v", Δλ, Δt)
		k.render(m, Δλ)
	} else {
		k.plat = m.Latitude
		k.plon = m.Longitude
	}
	k.pts = m.Timestamp
}

func (k kmlWriter) Init() error {
	fmt.Fprintf(k.w, kmlHeader, *title)
	return k.w.err
}

func (k kmlWriter) Close() error {
	k.w.Write([]byte(kmlFooter))
	return k.w.Close()
}

func d2r(d float64) float64 {
	return d * math.Pi / 180.0
}

func distance(lon1, lat1, lon2, lat2 float64) float64 {
	φ1 := d2r(lat1)
	φ2 := d2r(lat2)
	Δφ := d2r(lat2 - lat1)
	Δλ := d2r(lon2 - lon1)

	a := math.Sin(Δφ/2)*math.Sin(Δφ/2) +
		math.Cos(φ1)*math.Cos(φ2)*
			math.Sin(Δλ/2)*math.Sin(Δλ/2)
	c := 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))

	return 6371000 * c
}

func main() {
	flag.Parse()
	h := &kmlWriter{w: errRememberer{w: os.Stdout}}
	h.Init()
	err := nmea.Process(os.Stdin, h, func(s string, err error) error {
		if err != nil {
			log.Printf("On %q: %v", s, err)
		}
		return nil
	})

	if err != nil {
		log.Fatalf("Error processing stuff: %v", err)
	}
	if err := h.Close(); err != nil {
		log.Fatalf("Error finishing up KML output: %v", err)
	}
}
<commit_msg>Remove a bunch of unused fields from prior versions<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"time"

	"text/template"

	"github.com/dustin/go-nmea"
)

const kmlHeader = `<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2"
          xmlns:gx="http://www.google.com/kml/ext/2.2">

<Document>
<name>%s</name>
`

const kmlPoint = `<Placemark>
    <name>{{.TS}}</name>
    <TimeStamp>{{.TS}}</TimeStamp>
    <Point><coordinates>{{.Lon}},{{.Lat}},0.0</coordinates></Point>
</Placemark>
`

const kmlFooter = `</Document></kml>`

const tsFormat = "2006-01-02T15:04:05Z"

var (
	minDist = flag.Int("minDist", 1000, "minimum distance (meters) between points")
	minTime = flag.Duration("minTime", 1*time.Minute, "minimum time between points")
	title   = flag.String("title", "Road Trip", "KML title")

	tmpl = template.Must(template.New("").Parse(kmlPoint))
)

type errRememberer struct {
	w   io.WriteCloser
	err error
}

func (e errRememberer) Write(b []byte) (int, error) {
	if e.err != nil {
		return 0, e.err
	}

	var n int
	n, e.err = e.w.Write(b)

	return n, e.err
}

func (e errRememberer) Close() error {
	if e.err != nil {
		e.w.Close()
		return e.err
	}
	return e.w.Close()
}

type kmlWriter struct {
	w          errRememberer
	plat, plon float64
	pts        time.Time
}

func (k *kmlWriter) render(m nmea.RMC, Δλ float64) {
	tmpl.Execute(k.w, struct {
		Lon, Lat float64
		TS       string
		D        float64
		H        float64
	}{m.Longitude, m.Latitude, m.Timestamp.Format(tsFormat), Δλ, m.Angle})
}

func (k *kmlWriter) HandleRMC(m nmea.RMC) {
	if k.plat == 0 {
		k.render(m, 0)
		k.plat = m.Latitude
		k.plon = m.Longitude
		k.pts = m.Timestamp
		return
	}
	Δλ := distance(m.Longitude, m.Latitude, k.plon, k.plat)
	Δt := m.Timestamp.Sub(k.pts)
	if Δλ < float64(*minDist) && Δt > *minTime {
		log.Printf("Δλ = %v, Δt = %v", Δλ, Δt)
		k.render(m, Δλ)
	} else {
		k.plat = m.Latitude
		k.plon = m.Longitude
	}
	k.pts = m.Timestamp
}

func (k kmlWriter) Init() error {
	fmt.Fprintf(k.w, kmlHeader, *title)
	return k.w.err
}

func (k kmlWriter) Close() error {
	k.w.Write([]byte(kmlFooter))
	return k.w.Close()
}

func d2r(d float64) float64 {
	return d * math.Pi / 180.0
}

func distance(lon1, lat1, lon2, lat2 float64) float64 {
	φ1 := d2r(lat1)
	φ2 := d2r(lat2)
	Δφ := d2r(lat2 - lat1)
	Δλ := d2r(lon2 - lon1)

	a := math.Sin(Δφ/2)*math.Sin(Δφ/2) +
		math.Cos(φ1)*math.Cos(φ2)*
			math.Sin(Δλ/2)*math.Sin(Δλ/2)
	c := 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))

	return 6371000 * c
}

func main() {
	flag.Parse()
	h := &kmlWriter{w: errRememberer{w: os.Stdout}}
	h.Init()
	err := nmea.Process(os.Stdin, h, func(s string, err error) error {
		if err != nil {
			log.Printf("On %q: %v", s, err)
		}
		return nil
	})

	if err != nil {
		log.Fatalf("Error processing stuff: %v", err)
	}
	if err := h.Close(); err != nil {
		log.Fatalf("Error finishing up KML output: %v", err)
	}
}
<|endoftext|>


<commit_before>package types

import (
	"os"
	"time"

	"github.com/docker/docker/daemon/network"
	"github.com/docker/docker/pkg/nat"
	"github.com/docker/docker/pkg/version"
	"github.com/docker/docker/registry"
	"github.com/docker/docker/runconfig"
)

// ContainerCreateResponse contains the information returned to a client on the
// creation of a new container.
type ContainerCreateResponse struct {
	// ID is the ID of the created container.
	ID string `json:"Id"`

	// Warnings are any warnings encountered during the creation of the container.
	Warnings []string `json:"Warnings"`
}

// ContainerExecCreateResponse contains response of Remote API:
// POST "/containers/{name:.*}/exec"
type ContainerExecCreateResponse struct {
	// ID is the exec ID.
	ID string `json:"Id"`
}

// AuthResponse contains response of Remote API:
// POST "/auth"
type AuthResponse struct {
	// Status is the authentication status
	Status string `json:"Status"`
}

// ContainerWaitResponse contains response of Remote API:
// POST "/containers/"+containerID+"/wait"
type ContainerWaitResponse struct {
	// StatusCode is the status code of the wait job
	StatusCode int `json:"StatusCode"`
}

// ContainerCommitResponse contains response of Remote API:
// POST "/commit?container="+containerID
type ContainerCommitResponse struct {
	ID string `json:"Id"`
}

// ContainerChange contains response of Remote API:
// GET "/containers/{name:.*}/changes"
type ContainerChange struct {
	Kind int
	Path string
}

// ImageHistory contains response of Remote API:
// GET "/images/{name:.*}/history"
type ImageHistory struct {
	ID        string `json:"Id"`
	Created   int64
	CreatedBy string
	Tags      []string
	Size      int64
	Comment   string
}

// ImageDelete contains response of Remote API:
// DELETE "/images/{name:.*}"
type ImageDelete struct {
	Untagged string `json:",omitempty"`
	Deleted  string `json:",omitempty"`
}

// Image contains response of Remote API:
// GET "/images/json"
type Image struct {
	ID          string `json:"Id"`
	ParentID    string `json:"ParentId"`
	RepoTags    []string
	RepoDigests []string
	Created     int64
	Size        int64
	VirtualSize int64
	Labels      map[string]string
}

// GraphDriverData returns Image's graph driver config info
// when calling inspect command
type GraphDriverData struct {
	Name string
	Data map[string]string
}

// ImageInspect contains response of Remote API:
// GET "/images/{name:.*}/json"
type ImageInspect struct {
	ID              string `json:"Id"`
	RepoTags        []string
	RepoDigests     []string
	Parent          string
	Comment         string
	Created         string
	Container       string
	ContainerConfig *runconfig.Config
	DockerVersion   string
	Author          string
	Config          *runconfig.Config
	Architecture    string
	Os              string
	Size            int64
	VirtualSize     int64
	GraphDriver     GraphDriverData
}

// Port stores open ports info of container
// e.g. {"PrivatePort": 8080, "PublicPort": 80, "Type": "tcp"}
type Port struct {
	IP          string `json:",omitempty"`
	PrivatePort int
	PublicPort  int `json:",omitempty"`
	Type        string
}

// Container contains response of Remote API:
// GET  "/containers/json"
type Container struct {
	ID         string `json:"Id"`
	Names      []string
	Image      string
	ImageID    string
	Command    string
	Created    int64
	Ports      []Port
	SizeRw     int64 `json:",omitempty"`
	SizeRootFs int64 `json:",omitempty"`
	Labels     map[string]string
	Status     string
	HostConfig struct {
		NetworkMode string `json:",omitempty"`
	}
}

// CopyConfig contains request body of Remote API:
// POST "/containers/"+containerID+"/copy"
type CopyConfig struct {
	Resource string
}

// ContainerPathStat is used to encode the header from
// GET "/containers/{name:.*}/archive"
// "Name" is the file or directory name.
type ContainerPathStat struct {
	Name       string      `json:"name"`
	Size       int64       `json:"size"`
	Mode       os.FileMode `json:"mode"`
	Mtime      time.Time   `json:"mtime"`
	LinkTarget string      `json:"linkTarget"`
}

// ContainerProcessList contains response of Remote API:
// GET "/containers/{name:.*}/top"
type ContainerProcessList struct {
	Processes [][]string
	Titles    []string
}

// Version contains response of Remote API:
// GET "/version"
type Version struct {
	Version       string
	APIVersion    version.Version `json:"ApiVersion"`
	GitCommit     string
	GoVersion     string
	Os            string
	Arch          string
	KernelVersion string `json:",omitempty"`
	Experimental  bool   `json:",omitempty"`
	BuildTime     string `json:",omitempty"`
}

// Info contains response of Remote API:
// GET "/info"
type Info struct {
	ID                 string
	Containers         int
	Images             int
	Driver             string
	DriverStatus       [][2]string
	MemoryLimit        bool
	SwapLimit          bool
	CPUCfsPeriod       bool `json:"CpuCfsPeriod"`
	CPUCfsQuota        bool `json:"CpuCfsQuota"`
	CPUShares          bool
	CPUSet             bool
	IPv4Forwarding     bool
	BridgeNfIptables   bool
	BridgeNfIP6tables  bool `json:"BridgeNfIp6tables"`
	Debug              bool
	NFd                int
	OomKillDisable     bool
	NGoroutines        int
	SystemTime         string
	ExecutionDriver    string
	LoggingDriver      string
	NEventsListener    int
	KernelVersion      string
	OperatingSystem    string
	IndexServerAddress string
	RegistryConfig     *registry.ServiceConfig
	InitSha1           string
	InitPath           string
	NCPU               int
	MemTotal           int64
	DockerRootDir      string
	HTTPProxy          string `json:"HttpProxy"`
	HTTPSProxy         string `json:"HttpsProxy"`
	NoProxy            string
	Name               string
	Labels             []string
	ExperimentalBuild  bool
	ServerVersion      string
	ClusterStore       string
	ClusterAdvertise   string
}

// ExecStartCheck is a temp struct used by execStart
// Config fields is part of ExecConfig in runconfig package
type ExecStartCheck struct {
	// ExecStart will first check if it's detached
	Detach bool
	// Check if there's a tty
	Tty bool
}

// ContainerState stores container's running state
// it's part of ContainerJSONBase and will return by "inspect" command
type ContainerState struct {
	Status     string
	Running    bool
	Paused     bool
	Restarting bool
	OOMKilled  bool
	Dead       bool
	Pid        int
	ExitCode   int
	Error      string
	StartedAt  string
	FinishedAt string
}

// ContainerJSONBase contains response of Remote API:
// GET "/containers/{name:.*}/json"
type ContainerJSONBase struct {
	ID              string `json:"Id"`
	Created         string
	Path            string
	Args            []string
	State           *ContainerState
	Image           string
	ResolvConfPath  string
	HostnamePath    string
	HostsPath       string
	LogPath         string
	Name            string
	RestartCount    int
	Driver          string
	MountLabel      string
	ProcessLabel    string
	AppArmorProfile string
	ExecIDs         []string
	HostConfig      *runconfig.HostConfig
	GraphDriver     GraphDriverData
	SizeRw          *int64 `json:",omitempty"`
	SizeRootFs      *int64 `json:",omitempty"`
}

// ContainerJSON is newly used struct along with MountPoint
type ContainerJSON struct {
	*ContainerJSONBase
	Mounts          []MountPoint
	Config          *runconfig.Config
	NetworkSettings *NetworkSettings
}

// NetworkSettings exposes the network settings in the api
type NetworkSettings struct {
	NetworkSettingsBase
	DefaultNetworkSettings
	Networks map[string]*network.EndpointSettings
}

// NetworkSettingsBase holds basic information about networks
type NetworkSettingsBase struct {
	Bridge                 string
	SandboxID              string
	HairpinMode            bool
	LinkLocalIPv6Address   string
	LinkLocalIPv6PrefixLen int
	Ports                  nat.PortMap
	SandboxKey             string
	SecondaryIPAddresses   []network.Address
	SecondaryIPv6Addresses []network.Address
}

// DefaultNetworkSettings holds network information
// during the 2 release deprecation period.
// It will be removed in Docker 1.11.
type DefaultNetworkSettings struct {
	EndpointID          string
	Gateway             string
	GlobalIPv6Address   string
	GlobalIPv6PrefixLen int
	IPAddress           string
	IPPrefixLen         int
	IPv6Gateway         string
	MacAddress          string
}

// MountPoint represents a mount point configuration inside the container.
type MountPoint struct {
	Name        string `json:",omitempty"`
	Source      string
	Destination string
	Driver      string `json:",omitempty"`
	Mode        string
	RW          bool
}

// Volume represents the configuration of a volume for the remote API
type Volume struct {
	Name       string // Name is the name of the volume
	Driver     string // Driver is the Driver name used to create the volume
	Mountpoint string // Mountpoint is the location on disk of the volume
}

// VolumesListResponse contains the response for the remote API:
// GET "/volumes"
type VolumesListResponse struct {
	Volumes []*Volume // Volumes is the list of volumes being returned
}

// VolumeCreateRequest contains the response for the remote API:
// POST "/volumes/create"
type VolumeCreateRequest struct {
	Name       string            // Name is the requested name of the volume
	Driver     string            // Driver is the name of the driver that should be used to create the volume
	DriverOpts map[string]string // DriverOpts holds the driver specific options to use for when creating the volume.
}

// NetworkResource is the body of the "get network" http response message
type NetworkResource struct {
	Name       string
	ID         string `json:"Id"`
	Scope      string
	Driver     string
	IPAM       network.IPAM
	Containers map[string]EndpointResource
	Options    map[string]string
}

// EndpointResource contains network resources allocated and used for a container in a network
type EndpointResource struct {
	Name        string
	EndpointID  string
	MacAddress  string
	IPv4Address string
	IPv6Address string
}

// NetworkCreate is the expected body of the "create network" http request message
type NetworkCreate struct {
	Name           string
	CheckDuplicate bool
	Driver         string
	IPAM           network.IPAM
	Options        map[string]string
}

// NetworkCreateResponse is the response message sent by the server for network create call
type NetworkCreateResponse struct {
	ID      string `json:"Id"`
	Warning string
}

// NetworkConnect represents the data to be used to connect a container to the network
type NetworkConnect struct {
	Container string
}

// NetworkDisconnect represents the data to be used to disconnect a container from the network
type NetworkDisconnect struct {
	Container string
}
<commit_msg>/info: Add keys Architecture, OSType<commit_after>package types

import (
	"os"
	"time"

	"github.com/docker/docker/daemon/network"
	"github.com/docker/docker/pkg/nat"
	"github.com/docker/docker/pkg/version"
	"github.com/docker/docker/registry"
	"github.com/docker/docker/runconfig"
)

// ContainerCreateResponse contains the information returned to a client on the
// creation of a new container.
type ContainerCreateResponse struct {
	// ID is the ID of the created container.
	ID string `json:"Id"`

	// Warnings are any warnings encountered during the creation of the container.
	Warnings []string `json:"Warnings"`
}

// ContainerExecCreateResponse contains response of Remote API:
// POST "/containers/{name:.*}/exec"
type ContainerExecCreateResponse struct {
	// ID is the exec ID.
	ID string `json:"Id"`
}

// AuthResponse contains response of Remote API:
// POST "/auth"
type AuthResponse struct {
	// Status is the authentication status
	Status string `json:"Status"`
}

// ContainerWaitResponse contains response of Remote API:
// POST "/containers/"+containerID+"/wait"
type ContainerWaitResponse struct {
	// StatusCode is the status code of the wait job
	StatusCode int `json:"StatusCode"`
}

// ContainerCommitResponse contains response of Remote API:
// POST "/commit?container="+containerID
type ContainerCommitResponse struct {
	ID string `json:"Id"`
}

// ContainerChange contains response of Remote API:
// GET "/containers/{name:.*}/changes"
type ContainerChange struct {
	Kind int
	Path string
}

// ImageHistory contains response of Remote API:
// GET "/images/{name:.*}/history"
type ImageHistory struct {
	ID        string `json:"Id"`
	Created   int64
	CreatedBy string
	Tags      []string
	Size      int64
	Comment   string
}

// ImageDelete contains response of Remote API:
// DELETE "/images/{name:.*}"
type ImageDelete struct {
	Untagged string `json:",omitempty"`
	Deleted  string `json:",omitempty"`
}

// Image contains response of Remote API:
// GET "/images/json"
type Image struct {
	ID          string `json:"Id"`
	ParentID    string `json:"ParentId"`
	RepoTags    []string
	RepoDigests []string
	Created     int64
	Size        int64
	VirtualSize int64
	Labels      map[string]string
}

// GraphDriverData returns Image's graph driver config info
// when calling inspect command
type GraphDriverData struct {
	Name string
	Data map[string]string
}

// ImageInspect contains response of Remote API:
// GET "/images/{name:.*}/json"
type ImageInspect struct {
	ID              string `json:"Id"`
	RepoTags        []string
	RepoDigests     []string
	Parent          string
	Comment         string
	Created         string
	Container       string
	ContainerConfig *runconfig.Config
	DockerVersion   string
	Author          string
	Config          *runconfig.Config
	Architecture    string
	Os              string
	Size            int64
	VirtualSize     int64
	GraphDriver     GraphDriverData
}

// Port stores open ports info of container
// e.g. {"PrivatePort": 8080, "PublicPort": 80, "Type": "tcp"}
type Port struct {
	IP          string `json:",omitempty"`
	PrivatePort int
	PublicPort  int `json:",omitempty"`
	Type        string
}

// Container contains response of Remote API:
// GET  "/containers/json"
type Container struct {
	ID         string `json:"Id"`
	Names      []string
	Image      string
	ImageID    string
	Command    string
	Created    int64
	Ports      []Port
	SizeRw     int64 `json:",omitempty"`
	SizeRootFs int64 `json:",omitempty"`
	Labels     map[string]string
	Status     string
	HostConfig struct {
		NetworkMode string `json:",omitempty"`
	}
}

// CopyConfig contains request body of Remote API:
// POST "/containers/"+containerID+"/copy"
type CopyConfig struct {
	Resource string
}

// ContainerPathStat is used to encode the header from
// GET "/containers/{name:.*}/archive"
// "Name" is the file or directory name.
type ContainerPathStat struct {
	Name       string      `json:"name"`
	Size       int64       `json:"size"`
	Mode       os.FileMode `json:"mode"`
	Mtime      time.Time   `json:"mtime"`
	LinkTarget string      `json:"linkTarget"`
}

// ContainerProcessList contains response of Remote API:
// GET "/containers/{name:.*}/top"
type ContainerProcessList struct {
	Processes [][]string
	Titles    []string
}

// Version contains response of Remote API:
// GET "/version"
type Version struct {
	Version       string
	APIVersion    version.Version `json:"ApiVersion"`
	GitCommit     string
	GoVersion     string
	Os            string
	Arch          string
	KernelVersion string `json:",omitempty"`
	Experimental  bool   `json:",omitempty"`
	BuildTime     string `json:",omitempty"`
}

// Info contains response of Remote API:
// GET "/info"
type Info struct {
	ID                 string
	Containers         int
	Images             int
	Driver             string
	DriverStatus       [][2]string
	MemoryLimit        bool
	SwapLimit          bool
	CPUCfsPeriod       bool `json:"CpuCfsPeriod"`
	CPUCfsQuota        bool `json:"CpuCfsQuota"`
	CPUShares          bool
	CPUSet             bool
	IPv4Forwarding     bool
	BridgeNfIptables   bool
	BridgeNfIP6tables  bool `json:"BridgeNfIp6tables"`
	Debug              bool
	NFd                int
	OomKillDisable     bool
	NGoroutines        int
	SystemTime         string
	ExecutionDriver    string
	LoggingDriver      string
	NEventsListener    int
	KernelVersion      string
	OperatingSystem    string
	OSType             string
	Architecture       string
	IndexServerAddress string
	RegistryConfig     *registry.ServiceConfig
	InitSha1           string
	InitPath           string
	NCPU               int
	MemTotal           int64
	DockerRootDir      string
	HTTPProxy          string `json:"HttpProxy"`
	HTTPSProxy         string `json:"HttpsProxy"`
	NoProxy            string
	Name               string
	Labels             []string
	ExperimentalBuild  bool
	ServerVersion      string
	ClusterStore       string
	ClusterAdvertise   string
}

// ExecStartCheck is a temp struct used by execStart
// Config fields is part of ExecConfig in runconfig package
type ExecStartCheck struct {
	// ExecStart will first check if it's detached
	Detach bool
	// Check if there's a tty
	Tty bool
}

// ContainerState stores container's running state
// it's part of ContainerJSONBase and will return by "inspect" command
type ContainerState struct {
	Status     string
	Running    bool
	Paused     bool
	Restarting bool
	OOMKilled  bool
	Dead       bool
	Pid        int
	ExitCode   int
	Error      string
	StartedAt  string
	FinishedAt string
}

// ContainerJSONBase contains response of Remote API:
// GET "/containers/{name:.*}/json"
type ContainerJSONBase struct {
	ID              string `json:"Id"`
	Created         string
	Path            string
	Args            []string
	State           *ContainerState
	Image           string
	ResolvConfPath  string
	HostnamePath    string
	HostsPath       string
	LogPath         string
	Name            string
	RestartCount    int
	Driver          string
	MountLabel      string
	ProcessLabel    string
	AppArmorProfile string
	ExecIDs         []string
	HostConfig      *runconfig.HostConfig
	GraphDriver     GraphDriverData
	SizeRw          *int64 `json:",omitempty"`
	SizeRootFs      *int64 `json:",omitempty"`
}

// ContainerJSON is newly used struct along with MountPoint
type ContainerJSON struct {
	*ContainerJSONBase
	Mounts          []MountPoint
	Config          *runconfig.Config
	NetworkSettings *NetworkSettings
}

// NetworkSettings exposes the network settings in the api
type NetworkSettings struct {
	NetworkSettingsBase
	DefaultNetworkSettings
	Networks map[string]*network.EndpointSettings
}

// NetworkSettingsBase holds basic information about networks
type NetworkSettingsBase struct {
	Bridge                 string
	SandboxID              string
	HairpinMode            bool
	LinkLocalIPv6Address   string
	LinkLocalIPv6PrefixLen int
	Ports                  nat.PortMap
	SandboxKey             string
	SecondaryIPAddresses   []network.Address
	SecondaryIPv6Addresses []network.Address
}

// DefaultNetworkSettings holds network information
// during the 2 release deprecation period.
// It will be removed in Docker 1.11.
type DefaultNetworkSettings struct {
	EndpointID          string
	Gateway             string
	GlobalIPv6Address   string
	GlobalIPv6PrefixLen int
	IPAddress           string
	IPPrefixLen         int
	IPv6Gateway         string
	MacAddress          string
}

// MountPoint represents a mount point configuration inside the container.
type MountPoint struct {
	Name        string `json:",omitempty"`
	Source      string
	Destination string
	Driver      string `json:",omitempty"`
	Mode        string
	RW          bool
}

// Volume represents the configuration of a volume for the remote API
type Volume struct {
	Name       string // Name is the name of the volume
	Driver     string // Driver is the Driver name used to create the volume
	Mountpoint string // Mountpoint is the location on disk of the volume
}

// VolumesListResponse contains the response for the remote API:
// GET "/volumes"
type VolumesListResponse struct {
	Volumes []*Volume // Volumes is the list of volumes being returned
}

// VolumeCreateRequest contains the response for the remote API:
// POST "/volumes/create"
type VolumeCreateRequest struct {
	Name       string            // Name is the requested name of the volume
	Driver     string            // Driver is the name of the driver that should be used to create the volume
	DriverOpts map[string]string // DriverOpts holds the driver specific options to use for when creating the volume.
}

// NetworkResource is the body of the "get network" http response message
type NetworkResource struct {
	Name       string
	ID         string `json:"Id"`
	Scope      string
	Driver     string
	IPAM       network.IPAM
	Containers map[string]EndpointResource
	Options    map[string]string
}

// EndpointResource contains network resources allocated and used for a container in a network
type EndpointResource struct {
	Name        string
	EndpointID  string
	MacAddress  string
	IPv4Address string
	IPv6Address string
}

// NetworkCreate is the expected body of the "create network" http request message
type NetworkCreate struct {
	Name           string
	CheckDuplicate bool
	Driver         string
	IPAM           network.IPAM
	Options        map[string]string
}

// NetworkCreateResponse is the response message sent by the server for network create call
type NetworkCreateResponse struct {
	ID      string `json:"Id"`
	Warning string
}

// NetworkConnect represents the data to be used to connect a container to the network
type NetworkConnect struct {
	Container string
}

// NetworkDisconnect represents the data to be used to disconnect a container from the network
type NetworkDisconnect struct {
	Container string
}
<|endoftext|>


<commit_before>package service

import (
	"context"

	"github.com/Jeffail/benthos/v3/internal/bloblang/mapping"
	"github.com/Jeffail/benthos/v3/lib/message"
	"github.com/Jeffail/benthos/v3/lib/processor"
	"github.com/Jeffail/benthos/v3/lib/types"
	"github.com/Jeffail/benthos/v3/public/bloblang"
)

// MessageHandlerFunc is a function signature defining a component that consumes
// Benthos messages. An error must be returned if the context is cancelled, or
// if the message could not be delivered or processed.
type MessageHandlerFunc func(context.Context, *Message) error

// Message represents a single discrete message passing through a Benthos
// pipeline. It is safe to mutate the message via Set methods, but the
// underlying byte data should not be edited directly.
type Message struct {
	part       types.Part
	partCopied bool
}

// MessageBatch describes a collection of one or more messages.
type MessageBatch []*Message

// Copy creates a new slice of the same messages, which can be modified without
// changing the contents of the original batch.
func (b MessageBatch) Copy() MessageBatch {
	bCopy := make(MessageBatch, len(b))
	for i, m := range b {
		bCopy[i] = m.Copy()
	}
	return bCopy
}

// NewMessage creates a new message with an initial raw bytes content. The
// initial content can be nil, which is recommended if you intend to set it with
// structured contents.
func NewMessage(content []byte) *Message {
	return &Message{
		part:       message.NewPart(content),
		partCopied: true,
	}
}

func newMessageFromPart(part types.Part) *Message {
	return &Message{part, false}
}

// Copy creates a shallow copy of a message that is safe to mutate with Set
// methods without mutating the original. Both messages will share a context,
// and therefore a tracing ID, if one has been associated with them.
//
// Note that this does not perform a deep copy of the byte or structured
// contents of the message, and therefore it is not safe to perform inline
// mutations on those values without copying them.
func (m *Message) Copy() *Message {
	return &Message{
		part:       m.part.Copy(),
		partCopied: true,
	}
}

func (m *Message) ensureCopied() {
	if !m.partCopied {
		m.part = m.part.Copy()
		m.partCopied = true
	}
}

// AsBytes returns the underlying byte array contents of a message or, if the
// contents are a structured type, attempts to marshal the contents as a JSON
// document and returns either the byte array result or an error.
//
// It is NOT safe to mutate the contents of the returned slice.
func (m *Message) AsBytes() ([]byte, error) {
	// TODO: Escalate errors in marshalling once we're able.
	return m.part.Get(), nil
}

// AsStructured returns the underlying structured contents of a message or, if
// the contents are a byte array, attempts to parse the bytes contents as a JSON
// document and returns either the structured result or an error.
//
// It is NOT safe to mutate the contents of the returned value if it is a
// reference type (slice or map). In order to safely mutate the structured
// contents of a message use AsStructuredMut.
func (m *Message) AsStructured() (interface{}, error) {
	return m.part.JSON()
}

// AsStructuredMut returns the underlying structured contents of a message or,
// if the contents are a byte array, attempts to parse the bytes contents as a
// JSON document and returns either the structured result or an error.
//
// It is safe to mutate the contents of the returned value even if it is a
// reference type (slice or map), as the structured contents will be lazily deep
// cloned if it is still owned by an upstream component.
func (m *Message) AsStructuredMut() (interface{}, error) {
	// TODO: Use refactored APIs to determine if the contents are owned.
	v, err := m.part.JSON()
	if err != nil {
		return nil, err
	}
	return message.CopyJSON(v)
}

// SetBytes sets the underlying contents of the message as a byte slice.
func (m *Message) SetBytes(b []byte) {
	m.ensureCopied()
	m.part.Set(b)
}

// SetStructured sets the underlying contents of the message as a structured
// type. This structured value should be a scalar Go type, or either a
// map[string]interface{} or []interface{} containing the same types all the way
// through the hierarchy, this ensures that other processors are able to work
// with the contents and that they can be JSON marshalled when coerced into a
// byte array.
func (m *Message) SetStructured(i interface{}) {
	m.ensureCopied()
	m.part.SetJSON(i)
}

// SetError marks the message as having failed a processing step and adds the
// error to it as context. Messages marked with errors can be handled using a
// range of methods outlined in https://www.benthos.dev/docs/configuration/error_handling.
func (m *Message) SetError(err error) {
	m.ensureCopied()
	processor.FlagErr(m.part, err)
}

// MetaGet attempts to find a metadata key from the message and returns a string
// result and a boolean indicating whether it was found.
func (m *Message) MetaGet(key string) (string, bool) {
	v := m.part.Metadata().Get(key)
	return v, len(v) > 0
}

// MetaSet sets the value of a metadata key. If the value is an empty string the
// metadata key is deleted.
func (m *Message) MetaSet(key, value string) {
	m.ensureCopied()
	if value == "" {
		m.part.Metadata().Delete(key)
	} else {
		m.part.Metadata().Set(key, value)
	}
}

// MetaDelete removes a key from the message metadata.
func (m *Message) MetaDelete(key string) {
	m.ensureCopied()
	m.part.Metadata().Delete(key)
}

// MetaWalk iterates each metadata key/value pair and executes a provided
// closure on each iteration. To stop iterating, return an error from the
// closure. An error returned by the closure will be returned by this function.
func (m *Message) MetaWalk(fn func(string, string) error) error {
	return m.part.Metadata().Iter(fn)
}

//------------------------------------------------------------------------------

// BloblangQuery executes a parsed Bloblang mapping on a message and returns a
// message back or an error if the mapping fails. If the mapping results in the
// root being deleted the returned message will be nil, which indicates it has
// been filtered.
func (m *Message) BloblangQuery(blobl *bloblang.Executor) (*Message, error) {
	uw := blobl.XUnwrapper().(interface {
		Unwrap() *mapping.Executor
	}).Unwrap()

	msg := message.New(nil)
	msg.Append(m.part)

	res, err := uw.MapPart(0, msg)
	if err != nil {
		return nil, err
	}
	if res != nil {
		return newMessageFromPart(res), nil
	}
	return nil, nil
}

// BloblangQuery executes a parsed Bloblang mapping on a message batch, from the
// perspective of a particular message index, and returns a message back or an
// error if the mapping fails. If the mapping results in the root being deleted
// the returned message will be nil, which indicates it has been filtered.
//
// This method allows mappings to perform windowed aggregations across message
// batches.
func (b MessageBatch) BloblangQuery(index int, blobl *bloblang.Executor) (*Message, error) {
	uw := blobl.XUnwrapper().(interface {
		Unwrap() *mapping.Executor
	}).Unwrap()

	msg := message.New(nil)
	for _, m := range b {
		msg.Append(m.part)
	}

	res, err := uw.MapPart(index, msg)
	if err != nil {
		return nil, err
	}
	if res != nil {
		return newMessageFromPart(res), nil
	}
	return nil, nil
}

// InterpolatedString resolves an interpolated string expression on a message
// batch, from the perspective of a particular message index.
//
// This method allows interpolation functions to perform windowed aggregations
// across message batches, and is a more powerful way to interpolate strings
// than the standard .String method.
func (b MessageBatch) InterpolatedString(index int, i *InterpolatedString) string {
	msg := message.New(nil)
	for _, m := range b {
		msg.Append(m.part)
	}
	return i.expr.String(index, msg)
}
<commit_msg>Add context methods to message type<commit_after>package service

import (
	"context"

	"github.com/Jeffail/benthos/v3/internal/bloblang/mapping"
	"github.com/Jeffail/benthos/v3/lib/message"
	"github.com/Jeffail/benthos/v3/lib/processor"
	"github.com/Jeffail/benthos/v3/lib/types"
	"github.com/Jeffail/benthos/v3/public/bloblang"
)

// MessageHandlerFunc is a function signature defining a component that consumes
// Benthos messages. An error must be returned if the context is cancelled, or
// if the message could not be delivered or processed.
type MessageHandlerFunc func(context.Context, *Message) error

// Message represents a single discrete message passing through a Benthos
// pipeline. It is safe to mutate the message via Set methods, but the
// underlying byte data should not be edited directly.
type Message struct {
	part       types.Part
	partCopied bool
}

// MessageBatch describes a collection of one or more messages.
type MessageBatch []*Message

// Copy creates a new slice of the same messages, which can be modified without
// changing the contents of the original batch.
func (b MessageBatch) Copy() MessageBatch {
	bCopy := make(MessageBatch, len(b))
	for i, m := range b {
		bCopy[i] = m.Copy()
	}
	return bCopy
}

// NewMessage creates a new message with an initial raw bytes content. The
// initial content can be nil, which is recommended if you intend to set it with
// structured contents.
func NewMessage(content []byte) *Message {
	return &Message{
		part:       message.NewPart(content),
		partCopied: true,
	}
}

func newMessageFromPart(part types.Part) *Message {
	return &Message{part, false}
}

// Copy creates a shallow copy of a message that is safe to mutate with Set
// methods without mutating the original. Both messages will share a context,
// and therefore a tracing ID, if one has been associated with them.
//
// Note that this does not perform a deep copy of the byte or structured
// contents of the message, and therefore it is not safe to perform inline
// mutations on those values without copying them.
func (m *Message) Copy() *Message {
	return &Message{
		part:       m.part.Copy(),
		partCopied: true,
	}
}

func (m *Message) ensureCopied() {
	if !m.partCopied {
		m.part = m.part.Copy()
		m.partCopied = true
	}
}

// Context returns a context associated with the message, or a background
// context in the absence of one.
func (m *Message) Context() context.Context {
	return message.GetContext(m.part)
}

// WithContext returns a new message with a provided context associated with it.
func (m *Message) WithContext(ctx context.Context) *Message {
	return &Message{
		part:       message.WithContext(ctx, m.part),
		partCopied: m.partCopied,
	}
}

// AsBytes returns the underlying byte array contents of a message or, if the
// contents are a structured type, attempts to marshal the contents as a JSON
// document and returns either the byte array result or an error.
//
// It is NOT safe to mutate the contents of the returned slice.
func (m *Message) AsBytes() ([]byte, error) {
	// TODO: Escalate errors in marshalling once we're able.
	return m.part.Get(), nil
}

// AsStructured returns the underlying structured contents of a message or, if
// the contents are a byte array, attempts to parse the bytes contents as a JSON
// document and returns either the structured result or an error.
//
// It is NOT safe to mutate the contents of the returned value if it is a
// reference type (slice or map). In order to safely mutate the structured
// contents of a message use AsStructuredMut.
func (m *Message) AsStructured() (interface{}, error) {
	return m.part.JSON()
}

// AsStructuredMut returns the underlying structured contents of a message or,
// if the contents are a byte array, attempts to parse the bytes contents as a
// JSON document and returns either the structured result or an error.
//
// It is safe to mutate the contents of the returned value even if it is a
// reference type (slice or map), as the structured contents will be lazily deep
// cloned if it is still owned by an upstream component.
func (m *Message) AsStructuredMut() (interface{}, error) {
	// TODO: Use refactored APIs to determine if the contents are owned.
	v, err := m.part.JSON()
	if err != nil {
		return nil, err
	}
	return message.CopyJSON(v)
}

// SetBytes sets the underlying contents of the message as a byte slice.
func (m *Message) SetBytes(b []byte) {
	m.ensureCopied()
	m.part.Set(b)
}

// SetStructured sets the underlying contents of the message as a structured
// type. This structured value should be a scalar Go type, or either a
// map[string]interface{} or []interface{} containing the same types all the way
// through the hierarchy, this ensures that other processors are able to work
// with the contents and that they can be JSON marshalled when coerced into a
// byte array.
func (m *Message) SetStructured(i interface{}) {
	m.ensureCopied()
	m.part.SetJSON(i)
}

// SetError marks the message as having failed a processing step and adds the
// error to it as context. Messages marked with errors can be handled using a
// range of methods outlined in https://www.benthos.dev/docs/configuration/error_handling.
func (m *Message) SetError(err error) {
	m.ensureCopied()
	processor.FlagErr(m.part, err)
}

// MetaGet attempts to find a metadata key from the message and returns a string
// result and a boolean indicating whether it was found.
func (m *Message) MetaGet(key string) (string, bool) {
	v := m.part.Metadata().Get(key)
	return v, len(v) > 0
}

// MetaSet sets the value of a metadata key. If the value is an empty string the
// metadata key is deleted.
func (m *Message) MetaSet(key, value string) {
	m.ensureCopied()
	if value == "" {
		m.part.Metadata().Delete(key)
	} else {
		m.part.Metadata().Set(key, value)
	}
}

// MetaDelete removes a key from the message metadata.
func (m *Message) MetaDelete(key string) {
	m.ensureCopied()
	m.part.Metadata().Delete(key)
}

// MetaWalk iterates each metadata key/value pair and executes a provided
// closure on each iteration. To stop iterating, return an error from the
// closure. An error returned by the closure will be returned by this function.
func (m *Message) MetaWalk(fn func(string, string) error) error {
	return m.part.Metadata().Iter(fn)
}

//------------------------------------------------------------------------------

// BloblangQuery executes a parsed Bloblang mapping on a message and returns a
// message back or an error if the mapping fails. If the mapping results in the
// root being deleted the returned message will be nil, which indicates it has
// been filtered.
func (m *Message) BloblangQuery(blobl *bloblang.Executor) (*Message, error) {
	uw := blobl.XUnwrapper().(interface {
		Unwrap() *mapping.Executor
	}).Unwrap()

	msg := message.New(nil)
	msg.Append(m.part)

	res, err := uw.MapPart(0, msg)
	if err != nil {
		return nil, err
	}
	if res != nil {
		return newMessageFromPart(res), nil
	}
	return nil, nil
}

// BloblangQuery executes a parsed Bloblang mapping on a message batch, from the
// perspective of a particular message index, and returns a message back or an
// error if the mapping fails. If the mapping results in the root being deleted
// the returned message will be nil, which indicates it has been filtered.
//
// This method allows mappings to perform windowed aggregations across message
// batches.
func (b MessageBatch) BloblangQuery(index int, blobl *bloblang.Executor) (*Message, error) {
	uw := blobl.XUnwrapper().(interface {
		Unwrap() *mapping.Executor
	}).Unwrap()

	msg := message.New(nil)
	for _, m := range b {
		msg.Append(m.part)
	}

	res, err := uw.MapPart(index, msg)
	if err != nil {
		return nil, err
	}
	if res != nil {
		return newMessageFromPart(res), nil
	}
	return nil, nil
}

// InterpolatedString resolves an interpolated string expression on a message
// batch, from the perspective of a particular message index.
//
// This method allows interpolation functions to perform windowed aggregations
// across message batches, and is a more powerful way to interpolate strings
// than the standard .String method.
func (b MessageBatch) InterpolatedString(index int, i *InterpolatedString) string {
	msg := message.New(nil)
	for _, m := range b {
		msg.Append(m.part)
	}
	return i.expr.String(index, msg)
}
<|endoftext|>


<commit_before>package cast

import (
	"math"

	"testing"
)

func TestUint64FromUint64(t *testing.T) {

	tests := []struct{
		Value uint64
	}{
		{
			Value: 0,
		},
		{
			Value: 1,
		},
		{
			Value: math.MaxUint64,
		},
	}

	{
		const numRand = 20
		for i:=0; i<numRand; i++ {
			test := struct{
				Value uint64
			}{
				Value: (uint64(randomness.Uint32()) << 32) + uint64(randomness.Uint32()),
			}
			tests = append(tests, test)
		}
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint64(x)

		if expected, actual := test.Value, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint64er(t *testing.T) {

	tests := []struct{
		Value    uint64er
		Expected uint64
	}{
		{
			Value: testUint64erZero(),
			Expected:          0,
		},
		{
			Value: testUint64erOne(),
			Expected:          1,
		},
		{
			Value: testUint64erMax(),
			Expected:     math.MaxUint64,
		},
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint64(x)

		if expected, actual := test.Expected, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint32(t *testing.T) {

	tests := []struct{
		Value uint32
	}{
		{
			Value: 0,
		},
		{
			Value: 1,
		},
		{
			Value: math.MaxUint32,
		},
	}

	{
		const numRand = 20
		for i:=0; i<numRand; i++ {
			test := struct{
				Value uint32
			}{
				Value: uint32(randomness.Int63n(math.MaxUint32)),
			}
			tests = append(tests, test)
		}
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint32(x)

		if expected, actual := test.Value, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint32er(t *testing.T) {

	tests := []struct{
		Value    uint32er
		Expected uint32
	}{
		{
			Value: testUint32erZero(),
			Expected:          0,
		},
		{
			Value: testUint32erOne(),
			Expected:          1,
		},
		{
			Value: testUint32erMax(),
			Expected:     math.MaxUint32,
		},
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint32(x)

		if expected, actual := test.Expected, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint16(t *testing.T) {

	tests := []struct{
		Value uint16
	}{
		{
			Value: 0,
		},
		{
			Value: 1,
		},
		{
			Value: math.MaxUint16,
		},
	}

	{
		const numRand = 20
		for i:=0; i<numRand; i++ {
			test := struct{
				Value uint16
			}{
				Value: uint16(randomness.Int63n(math.MaxUint16)),
			}
			tests = append(tests, test)
		}
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint16(x)

		if expected, actual := test.Value, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint16er(t *testing.T) {

	tests := []struct{
		Value    uint16er
		Expected uint16
	}{
		{
			Value: testUint16erZero(),
			Expected:          0,
		},
		{
			Value: testUint16erOne(),
			Expected:          1,
		},
		{
			Value: testUint16erMax(),
			Expected:     math.MaxUint16,
		},
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint16(x)

		if expected, actual := test.Expected, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint8(t *testing.T) {

	tests := []struct{
		Value uint8
	}{
		{
			Value: 0,
		},
		{
			Value: 1,
		},
		{
			Value: math.MaxUint8,
		},
	}

	{
		const numRand = 20
		for i:=0; i<numRand; i++ {
			test := struct{
				Value uint8
			}{
				Value: uint8(randomness.Int63n(math.MaxUint8)),
			}
			tests = append(tests, test)
		}
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint8(x)

		if expected, actual := test.Value, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}
<commit_msg>added more tests<commit_after>package cast

import (
	"math"

	"testing"
)

func TestUint64FromUint64(t *testing.T) {

	tests := []struct{
		Value uint64
	}{
		{
			Value: 0,
		},
		{
			Value: 1,
		},
		{
			Value: math.MaxUint64,
		},
	}

	{
		const numRand = 20
		for i:=0; i<numRand; i++ {
			test := struct{
				Value uint64
			}{
				Value: (uint64(randomness.Uint32()) << 32) + uint64(randomness.Uint32()),
			}
			tests = append(tests, test)
		}
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint64(x)

		if expected, actual := test.Value, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint64er(t *testing.T) {

	tests := []struct{
		Value    uint64er
		Expected uint64
	}{
		{
			Value: testUint64erZero(),
			Expected:          0,
		},
		{
			Value: testUint64erOne(),
			Expected:          1,
		},
		{
			Value: testUint64erMax(),
			Expected:     math.MaxUint64,
		},
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint64(x)

		if expected, actual := test.Expected, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint32(t *testing.T) {

	tests := []struct{
		Value uint32
	}{
		{
			Value: 0,
		},
		{
			Value: 1,
		},
		{
			Value: math.MaxUint32,
		},
	}

	{
		const numRand = 20
		for i:=0; i<numRand; i++ {
			test := struct{
				Value uint32
			}{
				Value: uint32(randomness.Int63n(math.MaxUint32)),
			}
			tests = append(tests, test)
		}
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint32(x)

		if expected, actual := test.Value, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint32er(t *testing.T) {

	tests := []struct{
		Value    uint32er
		Expected uint32
	}{
		{
			Value: testUint32erZero(),
			Expected:          0,
		},
		{
			Value: testUint32erOne(),
			Expected:          1,
		},
		{
			Value: testUint32erMax(),
			Expected:     math.MaxUint32,
		},
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint32(x)

		if expected, actual := test.Expected, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint16(t *testing.T) {

	tests := []struct{
		Value uint16
	}{
		{
			Value: 0,
		},
		{
			Value: 1,
		},
		{
			Value: math.MaxUint16,
		},
	}

	{
		const numRand = 20
		for i:=0; i<numRand; i++ {
			test := struct{
				Value uint16
			}{
				Value: uint16(randomness.Int63n(math.MaxUint16)),
			}
			tests = append(tests, test)
		}
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint16(x)

		if expected, actual := test.Value, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint16er(t *testing.T) {

	tests := []struct{
		Value    uint16er
		Expected uint16
	}{
		{
			Value: testUint16erZero(),
			Expected:          0,
		},
		{
			Value: testUint16erOne(),
			Expected:          1,
		},
		{
			Value: testUint16erMax(),
			Expected:     math.MaxUint16,
		},
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint16(x)

		if expected, actual := test.Expected, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint8(t *testing.T) {

	tests := []struct{
		Value uint8
	}{
		{
			Value: 0,
		},
		{
			Value: 1,
		},
		{
			Value: math.MaxUint8,
		},
	}

	{
		const numRand = 20
		for i:=0; i<numRand; i++ {
			test := struct{
				Value uint8
			}{
				Value: uint8(randomness.Int63n(math.MaxUint8)),
			}
			tests = append(tests, test)
		}
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint8(x)

		if expected, actual := test.Value, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}

func TestUint64FromUint8er(t *testing.T) {

	tests := []struct{
		Value    uint8er
		Expected uint8
	}{
		{
			Value: testUint8erZero(),
			Expected:         0,
		},
		{
			Value: testUint8erOne(),
			Expected:         1,
		},
		{
			Value: testUint8erMax(),
			Expected:    math.MaxUint8,
		},
	}

	for testNumber, test := range tests {

		x, err := Uint64(test.Value)
		if nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		y := uint8(x)

		if expected, actual := test.Expected, y; expected != actual {
			t.Errorf("For test #%d, expected %v, but actually got %v.", testNumber, expected, actual)
			continue
		}
	}
}
<|endoftext|>