<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/cycps/addie"
	"github.com/cycps/addie/db"
	"github.com/cycps/addie/protocol"
	"github.com/julienschmidt/httprouter"
	"log"
	"net/http"
	"os"
)

var design addie.Design
var cypdir = os.ExpandEnv("$HOME/.cypress")

func main() {

	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "usage: go-addie <design id>\n")
		os.Exit(1)
	}

	loadDesign(os.Args[1])
	listen()
}

func loadDesign(id string) {

	design = addie.EmptyDesign(id)

}

func unpack(r *http.Request, x interface{}) error {
	buf := new(bytes.Buffer)
	buf.ReadFrom(r.Body)
	err := json.Unmarshal(buf.Bytes(), &x)
	if err != nil {
		log.Println("[unpack] bad message")
		log.Println(err)
		log.Println(buf.String())
		return nil
	}
	return nil
}

func dbCreate(e addie.Identify) {
	log.Printf("[dbCreate] %T '%s'", e, e.Identify())
	var err error = nil

	switch t := e.(type) {
	case addie.Computer:
		c := e.(addie.Computer)
		err = db.CreateComputer(c)
	case addie.Switch:
		s := e.(addie.Switch)
		err = db.CreateSwitch(s)
	case addie.Router:
		r := e.(addie.Router)
		err = db.CreateRouter(r)
	case addie.Link:
		l := e.(addie.Link)
		err = db.CreateLink(l)
		/*
			case addie.Model:
				m := e.(addie.Model)
			case addie.Sensor:
				s := e.(addie.Sensor)
			case addie.Actuator:
				a := e.(addie.Actuator)
		*/
	default:
		log.Printf("[dbCreate] unkown or unsupported element type: %T \n", t)
	}

	if err != nil {
		log.Println(err)
	}

}

func dbUpdate(oid addie.Id, e addie.Identify) {
	log.Printf("[dbUpdate] %T '%s'", e, e.Identify())
	var err error = nil

	old, ok := design.Elements[oid]
	if !ok {
		log.Printf("[Update] bad oid %v\n", oid)
		return
	}

	//todo perform check old.(type) conversion
	switch t := e.(type) {
	case addie.Computer:
		c := e.(addie.Computer)
		_, err = db.UpdateComputer(oid, old.(addie.Computer), c)
	case addie.Switch:
		s := e.(addie.Switch)
		_, err = db.UpdateSwitch(oid, old.(addie.Switch), s)
	case addie.Router:
		r := e.(addie.Router)
		_, err = db.UpdateRouter(oid, old.(addie.Router), r)
	case addie.Link:
		l := e.(addie.Link)
		_, err = db.UpdateLink(oid, l)
		/*
			case addie.Model:
				m := e.(addie.Model)
			case addie.Sensor:
				s := e.(addie.Sensor)
			case addie.Actuator:
				a := e.(addie.Actuator)
		*/
	default:
		log.Printf("[dbUpdate] unkown or unsupported element type: %T \n", t)
	}

	if err != nil {
		log.Println(err)
	}

}

//func updateElement(id addie.Id, e addie.Identify) {
func dedup(id addie.Id, e addie.Identify) {

	//_, ok := design.Elements[e.Identify()]

	//design.Elements[e.Identify()] = e
	if e.Identify() != id {
		log.Println("deleting ", id)
		log.Println("new-id ", e.Identify())
		delete(design.Elements, id)
	}

	//collect updates and inserts, then do updates first as inserts
	//may depend on updated data
	//var updates, inserts []addie.Identify

	/*
		if !ok {
			inserts = append(inserts, e)
		} else {
			updates = append(updates, e)
		}

		for _, u := range updates {
			dbUpdate(u)
		}
		for _, i := range inserts {
			dbInsert(i)
		}
	*/

}

func onUpdate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

	//unpack the message
	msg := new(protocol.Update)
	err := unpack(r, msg)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	var updates, creates []addie.Identify
	var update_oids []addie.Id

	var place = func(oid addie.Id, e addie.Identify) {
		_, ok := design.Elements[e.Identify()]
		if !ok {
			creates = append(creates, e)
		} else {
			updates = append(updates, e)
			update_oids = append(update_oids, oid)
		}
	}

	for _, u := range msg.Elements {

		switch u.Type {
		case "Computer":
			var c addie.Computer
			err := json.Unmarshal(u.Element, &c)
			log.Println(c)
			if err != nil {
				log.Println("unable to unmarshal computer")
			}
			dedup(u.OID, c)
			place(u.OID, c)
		case "Switch":
			var s addie.Switch
			err := json.Unmarshal(u.Element, &s)
			if err != nil {
				log.Println("unable to unmarshal switch")
			}
			dedup(u.OID, s)
			place(u.OID, s)
		case "Router":
			var r addie.Router
			err := json.Unmarshal(u.Element, &r)
			if err != nil {
				log.Println("unable to unmarshal router")
			}
			dedup(u.OID, r)
			place(u.OID, r)
		case "Model":
			var m addie.Model
			err := json.Unmarshal(u.Element, &m)
			if err != nil {
				log.Println("unable to unmarshal model")
			}
			dedup(u.OID, m)
			place(u.OID, m)
		case "Sensor":
			var s addie.Sensor
			err := json.Unmarshal(u.Element, &s)
			if err != nil {
				log.Println("unable to unmarshal sensor")
			}
			dedup(u.OID, s)
			place(u.OID, s)
		case "Actuator":
			var a addie.Actuator
			err := json.Unmarshal(u.Element, &a)
			if err != nil {
				log.Println("unable to unmarshal sensor")
			}
			dedup(u.OID, a)
			place(u.OID, a)
		case "Link":
			var l addie.Link
			err := json.Unmarshal(u.Element, &l)
			if err != nil {
				log.Println("unable to unmarshal link")
			}
			dedup(u.OID, l)
			place(u.OID, l)
		default:
			log.Println("unkown element type: ", u.Type)
		}

	}

	for i, u := range updates {
		dbUpdate(update_oids[i], u)
		design.Elements[u.Identify()] = u
	}
	for _, c := range creates {
		dbCreate(c)
		design.Elements[c.Identify()] = c
	}

	log.Println("\n", design.String())

	//send response
	w.WriteHeader(http.StatusOK)

}

func onDelete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

	//TODO thundermuffin

}

func listen() {

	router := httprouter.New()
	router.POST("/:xpid/design/update", onUpdate)
	//router.POST("/:xpid/design/delete", onDelete)

	log.Printf("listening on https://::0:8080/%s/", design.Name)
	log.Fatal(
		http.ListenAndServeTLS(":8080",
			cypdir+"/keys/cert.pem",
			cypdir+"/keys/key.pem",
			router))

}
<commit_msg>clean up a spurrious msg<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/cycps/addie"
	"github.com/cycps/addie/db"
	"github.com/cycps/addie/protocol"
	"github.com/julienschmidt/httprouter"
	"log"
	"net/http"
	"os"
)

var design addie.Design
var cypdir = os.ExpandEnv("$HOME/.cypress")

func main() {

	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "usage: go-addie <design id>\n")
		os.Exit(1)
	}

	loadDesign(os.Args[1])
	listen()
}

func loadDesign(id string) {

	design = addie.EmptyDesign(id)

}

func unpack(r *http.Request, x interface{}) error {
	buf := new(bytes.Buffer)
	buf.ReadFrom(r.Body)
	err := json.Unmarshal(buf.Bytes(), &x)
	if err != nil {
		log.Println("[unpack] bad message")
		log.Println(err)
		log.Println(buf.String())
		return nil
	}
	return nil
}

func dbCreate(e addie.Identify) {
	log.Printf("[dbCreate] %T '%s'", e, e.Identify())
	var err error = nil

	switch t := e.(type) {
	case addie.Computer:
		c := e.(addie.Computer)
		err = db.CreateComputer(c)
	case addie.Switch:
		s := e.(addie.Switch)
		err = db.CreateSwitch(s)
	case addie.Router:
		r := e.(addie.Router)
		err = db.CreateRouter(r)
	case addie.Link:
		l := e.(addie.Link)
		err = db.CreateLink(l)
		/*
			case addie.Model:
				m := e.(addie.Model)
			case addie.Sensor:
				s := e.(addie.Sensor)
			case addie.Actuator:
				a := e.(addie.Actuator)
		*/
	default:
		log.Printf("[dbCreate] unkown or unsupported element type: %T \n", t)
	}

	if err != nil {
		log.Println(err)
	}

}

func dbUpdate(oid addie.Id, e addie.Identify) {
	log.Printf("[dbUpdate] %T '%s'", e, e.Identify())
	var err error = nil

	old, ok := design.Elements[oid]
	if !ok {
		log.Printf("[Update] bad oid %v\n", oid)
		return
	}

	//todo perform check old.(type) conversion
	switch t := e.(type) {
	case addie.Computer:
		c := e.(addie.Computer)
		_, err = db.UpdateComputer(oid, old.(addie.Computer), c)
	case addie.Switch:
		s := e.(addie.Switch)
		_, err = db.UpdateSwitch(oid, old.(addie.Switch), s)
	case addie.Router:
		r := e.(addie.Router)
		_, err = db.UpdateRouter(oid, old.(addie.Router), r)
	case addie.Link:
		l := e.(addie.Link)
		_, err = db.UpdateLink(oid, l)
		/*
			case addie.Model:
				m := e.(addie.Model)
			case addie.Sensor:
				s := e.(addie.Sensor)
			case addie.Actuator:
				a := e.(addie.Actuator)
		*/
	default:
		log.Printf("[dbUpdate] unkown or unsupported element type: %T \n", t)
	}

	if err != nil {
		log.Println(err)
	}

}

//func updateElement(id addie.Id, e addie.Identify) {
func dedup(id addie.Id, e addie.Identify) {

	//_, ok := design.Elements[e.Identify()]

	//design.Elements[e.Identify()] = e
	if e.Identify() != id {
		log.Println("deleting ", id)
		log.Println("new-id ", e.Identify())
		delete(design.Elements, id)
	}

	//collect updates and inserts, then do updates first as inserts
	//may depend on updated data
	//var updates, inserts []addie.Identify

	/*
		if !ok {
			inserts = append(inserts, e)
		} else {
			updates = append(updates, e)
		}

		for _, u := range updates {
			dbUpdate(u)
		}
		for _, i := range inserts {
			dbInsert(i)
		}
	*/

}

func onUpdate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

	//unpack the message
	msg := new(protocol.Update)
	err := unpack(r, msg)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	var updates, creates []addie.Identify
	var update_oids []addie.Id

	var place = func(oid addie.Id, e addie.Identify) {
		_, ok := design.Elements[e.Identify()]
		if !ok {
			creates = append(creates, e)
		} else {
			updates = append(updates, e)
			update_oids = append(update_oids, oid)
		}
	}

	for _, u := range msg.Elements {

		switch u.Type {
		case "Computer":
			var c addie.Computer
			err := json.Unmarshal(u.Element, &c)
			if err != nil {
				log.Println("unable to unmarshal computer")
			}
			dedup(u.OID, c)
			place(u.OID, c)
		case "Switch":
			var s addie.Switch
			err := json.Unmarshal(u.Element, &s)
			if err != nil {
				log.Println("unable to unmarshal switch")
			}
			dedup(u.OID, s)
			place(u.OID, s)
		case "Router":
			var r addie.Router
			err := json.Unmarshal(u.Element, &r)
			if err != nil {
				log.Println("unable to unmarshal router")
			}
			dedup(u.OID, r)
			place(u.OID, r)
		case "Model":
			var m addie.Model
			err := json.Unmarshal(u.Element, &m)
			if err != nil {
				log.Println("unable to unmarshal model")
			}
			dedup(u.OID, m)
			place(u.OID, m)
		case "Sensor":
			var s addie.Sensor
			err := json.Unmarshal(u.Element, &s)
			if err != nil {
				log.Println("unable to unmarshal sensor")
			}
			dedup(u.OID, s)
			place(u.OID, s)
		case "Actuator":
			var a addie.Actuator
			err := json.Unmarshal(u.Element, &a)
			if err != nil {
				log.Println("unable to unmarshal sensor")
			}
			dedup(u.OID, a)
			place(u.OID, a)
		case "Link":
			var l addie.Link
			err := json.Unmarshal(u.Element, &l)
			if err != nil {
				log.Println("unable to unmarshal link")
			}
			dedup(u.OID, l)
			place(u.OID, l)
		default:
			log.Println("unkown element type: ", u.Type)
		}

	}

	for i, u := range updates {
		dbUpdate(update_oids[i], u)
		design.Elements[u.Identify()] = u
	}
	for _, c := range creates {
		dbCreate(c)
		design.Elements[c.Identify()] = c
	}

	log.Println("\n", design.String())

	//send response
	w.WriteHeader(http.StatusOK)

}

func onDelete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {

	//TODO thundermuffin

}

func listen() {

	router := httprouter.New()
	router.POST("/:xpid/design/update", onUpdate)
	//router.POST("/:xpid/design/delete", onDelete)

	log.Printf("listening on https://::0:8080/%s/", design.Name)
	log.Fatal(
		http.ListenAndServeTLS(":8080",
			cypdir+"/keys/cert.pem",
			cypdir+"/keys/key.pem",
			router))

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"log"
	"math/rand"
	"path/filepath"
	"sync"

	. "github.com/dvyukov/go-fuzz/go-fuzz-defs"
)

type SonarSite struct {
	id  int    // unique site id (const)
	loc string // file:line.pos,line.pos (const)
	sync.Mutex
	dynamic    bool   // both operands are not constant
	takenFuzz  [2]int // number of times condition evaluated to false/true during fuzzing
	takenTotal [2]int // number of times condition evaluated to false/true in total
	val        [2][]byte
}

type SonarSample struct {
	site  *SonarSite
	flags byte
	val   [2][]byte
}

func (s *Slave) parseSonarData(sonar []byte) (res []SonarSample) {
	ro := s.hub.ro.Load().(*ROData)
	sonar = makeCopy(sonar)
	for len(sonar) > SonarHdrLen {
		id := binary.LittleEndian.Uint32(sonar)
		flags := byte(id)
		id >>= 8
		n1 := sonar[4]
		n2 := sonar[5]
		sonar = sonar[SonarHdrLen:]
		if n1 > SonarMaxLen || n2 > SonarMaxLen || len(sonar) < int(n1)+int(n2) {
			log.Fatalf("corrputed sonar data: hdr=[%v/%v/%v] data=%v", flags, n1, n2, len(sonar))
		}
		v1 := makeCopy(sonar[:n1])
		v2 := makeCopy(sonar[n1 : n1+n2])
		sonar = sonar[n1+n2:]

		// Trim trailing 0x00 and 0xff bytes (we don't know exact size of operands).
		if flags&SonarString == 0 {
			for len(v1) > 0 || len(v2) > 0 {
				i := len(v1) - 1
				if len(v2) > len(v1) {
					i = len(v2) - 1
				}
				var c1, c2 byte
				if i < len(v1) {
					c1 = v1[i]
				}
				if i < len(v2) {
					c2 = v2[i]
				}
				if (c1 == 0 || c1 == 0xff) && (c2 == 0 || c2 == 0xff) {
					if i < len(v1) {
						v1 = v1[:i]
					}
					if i < len(v2) {
						v2 = v2[:i]
					}
					continue
				}
				break
			}
		}

		res = append(res, SonarSample{&ro.sonarSites[id], flags, [2][]byte{v1, v2}})
	}
	return res
}

func (s *Slave) processSonarData(data, sonar []byte, depth int, smash bool) {
	ro := s.hub.ro.Load().(*ROData)
	updated := false
	checked := make(map[string]struct{})
	samples := s.parseSonarData(sonar)
	for _, sam := range samples {
		site := sam.site
		flags := sam.flags
		v1 := sam.val[0]
		v2 := sam.val[1]
		res := sam.evaluate()
		// Ignore sites that has at least one const operand and
		// are already taken both ways enough times.
		upd, skip := site.update(sam, smash, res)
		if site.takenTotal[0] == 10 && site.takenTotal[1] == 0 ||
			site.takenTotal[1] == 10 && site.takenTotal[0] == 0 {
			//s.bruteForce(data, samples, sam.site)
		}
		if upd {
			updated = true
		}
		if skip {
			continue
		}
		if smash && bytes.Equal(v1, v2) {
			// We systematically mutate all bytes during smashing,
			// no point in trying to break equality here.
			continue
		}
		check := func(v1, v2 []byte) {
			if len(v1) == 0 || bytes.Equal(v1, v2) {
				return
			}
			vv := string(v1) + "\t|\t" + string(v2)
			if _, ok := checked[vv]; ok {
				return
			}
			checked[vv] = struct{}{}
			pos := 0
			for {
				i := bytes.Index(data[pos:], v1)
				if i == -1 {
					break
				}
				i += pos
				pos = i + 1
				tmp := make([]byte, len(data)-len(v1)+len(v2))
				copy(tmp, data[:i])
				copy(tmp[i:], v2)
				copy(tmp[i+len(v2):], data[i+len(v1):])
				s.testInput(tmp, depth+1, execSonarHint)
				if flags&SonarString != 0 && len(v1) != len(v2) {
					// Update length field.
					// TODO: handle multi-byte/big-endian length fields.
					diff := byte(len(v2) - len(v1))
					for idx := i - 1; idx >= 0 && idx+5 >= i; idx-- {
						//tmp1 := append([]byte{}, tmp...)
						tmp[idx] += diff
						s.testInput(tmp, depth+1, execSonarHint)
						tmp[idx] -= diff
					}
				}
			}
		}
		check1 := func(v1, v2 []byte) {
			check(v1, v2)
			// Try several common wire encodings of the values:
			// network format (big endian), hex, base-128.
			// TODO: try more encodings if it proves to be useful:
			// base-64, quoted-printable, xml-escaping? hex+increment/decrement?
			if flags&SonarString == 0 {
				// Increment and decrement take care of less and greater comparison operators
				// as well as of off-by-one bugs.
				check(v1, increment(v2))
				check(v1, decrement(v2))
				if len(v1) > 1 {
					check(reverse(v1), reverse(v2))
					check(reverse(v1), reverse(increment(v2)))
					check(reverse(v1), reverse(decrement(v2)))
				}
			}
			check([]byte(hex.EncodeToString(v1)), []byte(hex.EncodeToString(v2)))
			if flags&SonarString == 0 {
				var u1, u2 uint64
				for i := 0; i < len(v1); i++ {
					u1 += uint64(v1[i]) << uint(i*8)
				}
				for i := 0; i < len(v2); i++ {
					u2 += uint64(v2[i]) << uint(i*8)
				}
				var vv1, vv2 [10]byte
				n1 := binary.PutUvarint(vv1[:], u1)
				n2 := binary.PutUvarint(vv2[:], u2)
				check(vv1[:n1], vv2[:n2])
			}
		}
		if flags&SonarConst1 == 0 {
			check1(v1, v2)
		}
		if flags&SonarConst2 == 0 {
			check1(v2, v1)
		}
	}
	if updated && *flagDumpCover {
		dumpMu.Lock()
		defer dumpMu.Unlock()
		dumpSonar(filepath.Join(*flagWorkdir, "sonarprofile"), ro.sonarSites)
	}
}

var dumpMu sync.Mutex

func (site *SonarSite) update(sam SonarSample, smash, resb bool) (updated, skip bool) {
	res := 0
	if resb {
		res = 1
	}
	site.Lock()
	defer site.Unlock()
	v1 := sam.val[0]
	v2 := sam.val[1]
	if !site.dynamic && sam.flags&SonarConst1+sam.flags&SonarConst2 == 0 {
		if site.val[0] == nil {
			site.val[0] = makeCopy(v1)
		}
		if site.val[1] == nil {
			site.val[1] = makeCopy(v2)
		}
		if !bytes.Equal(site.val[0], v1) && !bytes.Equal(site.val[1], v2) {
			site.val[0] = nil
			site.val[1] = nil
			site.dynamic = true
		}
	}
	if site.takenTotal[res] == 0 {
		updated = true
	}
	site.takenTotal[res]++
	if !smash {
		site.takenFuzz[res]++
	}
	if !site.dynamic && (site.takenFuzz[0] > 10 && site.takenFuzz[1] > 10 || site.takenFuzz[0]+site.takenFuzz[1] > 100) {
		if rand.Intn(10000) != 0 {
			skip = true
			return
		}
	}
	return
}

func (s *Slave) bruteForce(data []byte, samples []SonarSample, site *SonarSite) {
	si := -1
	var sam SonarSample
	for i, sam1 := range samples {
		if sam1.site == site {
			sam = sam1
			si = i
			break
		}
	}
	if si == -1 {
		panic("can't find the sample")
	}
	var deps []int
	for i := range data {
		data[i] ^= 0xff
		sonar := s.testInputSonar(data, 0)
		data[i] ^= 0xff
		samples1 := s.parseSonarData(sonar)
		var sam1 SonarSample
		si1 := -1
		for i, sam2 := range samples1 {
			if sam2.site == site {
				si1 = i
				sam1 = sam2
				break
			}
		}
		if si1 == -1 {
			continue
		}
		if bytes.Equal(sam.val[0], sam1.val[0]) && bytes.Equal(sam.val[1], sam1.val[1]) {
			continue
		}
		matches := 0
		for i := 0; i < si && i < si1; i++ {
			if samples[i].site == samples1[i].site {
				matches++
			}
		}
		if matches+1 < si {
			for i := si - 1; i > 0 && i+si1-si > 0; i-- {
				if samples[i].site == samples1[i+si1-si].site {
					matches++
				}
			}
		}
		log.Printf("BRUTEFORCE: byte %v matches %v si=%v/%v", i, matches, si, si1)
		deps = append(deps, i)
	}
	log.Printf("BRUTEFORCE: %v deps=%+v", sam.site.loc, deps)
}

func (sam *SonarSample) evaluate() bool {
	v1 := sam.val[0]
	v2 := sam.val[1]
	if sam.flags&SonarString != 0 {
		s1 := string(v1)
		s2 := string(v2)
		switch sam.flags & SonarOpMask {
		case SonarEQL:
			return s1 == s2
		case SonarNEQ:
			return s1 != s2
		case SonarLSS:
			return s1 < s2
		case SonarGTR:
			return s1 > s2
		case SonarLEQ:
			return s1 <= s2
		case SonarGEQ:
			return s1 >= s2
		default:
			panic("bad")
		}
	}
	if len(v1) == 0 || len(v2) == 0 || len(v1) > 8 || len(v2) > 8 || len(v1) != len(v2) {
		return false
	}
	v1 = append([]byte{}, v1...)
	for len(v1) < 8 {
		if int8(v1[len(v1)-1]) >= 0 {
			v1 = append(v1, 0)
		} else {
			v1 = append(v1, 0xff)
		}
	}
	v2 = append([]byte{}, v2...)
	for len(v2) < 8 {
		if int8(v2[len(v2)-1]) >= 0 {
			v2 = append(v2, 0)
		} else {
			v2 = append(v2, 0xff)
		}
	}
	// Note: assuming le machine.
	if sam.flags&SonarSigned == 0 {
		s1 := binary.LittleEndian.Uint64(v1)
		s2 := binary.LittleEndian.Uint64(v2)
		switch sam.flags & SonarOpMask {
		case SonarEQL:
			return s1 == s2
		case SonarNEQ:
			return s1 != s2
		case SonarLSS:
			return s1 < s2
		case SonarGTR:
			return s1 > s2
		case SonarLEQ:
			return s1 <= s2
		case SonarGEQ:
			return s1 >= s2
		default:
			panic("bad")
		}
	} else {
		s1 := int64(binary.LittleEndian.Uint64(v1))
		s2 := int64(binary.LittleEndian.Uint64(v2))
		switch sam.flags & SonarOpMask {
		case SonarEQL:
			return s1 == s2
		case SonarNEQ:
			return s1 != s2
		case SonarLSS:
			return s1 < s2
		case SonarGTR:
			return s1 > s2
		case SonarLEQ:
			return s1 <= s2
		case SonarGEQ:
			return s1 >= s2
		default:
			panic("bad")
		}
	}
}

func dumpSonarData(site *SonarSite, flags byte, v1, v2 []byte) {
	// Debug output.
	op := ""
	switch flags & SonarOpMask {
	case SonarEQL:
		op = "=="
	case SonarNEQ:
		op = "!="
	case SonarLSS:
		op = "<"
	case SonarGTR:
		op = ">"
	case SonarLEQ:
		op = "<="
	case SonarGEQ:
		op = ">="
	default:
		log.Fatalf("bad")
	}
	sign := ""
	if flags&SonarSigned != 0 {
		sign = "(signed)"
	}
	isstr := ""
	if flags&SonarString != 0 {
		isstr = "(string)"
	}
	const1 := ""
	if flags&SonarConst1 != 0 {
		const1 = "c"
	}
	const2 := ""
	if flags&SonarConst2 != 0 {
		const2 = "c"
	}
	log.Printf("SONAR %v%v %v %v%v %v%v %v",
		hex.EncodeToString(v1), const1, op, hex.EncodeToString(v2), const2, sign, isstr, site.loc)
}
<commit_msg>add a todo + fix some http corpus bugs<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"log"
	"math/rand"
	"path/filepath"
	"sync"

	. "github.com/dvyukov/go-fuzz/go-fuzz-defs"
)

type SonarSite struct {
	id  int    // unique site id (const)
	loc string // file:line.pos,line.pos (const)
	sync.Mutex
	dynamic    bool   // both operands are not constant
	takenFuzz  [2]int // number of times condition evaluated to false/true during fuzzing
	takenTotal [2]int // number of times condition evaluated to false/true in total
	val        [2][]byte
}

type SonarSample struct {
	site  *SonarSite
	flags byte
	val   [2][]byte
}

func (s *Slave) parseSonarData(sonar []byte) (res []SonarSample) {
	ro := s.hub.ro.Load().(*ROData)
	sonar = makeCopy(sonar)
	for len(sonar) > SonarHdrLen {
		id := binary.LittleEndian.Uint32(sonar)
		flags := byte(id)
		id >>= 8
		n1 := sonar[4]
		n2 := sonar[5]
		sonar = sonar[SonarHdrLen:]
		if n1 > SonarMaxLen || n2 > SonarMaxLen || len(sonar) < int(n1)+int(n2) {
			log.Fatalf("corrputed sonar data: hdr=[%v/%v/%v] data=%v", flags, n1, n2, len(sonar))
		}
		v1 := makeCopy(sonar[:n1])
		v2 := makeCopy(sonar[n1 : n1+n2])
		sonar = sonar[n1+n2:]

		// Trim trailing 0x00 and 0xff bytes (we don't know exact size of operands).
		if flags&SonarString == 0 {
			for len(v1) > 0 || len(v2) > 0 {
				i := len(v1) - 1
				if len(v2) > len(v1) {
					i = len(v2) - 1
				}
				var c1, c2 byte
				if i < len(v1) {
					c1 = v1[i]
				}
				if i < len(v2) {
					c2 = v2[i]
				}
				if (c1 == 0 || c1 == 0xff) && (c2 == 0 || c2 == 0xff) {
					if i < len(v1) {
						v1 = v1[:i]
					}
					if i < len(v2) {
						v2 = v2[:i]
					}
					continue
				}
				break
			}
		}

		res = append(res, SonarSample{&ro.sonarSites[id], flags, [2][]byte{v1, v2}})
	}
	return res
}

func (s *Slave) processSonarData(data, sonar []byte, depth int, smash bool) {
	ro := s.hub.ro.Load().(*ROData)
	updated := false
	checked := make(map[string]struct{})
	samples := s.parseSonarData(sonar)
	for _, sam := range samples {
		site := sam.site
		flags := sam.flags
		v1 := sam.val[0]
		v2 := sam.val[1]
		res := sam.evaluate()
		// Ignore sites that has at least one const operand and
		// are already taken both ways enough times.
		upd, skip := site.update(sam, smash, res)
		if site.takenTotal[0] == 10 && site.takenTotal[1] == 0 ||
			site.takenTotal[1] == 10 && site.takenTotal[0] == 0 {
			//s.bruteForce(data, samples, sam.site)
		}
		if upd {
			updated = true
		}
		if skip {
			continue
		}
		if smash && bytes.Equal(v1, v2) {
			// We systematically mutate all bytes during smashing,
			// no point in trying to break equality here.
			continue
		}
		check := func(v1, v2 []byte) {
			if len(v1) == 0 || bytes.Equal(v1, v2) {
				return
			}
			vv := string(v1) + "\t|\t" + string(v2)
			if _, ok := checked[vv]; ok {
				return
			}
			checked[vv] = struct{}{}
			pos := 0
			for {
				i := bytes.Index(data[pos:], v1)
				if i == -1 {
					break
				}
				i += pos
				pos = i + 1
				tmp := make([]byte, len(data)-len(v1)+len(v2))
				copy(tmp, data[:i])
				copy(tmp[i:], v2)
				copy(tmp[i+len(v2):], data[i+len(v1):])
				s.testInput(tmp, depth+1, execSonarHint)
				if flags&SonarString != 0 && len(v1) != len(v2) {
					// Update length field.
					// TODO: handle multi-byte/big-endian length fields.
					diff := byte(len(v2) - len(v1))
					for idx := i - 1; idx >= 0 && idx+5 >= i; idx-- {
						//tmp1 := append([]byte{}, tmp...)
						tmp[idx] += diff
						s.testInput(tmp, depth+1, execSonarHint)
						tmp[idx] -= diff
					}
				}
			}
		}
		check1 := func(v1, v2 []byte) {
			check(v1, v2)
			// Try several common wire encodings of the values:
			// network format (big endian), hex, base-128.
			// TODO: try more encodings if it proves to be useful:
			// base-64, quoted-printable, xml-escaping? hex+increment/decrement?

			// TODO: ascii-encoding: 0xff -> "255"
			if flags&SonarString == 0 {
				// Increment and decrement take care of less and greater comparison operators
				// as well as of off-by-one bugs.
				check(v1, increment(v2))
				check(v1, decrement(v2))
				if len(v1) > 1 {
					check(reverse(v1), reverse(v2))
					check(reverse(v1), reverse(increment(v2)))
					check(reverse(v1), reverse(decrement(v2)))
				}
			}
			check([]byte(hex.EncodeToString(v1)), []byte(hex.EncodeToString(v2)))
			if flags&SonarString == 0 {
				var u1, u2 uint64
				for i := 0; i < len(v1); i++ {
					u1 += uint64(v1[i]) << uint(i*8)
				}
				for i := 0; i < len(v2); i++ {
					u2 += uint64(v2[i]) << uint(i*8)
				}
				var vv1, vv2 [10]byte
				n1 := binary.PutUvarint(vv1[:], u1)
				n2 := binary.PutUvarint(vv2[:], u2)
				check(vv1[:n1], vv2[:n2])
			}
		}
		if flags&SonarConst1 == 0 {
			check1(v1, v2)
		}
		if flags&SonarConst2 == 0 {
			check1(v2, v1)
		}
	}
	if updated && *flagDumpCover {
		dumpMu.Lock()
		defer dumpMu.Unlock()
		dumpSonar(filepath.Join(*flagWorkdir, "sonarprofile"), ro.sonarSites)
	}
}

var dumpMu sync.Mutex

func (site *SonarSite) update(sam SonarSample, smash, resb bool) (updated, skip bool) {
	res := 0
	if resb {
		res = 1
	}
	site.Lock()
	defer site.Unlock()
	v1 := sam.val[0]
	v2 := sam.val[1]
	if !site.dynamic && sam.flags&SonarConst1+sam.flags&SonarConst2 == 0 {
		if site.val[0] == nil {
			site.val[0] = makeCopy(v1)
		}
		if site.val[1] == nil {
			site.val[1] = makeCopy(v2)
		}
		if !bytes.Equal(site.val[0], v1) && !bytes.Equal(site.val[1], v2) {
			site.val[0] = nil
			site.val[1] = nil
			site.dynamic = true
		}
	}
	if site.takenTotal[res] == 0 {
		updated = true
	}
	site.takenTotal[res]++
	if !smash {
		site.takenFuzz[res]++
	}
	if !site.dynamic && (site.takenFuzz[0] > 10 && site.takenFuzz[1] > 10 || site.takenFuzz[0]+site.takenFuzz[1] > 100) {
		if rand.Intn(10000) != 0 {
			skip = true
			return
		}
	}
	return
}

func (s *Slave) bruteForce(data []byte, samples []SonarSample, site *SonarSite) {
	si := -1
	var sam SonarSample
	for i, sam1 := range samples {
		if sam1.site == site {
			sam = sam1
			si = i
			break
		}
	}
	if si == -1 {
		panic("can't find the sample")
	}
	var deps []int
	for i := range data {
		data[i] ^= 0xff
		sonar := s.testInputSonar(data, 0)
		data[i] ^= 0xff
		samples1 := s.parseSonarData(sonar)
		var sam1 SonarSample
		si1 := -1
		for i, sam2 := range samples1 {
			if sam2.site == site {
				si1 = i
				sam1 = sam2
				break
			}
		}
		if si1 == -1 {
			continue
		}
		if bytes.Equal(sam.val[0], sam1.val[0]) && bytes.Equal(sam.val[1], sam1.val[1]) {
			continue
		}
		matches := 0
		for i := 0; i < si && i < si1; i++ {
			if samples[i].site == samples1[i].site {
				matches++
			}
		}
		if matches+1 < si {
			for i := si - 1; i > 0 && i+si1-si > 0; i-- {
				if samples[i].site == samples1[i+si1-si].site {
					matches++
				}
			}
		}
		log.Printf("BRUTEFORCE: byte %v matches %v si=%v/%v", i, matches, si, si1)
		deps = append(deps, i)
	}
	log.Printf("BRUTEFORCE: %v deps=%+v", sam.site.loc, deps)
}

func (sam *SonarSample) evaluate() bool {
	v1 := sam.val[0]
	v2 := sam.val[1]
	if sam.flags&SonarString != 0 {
		s1 := string(v1)
		s2 := string(v2)
		switch sam.flags & SonarOpMask {
		case SonarEQL:
			return s1 == s2
		case SonarNEQ:
			return s1 != s2
		case SonarLSS:
			return s1 < s2
		case SonarGTR:
			return s1 > s2
		case SonarLEQ:
			return s1 <= s2
		case SonarGEQ:
			return s1 >= s2
		default:
			panic("bad")
		}
	}
	if len(v1) == 0 || len(v2) == 0 || len(v1) > 8 || len(v2) > 8 || len(v1) != len(v2) {
		return false
	}
	v1 = append([]byte{}, v1...)
	for len(v1) < 8 {
		if int8(v1[len(v1)-1]) >= 0 {
			v1 = append(v1, 0)
		} else {
			v1 = append(v1, 0xff)
		}
	}
	v2 = append([]byte{}, v2...)
	for len(v2) < 8 {
		if int8(v2[len(v2)-1]) >= 0 {
			v2 = append(v2, 0)
		} else {
			v2 = append(v2, 0xff)
		}
	}
	// Note: assuming le machine.
	if sam.flags&SonarSigned == 0 {
		s1 := binary.LittleEndian.Uint64(v1)
		s2 := binary.LittleEndian.Uint64(v2)
		switch sam.flags & SonarOpMask {
		case SonarEQL:
			return s1 == s2
		case SonarNEQ:
			return s1 != s2
		case SonarLSS:
			return s1 < s2
		case SonarGTR:
			return s1 > s2
		case SonarLEQ:
			return s1 <= s2
		case SonarGEQ:
			return s1 >= s2
		default:
			panic("bad")
		}
	} else {
		s1 := int64(binary.LittleEndian.Uint64(v1))
		s2 := int64(binary.LittleEndian.Uint64(v2))
		switch sam.flags & SonarOpMask {
		case SonarEQL:
			return s1 == s2
		case SonarNEQ:
			return s1 != s2
		case SonarLSS:
			return s1 < s2
		case SonarGTR:
			return s1 > s2
		case SonarLEQ:
			return s1 <= s2
		case SonarGEQ:
			return s1 >= s2
		default:
			panic("bad")
		}
	}
}

func dumpSonarData(site *SonarSite, flags byte, v1, v2 []byte) {
	// Debug output.
	op := ""
	switch flags & SonarOpMask {
	case SonarEQL:
		op = "=="
	case SonarNEQ:
		op = "!="
	case SonarLSS:
		op = "<"
	case SonarGTR:
		op = ">"
	case SonarLEQ:
		op = "<="
	case SonarGEQ:
		op = ">="
	default:
		log.Fatalf("bad")
	}
	sign := ""
	if flags&SonarSigned != 0 {
		sign = "(signed)"
	}
	isstr := ""
	if flags&SonarString != 0 {
		isstr = "(string)"
	}
	const1 := ""
	if flags&SonarConst1 != 0 {
		const1 = "c"
	}
	const2 := ""
	if flags&SonarConst2 != 0 {
		const2 = "c"
	}
	log.Printf("SONAR %v%v %v %v%v %v%v %v",
		hex.EncodeToString(v1), const1, op, hex.EncodeToString(v2), const2, sign, isstr, site.loc)
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package iptables

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"syscall"
)

// Adds the output of stderr to exec.ExitError
type Error struct {
	exec.ExitError
	cmd exec.Cmd
	msg string
}

func (e *Error) ExitStatus() int {
	return e.Sys().(syscall.WaitStatus).ExitStatus()
}

func (e *Error) Error() string {
	return fmt.Sprintf("running %v: exit status %v: %v", e.cmd.Args, e.ExitStatus(), e.msg)
}

// Protocol to differentiate between IPv4 and IPv6
type Protocol byte

const (
	ProtocolIPv4 Protocol = iota
	ProtocolIPv6
)

type IPTables struct {
	path     string
	proto    Protocol
	hasCheck bool
	hasWait  bool
}

// New creates a new IPTables.
// For backwards compatibility, this always uses IPv4, i.e. "iptables".
func New() (*IPTables, error) {
	return NewWithProtocol(ProtocolIPv4)
}

// New creates a new IPTables for the given proto.
// The proto will determine which command is used, either "iptables" or "ip6tables".
func NewWithProtocol(proto Protocol) (*IPTables, error) {
	path, err := exec.LookPath(getIptablesCommand(proto))
	if err != nil {
		return nil, err
	}
	checkPresent, waitPresent, err := getIptablesCommandSupport(path)
	if err != nil {
		return nil, fmt.Errorf("error checking iptables version: %v", err)
	}
	ipt := IPTables{
		path:     path,
		proto:    proto,
		hasCheck: checkPresent,
		hasWait:  waitPresent,
	}
	return &ipt, nil
}

// Proto returns the protocol used by this IPTables.
func (ipt *IPTables) Proto() Protocol {
	return ipt.proto
}

// Exists checks if given rulespec in specified table/chain exists
func (ipt *IPTables) Exists(table, chain string, rulespec ...string) (bool, error) {
	if !ipt.hasCheck {
		return ipt.existsForOldIptables(table, chain, rulespec)

	}
	cmd := append([]string{"-t", table, "-C", chain}, rulespec...)
	err := ipt.run(cmd...)
	eerr, eok := err.(*Error)
	switch {
	case err == nil:
		return true, nil
	case eok && eerr.ExitStatus() == 1:
		return false, nil
	default:
		return false, err
	}
}

// Insert inserts rulespec to specified table/chain (in specified pos)
func (ipt *IPTables) Insert(table, chain string, pos int, rulespec ...string) error {
	cmd := append([]string{"-t", table, "-I", chain, strconv.Itoa(pos)}, rulespec...)
	return ipt.run(cmd...)
}

// Append appends rulespec to specified table/chain
func (ipt *IPTables) Append(table, chain string, rulespec ...string) error {
	cmd := append([]string{"-t", table, "-A", chain}, rulespec...)
	return ipt.run(cmd...)
}

// AppendUnique acts like Append except that it won't add a duplicate
func (ipt *IPTables) AppendUnique(table, chain string, rulespec ...string) error {
	exists, err := ipt.Exists(table, chain, rulespec...)
	if err != nil {
		return err
	}

	if !exists {
		return ipt.Append(table, chain, rulespec...)
	}

	return nil
}

// Delete removes rulespec in specified table/chain
func (ipt *IPTables) Delete(table, chain string, rulespec ...string) error {
	cmd := append([]string{"-t", table, "-D", chain}, rulespec...)
	return ipt.run(cmd...)
}

// List rules in specified table/chain
func (ipt *IPTables) List(table, chain string) ([]string, error) {
	args := []string{"-t", table, "-S", chain}
	return ipt.executeList(args)
}

// List rules (with counters) in specified table/chain
func (ipt *IPTables) ListWithCounters(table, chain string) ([]string, error) {
	args := []string{"-t", table, "-v", "-S", chain}
	return ipt.executeList(args)
}

// ListChains returns a slice containing the name of each chain in the specified table.
func (ipt *IPTables) ListChains(table string) ([]string, error) {
	args := []string{"-t", table, "-S"}

	result, err := ipt.executeList(args)
	if err != nil {
		return nil, err
	}

	// Iterate over rules to find all default (-P) and user-specified (-N) chains.
	// Chains definition always come before rules.
	// Format is the following:
	// -P OUTPUT ACCEPT
	// -N Custom
	var chains []string
	for _, val := range result {
		if strings.HasPrefix(val, "-P") || strings.HasPrefix(val, "-N") {
			chains = append(chains, strings.Fields(val)[1])
		} else {
			break
		}
	}
	return chains, nil
}

// Stats lists rules including the byte and packet counts
func (ipt *IPTables) Stats(table, chain string) ([][]string, error) {
	args := []string{"-t", table, "-L", chain, "-n", "-v", "-x"}
	lines, err := ipt.executeList(args)
	if err != nil {
		return nil, err
	}

	appendSubnet := func(addr string) string {
		if strings.IndexByte(addr, byte('/')) < 0 {
			if strings.IndexByte(addr, '.') < 0 {
				return addr + "/128"
			}
			return addr + "/32"
		}
		return addr
	}

	ipv6 := ipt.proto == ProtocolIPv6

	rows := [][]string{}
	for i, line := range lines {
		// Skip over chain name and field header
		if i < 2 {
			continue
		}

		// Fields:
		// 0=pkts 1=bytes 2=target 3=prot 4=opt 5=in 6=out 7=source 8=destination 9=options
		line = strings.TrimSpace(line)
		fields := strings.Fields(line)

		// The ip6tables verbose output cannot be naively split due to the default "opt"
		// field containing 2 single spaces.
		if ipv6 {
			// Check if field 6 is "opt" or "source" address
			dest := fields[6]
			ip, _, _ := net.ParseCIDR(dest)
			if ip == nil {
				ip = net.ParseIP(dest)
			}

			// If we detected a CIDR or IP, the "opt" field is empty.. insert it.
			if ip != nil {
				f := []string{}
				f = append(f, fields[:4]...)
				f = append(f, "  ") // Empty "opt" field for ip6tables
				f = append(f, fields[4:]...)
				fields = f
			}
		}

		// Adjust "source" and "destination" to include netmask, to match regular
		// List output
		fields[7] = appendSubnet(fields[7])
		fields[8] = appendSubnet(fields[8])

		// Combine "options" fields 9... into a single space-delimited field.
		options := fields[9:]
		fields = fields[:9]
		fields = append(fields, strings.Join(options, " "))
		rows = append(rows, fields)
	}
	return rows, nil
}

func (ipt *IPTables) executeList(args []string) ([]string, error) {
	var stdout bytes.Buffer
	if err := ipt.runWithOutput(args, &stdout); err != nil {
		return nil, err
	}

	rules := strings.Split(stdout.String(), "\n")
	if len(rules) > 0 && rules[len(rules)-1] == "" {
		rules = rules[:len(rules)-1]
	}

	return rules, nil
}

// NewChain creates a new chain in the specified table.
// If the chain already exists, it will result in an error.
func (ipt *IPTables) NewChain(table, chain string) error {
	return ipt.run("-t", table, "-N", chain)
}

// ClearChain flushed (deletes all rules) in the specified table/chain.
// If the chain does not exist, a new one will be created
func (ipt *IPTables) ClearChain(table, chain string) error {
	err := ipt.NewChain(table, chain)

	eerr, eok := err.(*Error)
	switch {
	case err == nil:
		return nil
	case eok && eerr.ExitStatus() == 1:
		// chain already exists. Flush (clear) it.
		return ipt.run("-t", table, "-F", chain)
	default:
		return err
	}
}

// RenameChain renames the old chain to the new one.
func (ipt *IPTables) RenameChain(table, oldChain, newChain string) error {
	return ipt.run("-t", table, "-E", oldChain, newChain)
}

// DeleteChain deletes the chain in the specified table.
// The chain must be empty
func (ipt *IPTables) DeleteChain(table, chain string) error {
	return ipt.run("-t", table, "-X", chain)
}

// run runs an iptables command with the given arguments, ignoring
// any stdout output
func (ipt *IPTables) run(args ...string) error {
	return ipt.runWithOutput(args, nil)
}

// runWithOutput runs an iptables command with the given arguments,
// writing any stdout output to the given writer
func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error {
	args = append([]string{ipt.path}, args...)
	if ipt.hasWait {
		args = append(args, "--wait")
	} else {
		fmu, err := newXtablesFileLock()
		if err != nil {
			return err
		}
		ul, err := fmu.tryLock()
		if err != nil {
			return err
		}
		defer ul.Unlock()
	}

	var stderr bytes.Buffer
	cmd := exec.Cmd{
		Path:   ipt.path,
		Args:   args,
		Stdout: stdout,
		Stderr: &stderr,
	}

	if err := cmd.Run(); err != nil {
		return &Error{*(err.(*exec.ExitError)), cmd, stderr.String()}
	}

	return nil
}

// getIptablesCommand returns the correct command for the given protocol, either "iptables" or "ip6tables".
func getIptablesCommand(proto Protocol) string {
	if proto == ProtocolIPv6 {
		return "ip6tables"
	} else {
		return "iptables"
	}
}

// Checks if iptables has the "-C" and "--wait" flag
func getIptablesCommandSupport(path string) (bool, bool, error) {
	vstring, err := getIptablesVersionString(path)
	if err != nil {
		return false, false, err
	}

	v1, v2, v3, err := extractIptablesVersion(vstring)
	if err != nil {
		return false, false, err
	}

	return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), nil
}

// getIptablesVersion returns the first three components of the iptables version.
// e.g. "iptables v1.3.66" would return (1, 3, 66, nil)
func extractIptablesVersion(str string) (int, int, int, error) {
	versionMatcher := regexp.MustCompile("v([0-9]+)\\.([0-9]+)\\.([0-9]+)")
	result := versionMatcher.FindStringSubmatch(str)
	if result == nil {
		return 0, 0, 0, fmt.Errorf("no iptables version found in string: %s", str)
	}

	v1, err := strconv.Atoi(result[1])
	if err != nil {
		return 0, 0, 0, err
	}

	v2, err := strconv.Atoi(result[2])
	if err != nil {
		return 0, 0, 0, err
	}

	v3, err := strconv.Atoi(result[3])
	if err != nil {
		return 0, 0, 0, err
	}

	return v1, v2, v3, nil
}

// Runs "iptables --version" to get the version string
func getIptablesVersionString(path string) (string, error) {
	cmd := exec.Command(path, "--version")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return "", err
	}
	return out.String(), nil
}

// Checks if an iptables version is after 1.4.11, when --check was added
func iptablesHasCheckCommand(v1 int, v2 int, v3 int) bool {
	if v1 > 1 {
		return true
	}
	if v1 == 1 && v2 > 4 {
		return true
	}
	if v1 == 1 && v2 == 4 && v3 >= 11 {
		return true
	}
	return false
}

// Checks if an iptables version is after 1.4.20, when --wait was added
func iptablesHasWaitCommand(v1 int, v2 int, v3 int) bool {
	if v1 > 1 {
		return true
	}
	if v1 == 1 && v2 > 4 {
		return true
	}
	if v1 == 1 && v2 == 4 && v3 >= 20 {
		return true
	}
	return false
}

// Checks if a rule specification exists for a table
func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string) (bool, error) {
	rs := strings.Join(append([]string{"-A", chain}, rulespec...), " ")
	args := []string{"-t", table, "-S"}
	var stdout bytes.Buffer
	err := ipt.runWithOutput(args, &stdout)
	if err != nil {
		return false, err
	}
	return strings.Contains(stdout.String(), rs), nil
}
<commit_msg>Fixes: https://github.com/coreos/go-iptables/issues/35<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package iptables

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"syscall"
)

// Adds the output of stderr to exec.ExitError
type Error struct {
	exec.ExitError
	cmd exec.Cmd
	msg string
}

func (e *Error) ExitStatus() int {
	return e.Sys().(syscall.WaitStatus).ExitStatus()
}

func (e *Error) Error() string {
	return fmt.Sprintf("running %v: exit status %v: %v", e.cmd.Args, e.ExitStatus(), e.msg)
}

// Protocol to differentiate between IPv4 and IPv6
type Protocol byte

const (
	ProtocolIPv4 Protocol = iota
	ProtocolIPv6
)

type IPTables struct {
	path     string
	proto    Protocol
	hasCheck bool
	hasWait  bool
}

// New creates a new IPTables.
// For backwards compatibility, this always uses IPv4, i.e. "iptables".
func New() (*IPTables, error) {
	return NewWithProtocol(ProtocolIPv4)
}

// New creates a new IPTables for the given proto.
// The proto will determine which command is used, either "iptables" or "ip6tables".
func NewWithProtocol(proto Protocol) (*IPTables, error) {
	path, err := exec.LookPath(getIptablesCommand(proto))
	if err != nil {
		return nil, err
	}
	checkPresent, waitPresent, err := getIptablesCommandSupport(path)
	if err != nil {
		return nil, fmt.Errorf("error checking iptables version: %v", err)
	}
	ipt := IPTables{
		path:     path,
		proto:    proto,
		hasCheck: checkPresent,
		hasWait:  waitPresent,
	}
	return &ipt, nil
}

// Proto returns the protocol used by this IPTables.
func (ipt *IPTables) Proto() Protocol {
	return ipt.proto
}

// Exists checks if given rulespec in specified table/chain exists
func (ipt *IPTables) Exists(table, chain string, rulespec ...string) (bool, error) {
	if !ipt.hasCheck {
		return ipt.existsForOldIptables(table, chain, rulespec)

	}
	cmd := append([]string{"-t", table, "-C", chain}, rulespec...)
	err := ipt.run(cmd...)
	eerr, eok := err.(*Error)
	switch {
	case err == nil:
		return true, nil
	case eok && eerr.ExitStatus() == 1:
		return false, nil
	default:
		return false, err
	}
}

// Insert inserts rulespec to specified table/chain (in specified pos)
func (ipt *IPTables) Insert(table, chain string, pos int, rulespec ...string) error {
	cmd := append([]string{"-t", table, "-I", chain, strconv.Itoa(pos)}, rulespec...)
	return ipt.run(cmd...)
}

// Append appends rulespec to specified table/chain
func (ipt *IPTables) Append(table, chain string, rulespec ...string) error {
	cmd := append([]string{"-t", table, "-A", chain}, rulespec...)
	return ipt.run(cmd...)
}

// AppendUnique acts like Append except that it won't add a duplicate
func (ipt *IPTables) AppendUnique(table, chain string, rulespec ...string) error {
	exists, err := ipt.Exists(table, chain, rulespec...)
	if err != nil {
		return err
	}

	if !exists {
		return ipt.Append(table, chain, rulespec...)
	}

	return nil
}

// Delete removes rulespec in specified table/chain
func (ipt *IPTables) Delete(table, chain string, rulespec ...string) error {
	cmd := append([]string{"-t", table, "-D", chain}, rulespec...)
	return ipt.run(cmd...)
}

// List rules in specified table/chain
func (ipt *IPTables) List(table, chain string) ([]string, error) {
	args := []string{"-t", table, "-S", chain}
	return ipt.executeList(args)
}

// List rules (with counters) in specified table/chain
func (ipt *IPTables) ListWithCounters(table, chain string) ([]string, error) {
	args := []string{"-t", table, "-v", "-S", chain}
	return ipt.executeList(args)
}

// ListChains returns a slice containing the name of each chain in the specified table.
func (ipt *IPTables) ListChains(table string) ([]string, error) {
	args := []string{"-t", table, "-S"}

	result, err := ipt.executeList(args)
	if err != nil {
		return nil, err
	}

	// Iterate over rules to find all default (-P) and user-specified (-N) chains.
	// Chains definition always come before rules.
	// Format is the following:
	// -P OUTPUT ACCEPT
	// -N Custom
	var chains []string
	for _, val := range result {
		if strings.HasPrefix(val, "-P") || strings.HasPrefix(val, "-N") {
			chains = append(chains, strings.Fields(val)[1])
		} else {
			break
		}
	}
	return chains, nil
}

// Stats lists rules including the byte and packet counts
func (ipt *IPTables) Stats(table, chain string) ([][]string, error) {
	args := []string{"-t", table, "-L", chain, "-n", "-v", "-x"}
	lines, err := ipt.executeList(args)
	if err != nil {
		return nil, err
	}

	appendSubnet := func(addr string) string {
		if strings.IndexByte(addr, byte('/')) < 0 {
			if strings.IndexByte(addr, '.') < 0 {
				return addr + "/128"
			}
			return addr + "/32"
		}
		return addr
	}

	ipv6 := ipt.proto == ProtocolIPv6

	rows := [][]string{}
	for i, line := range lines {
		// Skip over chain name and field header
		if i < 2 {
			continue
		}

		// Fields:
		// 0=pkts 1=bytes 2=target 3=prot 4=opt 5=in 6=out 7=source 8=destination 9=options
		line = strings.TrimSpace(line)
		fields := strings.Fields(line)

		// The ip6tables verbose output cannot be naively split due to the default "opt"
		// field containing 2 single spaces.
		if ipv6 {
			// Check if field 6 is "opt" or "source" address
			dest := fields[6]
			ip, _, _ := net.ParseCIDR(dest)
			if ip == nil {
				ip = net.ParseIP(dest)
			}

			// If we detected a CIDR or IP, the "opt" field is empty.. insert it.
			if ip != nil {
				f := []string{}
				f = append(f, fields[:4]...)
				f = append(f, "  ") // Empty "opt" field for ip6tables
				f = append(f, fields[4:]...)
				fields = f
			}
		}

		// Adjust "source" and "destination" to include netmask, to match regular
		// List output
		fields[7] = appendSubnet(fields[7])
		fields[8] = appendSubnet(fields[8])

		// Combine "options" fields 9... into a single space-delimited field.
		options := fields[9:]
		fields = fields[:9]
		fields = append(fields, strings.Join(options, " "))
		rows = append(rows, fields)
	}
	return rows, nil
}

func (ipt *IPTables) executeList(args []string) ([]string, error) {
	var stdout bytes.Buffer
	if err := ipt.runWithOutput(args, &stdout); err != nil {
		return nil, err
	}

	rules := strings.Split(stdout.String(), "\n")
	if len(rules) > 0 && rules[len(rules)-1] == "" {
		rules = rules[:len(rules)-1]
	}

	return rules, nil
}

// NewChain creates a new chain in the specified table.
// If the chain already exists, it will result in an error.
func (ipt *IPTables) NewChain(table, chain string) error {
	return ipt.run("-t", table, "-N", chain)
}

// ClearChain flushed (deletes all rules) in the specified table/chain.
// If the chain does not exist, a new one will be created
func (ipt *IPTables) ClearChain(table, chain string) error {
	err := ipt.NewChain(table, chain)

	eerr, eok := err.(*Error)
	switch {
	case err == nil:
		return nil
	case eok && eerr.ExitStatus() == 1:
		// chain already exists. Flush (clear) it.
		return ipt.run("-t", table, "-F", chain)
	default:
		return err
	}
}

// RenameChain renames the old chain to the new one.
func (ipt *IPTables) RenameChain(table, oldChain, newChain string) error {
	return ipt.run("-t", table, "-E", oldChain, newChain)
}

// DeleteChain deletes the chain in the specified table.
// The chain must be empty
func (ipt *IPTables) DeleteChain(table, chain string) error {
	return ipt.run("-t", table, "-X", chain)
}

// run runs an iptables command with the given arguments, ignoring
// any stdout output
func (ipt *IPTables) run(args ...string) error {
	return ipt.runWithOutput(args, nil)
}

// runWithOutput runs an iptables command with the given arguments,
// writing any stdout output to the given writer
func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error {
	args = append([]string{ipt.path}, args...)
	if ipt.hasWait {
		args = append(args, "--wait")
	} else {
		fmu, err := newXtablesFileLock()
		if err != nil {
			return err
		}
		ul, err := fmu.tryLock()
		if err != nil {
			return err
		}
		defer ul.Unlock()
	}

	var stderr bytes.Buffer
	cmd := exec.Cmd{
		Path:   ipt.path,
		Args:   args,
		Stdout: stdout,
		Stderr: &stderr,
	}

	if err := cmd.Run(); err != nil {
		switch e := err.(type) {
		case *exec.ExitError:
			return &Error{*e, stderr.String()}
		default:
			return err
		}
	}

	return nil
}

// getIptablesCommand returns the correct command for the given protocol, either "iptables" or "ip6tables".
func getIptablesCommand(proto Protocol) string {
	if proto == ProtocolIPv6 {
		return "ip6tables"
	} else {
		return "iptables"
	}
}

// Checks if iptables has the "-C" and "--wait" flag
func getIptablesCommandSupport(path string) (bool, bool, error) {
	vstring, err := getIptablesVersionString(path)
	if err != nil {
		return false, false, err
	}

	v1, v2, v3, err := extractIptablesVersion(vstring)
	if err != nil {
		return false, false, err
	}

	return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), nil
}

// getIptablesVersion returns the first three components of the iptables version.
// e.g. "iptables v1.3.66" would return (1, 3, 66, nil)
func extractIptablesVersion(str string) (int, int, int, error) {
	versionMatcher := regexp.MustCompile("v([0-9]+)\\.([0-9]+)\\.([0-9]+)")
	result := versionMatcher.FindStringSubmatch(str)
	if result == nil {
		return 0, 0, 0, fmt.Errorf("no iptables version found in string: %s", str)
	}

	v1, err := strconv.Atoi(result[1])
	if err != nil {
		return 0, 0, 0, err
	}

	v2, err := strconv.Atoi(result[2])
	if err != nil {
		return 0, 0, 0, err
	}

	v3, err := strconv.Atoi(result[3])
	if err != nil {
		return 0, 0, 0, err
	}

	return v1, v2, v3, nil
}

// Runs "iptables --version" to get the version string
func getIptablesVersionString(path string) (string, error) {
	cmd := exec.Command(path, "--version")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return "", err
	}
	return out.String(), nil
}

// Checks if an iptables version is after 1.4.11, when --check was added
func iptablesHasCheckCommand(v1 int, v2 int, v3 int) bool {
	if v1 > 1 {
		return true
	}
	if v1 == 1 && v2 > 4 {
		return true
	}
	if v1 == 1 && v2 == 4 && v3 >= 11 {
		return true
	}
	return false
}

// Checks if an iptables version is after 1.4.20, when --wait was added
func iptablesHasWaitCommand(v1 int, v2 int, v3 int) bool {
	if v1 > 1 {
		return true
	}
	if v1 == 1 && v2 > 4 {
		return true
	}
	if v1 == 1 && v2 == 4 && v3 >= 20 {
		return true
	}
	return false
}

// Checks if a rule specification exists for a table
func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string) (bool, error) {
	rs := strings.Join(append([]string{"-A", chain}, rulespec...), " ")
	args := []string{"-t", table, "-S"}
	var stdout bytes.Buffer
	err := ipt.runWithOutput(args, &stdout)
	if err != nil {
		return false, err
	}
	return strings.Contains(stdout.String(), rs), nil
}
<|endoftext|>


<commit_before>package ircmatch

import "github.com/DanielOaks/e-nfa"

// Matcher represents an object that can match IRC strings.
type Matcher struct {
	internalENFA *enfa.ENFA
}

// MakeMatch creates a Matcher.
func MakeMatch(globTemplate string) Matcher {
	var newmatch Matcher

	// assemble internal enfa
	newmatch.internalENFA = enfa.NewENFA(0, false)

	var currentState int
	var lastWasStar bool
	for _, char := range globTemplate {
		if char == '*' {
			if lastWasStar {
				continue
			}
			newmatch.internalENFA.AddTransition(currentState, "*", currentState)
			lastWasStar = true
			continue
		} else if char == '?' {
			newmatch.internalENFA.AddState(currentState+1, false)
			newmatch.internalENFA.AddTransition(currentState, "?", currentState+1)
			currentState++
		} else {
			newmatch.internalENFA.AddState(currentState+1, false)
			newmatch.internalENFA.AddTransition(currentState, string(char), currentState+1)
			currentState++
		}

		lastWasStar = false
	}

	// create end state
	newmatch.internalENFA.AddState(currentState+1, true)
	newmatch.internalENFA.AddTransition(currentState, "", currentState+1)

	newmatch.internalENFA.PrintTransitionTable()

	return newmatch
}

// Match returns true if the given string matches this glob.
func (menfa *Matcher) Match(search string) bool {
	var searchChars []string
	for _, char := range search {
		searchChars = append(searchChars, string(char))
	}

	isMatch := menfa.internalENFA.VerifyInputs(searchChars)
	menfa.internalENFA.Reset()
	return isMatch
}
<commit_msg>ircmatch: Don't print debug data<commit_after>package ircmatch

import "github.com/DanielOaks/e-nfa"

// Matcher represents an object that can match IRC strings.
type Matcher struct {
	internalENFA *enfa.ENFA
}

// MakeMatch creates a Matcher.
func MakeMatch(globTemplate string) Matcher {
	var newmatch Matcher

	// assemble internal enfa
	newmatch.internalENFA = enfa.NewENFA(0, false)

	var currentState int
	var lastWasStar bool
	for _, char := range globTemplate {
		if char == '*' {
			if lastWasStar {
				continue
			}
			newmatch.internalENFA.AddTransition(currentState, "*", currentState)
			lastWasStar = true
			continue
		} else if char == '?' {
			newmatch.internalENFA.AddState(currentState+1, false)
			newmatch.internalENFA.AddTransition(currentState, "?", currentState+1)
			currentState++
		} else {
			newmatch.internalENFA.AddState(currentState+1, false)
			newmatch.internalENFA.AddTransition(currentState, string(char), currentState+1)
			currentState++
		}

		lastWasStar = false
	}

	// create end state
	newmatch.internalENFA.AddState(currentState+1, true)
	newmatch.internalENFA.AddTransition(currentState, "", currentState+1)

	return newmatch
}

// Match returns true if the given string matches this glob.
func (menfa *Matcher) Match(search string) bool {
	var searchChars []string
	for _, char := range search {
		searchChars = append(searchChars, string(char))
	}

	isMatch := menfa.internalENFA.VerifyInputs(searchChars)
	menfa.internalENFA.Reset()
	return isMatch
}
<|endoftext|>


<commit_before>package main

import (
	"strings"

	"github.com/animenotifier/arn"
)

func parseCharacterDescription(input string) (output string, attributes []*arn.CharacterAttribute) {
	// Parse attributes like these:
	// - Position: Club Manager
	// - Height: 162 cm (5' 4")
	// - Weight: 48 kg (106 lb)
	// - Birthday: November 24
	// - Hair color: Brown
	// - Eyes: Blue (anime), Green (manga)

	paragraphs := strings.Split(input, "\n\n")
	finalParagraphs := make([]string, 0, len(paragraphs))

	for _, paragraph := range paragraphs {
		// Is paragraph full of attributes?
		if strings.Contains(paragraph, "\n") {
			lines := strings.Split(paragraph, "\n")
			var lastAttribute *arn.CharacterAttribute

			for _, line := range lines {
				line = strings.Replace(line, " (\n)", "", -1)

				// Remove all kinds of starting and ending parantheses.
				if strings.HasPrefix(line, "(") {
					line = strings.TrimPrefix(line, "(")
					line = strings.TrimSuffix(line, ")")
				}

				line = strings.TrimSuffix(line, " (")
				line = strings.TrimPrefix(line, ")")

				parts := strings.Split(line, ":")

				if len(parts) < 2 {
					// Add to previous attribute
					if lastAttribute != nil {
						lastAttribute.Value += ", " + line
					}

					continue
				}

				name := strings.TrimSpace(parts[0])
				value := strings.TrimSpace(parts[1])

				if value == "" || value == `"` {
					continue
				}

				lastAttribute = &arn.CharacterAttribute{
					Name:  name,
					Value: value,
				}

				attributes = append(attributes, lastAttribute)
			}

			continue
		}

		// Remove all kinds of starting and ending parantheses.
		if strings.HasPrefix(paragraph, "(") {
			paragraph = strings.TrimPrefix(paragraph, "(")
			paragraph = strings.TrimSuffix(paragraph, ")")
		}

		// Replace source paragraph with an attribute
		if strings.HasPrefix(paragraph, "Source:") || strings.HasPrefix(paragraph, "source:") {
			source := paragraph[len("source:"):]
			source = strings.TrimSpace(source)

			attributes = append(attributes, &arn.CharacterAttribute{
				Name:  "Source",
				Value: source,
			})
			continue
		}

		finalParagraphs = append(finalParagraphs, paragraph)

		// originalLine := line

		// line = strings.TrimSpace(line)

		// colonPos := strings.Index(line, ":")

		// // If a colon has not been found or the colon is too far,
		// // treat it as a normal line.
		// if colonPos == -1 || colonPos < 2 || colonPos > 25 {
		// 	finalLines = append(finalLines, originalLine)
		// 	continue
		// }

		// key := line[:colonPos]
		// value := line[colonPos+1:]

		// value = strings.TrimSpace(value)

		// if key == "source" {
		// 	key = "Source"
		// }

		// attributes = append(attributes, &arn.CharacterAttribute{
		// 	Name:  key,
		// 	Value: value,
		// })

		// fmt.Println(color.CyanString(key), color.YellowString(value))
	}

	output = strings.Join(finalParagraphs, "\n\n")
	output = strings.TrimSpace(output)

	return output, attributes
}
<commit_msg>Updated MAL sync<commit_after>package main

import (
	"strings"

	"github.com/animenotifier/arn"
)

func parseCharacterDescription(input string) (output string, attributes []*arn.CharacterAttribute) {
	// Parse attributes like these:
	// - Position: Club Manager
	// - Height: 162 cm (5' 4")
	// - Weight: 48 kg (106 lb)
	// - Birthday: November 24
	// - Hair color: Brown
	// - Eyes: Blue (anime), Green (manga)

	paragraphs := strings.Split(input, "\n\n")
	finalParagraphs := make([]string, 0, len(paragraphs))

	for _, paragraph := range paragraphs {
		// Is paragraph full of attributes?
		if strings.Contains(paragraph, "\n") {
			lines := strings.Split(paragraph, "\n")
			var lastAttribute *arn.CharacterAttribute

			for _, line := range lines {
				// line = strings.Replace(line, " (\n)", "", -1)

				// Remove all kinds of starting and ending parantheses.
				if strings.HasPrefix(line, "(") {
					line = strings.TrimPrefix(line, "(")
					line = strings.TrimSuffix(line, ")")
				}

				if !strings.Contains(line, ":") {
					// Remove list indicators
					line = strings.TrimPrefix(line, "- ")
					line = strings.TrimPrefix(line, "* ")

					// Add to previous attribute
					if lastAttribute != nil {
						if lastAttribute.Value != "" {
							lastAttribute.Value += ", "
						}

						lastAttribute.Value += line
					}

					continue
				}

				parts := strings.Split(line, ":")
				name := strings.TrimSpace(parts[0])
				value := strings.TrimSpace(parts[1])

				lastAttribute = &arn.CharacterAttribute{
					Name:  name,
					Value: value,
				}

				attributes = append(attributes, lastAttribute)
			}

			continue
		}

		// Remove all kinds of starting and ending parantheses.
		if strings.HasPrefix(paragraph, "(") {
			paragraph = strings.TrimPrefix(paragraph, "(")
			paragraph = strings.TrimSuffix(paragraph, ")")
		}

		// Replace source paragraph with an attribute
		if strings.HasPrefix(paragraph, "Source:") || strings.HasPrefix(paragraph, "source:") {
			source := paragraph[len("source:"):]
			source = strings.TrimSpace(source)

			attributes = append(attributes, &arn.CharacterAttribute{
				Name:  "Source",
				Value: source,
			})
			continue
		}

		finalParagraphs = append(finalParagraphs, paragraph)

		// originalLine := line

		// line = strings.TrimSpace(line)

		// colonPos := strings.Index(line, ":")

		// // If a colon has not been found or the colon is too far,
		// // treat it as a normal line.
		// if colonPos == -1 || colonPos < 2 || colonPos > 25 {
		// 	finalLines = append(finalLines, originalLine)
		// 	continue
		// }

		// key := line[:colonPos]
		// value := line[colonPos+1:]

		// value = strings.TrimSpace(value)

		// if key == "source" {
		// 	key = "Source"
		// }

		// attributes = append(attributes, &arn.CharacterAttribute{
		// 	Name:  key,
		// 	Value: value,
		// })

		// fmt.Println(color.CyanString(key), color.YellowString(value))
	}

	output = strings.Join(finalParagraphs, "\n\n")
	output = strings.TrimSpace(output)

	return output, attributes
}
<|endoftext|>


<commit_before>package filter

import (
	"math"
	"time"

	"github.com/kwoodhouse93/audio-playground/source"
	"github.com/kwoodhouse93/audio-playground/types"
	"github.com/kwoodhouse93/audio-playground/utils"
)

// Zero sets a signal to 0
func Zero(source source.Source) source.Source {
	return func(step int) []float32 {
		out := source(step)
		out[0] = 0
		out[1] = 0
		return out
	}
}

// Delay stores the samples for a given duration then plays them back delayed
func Delay(src source.Source, delay time.Duration, sampleRate float64) source.Source {
	delaySamples := utils.TimeToSteps(delay, sampleRate)
	delayBuf := make([]*types.DelayLine, 2)
	delayBuf[0] = types.NewDelayLine(delaySamples)
	delayBuf[1] = types.NewDelayLine(delaySamples)
	return source.Cached(func(step int) []float32 {
		out := utils.MakeSample(2)
		s := src(step)
		out[0] = delayBuf[0].Step(s[0])
		out[1] = delayBuf[1].Step(s[1])
		return out
	})
}

// DelayFB is a delay with a feedback setting.
func DelayFB(src source.Source, delay time.Duration, feedback float32, sampleRate float64) source.Source {
	delaySamples := utils.TimeToSteps(delay, sampleRate)
	delayBuf := make([]*types.DelayLine, 2)
	delayBuf[0] = types.NewDelayLine(delaySamples)
	delayBuf[1] = types.NewDelayLine(delaySamples)
	return source.Cached(func(step int) []float32 {
		out := utils.MakeSample(2)
		s := src(step)
		out[0] = delayBuf[0].Read()
		out[1] = delayBuf[1].Read()
		delayBuf[0].Write(s[0] + (out[0] * feedback))
		delayBuf[1].Write(s[1] + (out[1] * feedback))
		return out
	})
}

// LowPass is a basic low pass filter
// H(s) = 1 / (s^2 + s/Q + 1)
func LowPass(src source.Source, cornerFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(cornerFreq, Q, sampleRate)

	b0 := (1 - cosw0) / 2
	b1 := 1 - cosw0
	b2 := (1 - cosw0) / 2
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// HighPass is a basic high pass filter
// H(s) = s^2 / (s^2 + s/Q + 1)
func HighPass(src source.Source, cornerFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(cornerFreq, Q, sampleRate)

	b0 := (1 + cosw0) / 2
	b1 := -(1 + cosw0)
	b2 := (1 + cosw0) / 2
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// BandPassPeakQ is a basic bandpass filter with constant skirt gain, and peak gain = Q
// H(s) = s / (s^2 + s/Q + 1)
func BandPassPeakQ(src source.Source, shelfFreq, Q, sampleRate float64) source.Source {
	cosw0, sinw0, alpha := calcCoefPrereqs(shelfFreq, Q, sampleRate)

	b0 := sinw0 / 2
	b1 := float32(0.0)
	b2 := -sinw0 / 2
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// BandPassPeak0 is a basic bandpass filter, iwth a constant 0 dB peak gain
// H(s) = (s/Q) / (s^2 + s/Q + 1)
func BandPassPeak0(src source.Source, shelfFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(shelfFreq, Q, sampleRate)

	b0 := alpha
	b1 := float32(0.0)
	b2 := -alpha
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// Notch is a basic notch filter
// H(s) = (s^2 + 1) / (s^2 + s/Q + 1)
func Notch(src source.Source, notchFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(notchFreq, Q, sampleRate)

	b0 := float32(1.0)
	b1 := -2 * cosw0
	b2 := float32(1.0)
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// AllPass is a basic all pass filter
// H(s) = (s^2 - s/Q + 1) / (s^2 + s/Q + 1)
// quadFreq is the frequency at which the input and output go into quadrature
func AllPass(src source.Source, quadFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(quadFreq, Q, sampleRate)

	b0 := 1 - alpha
	b1 := -2 * cosw0
	b2 := 1 + alpha
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

func calcCoefPrereqs(cornerFreq, Q, sampleRate float64) (cosw0, sinw0, alpha float32) {
	w0 := 2 * math.Pi * (cornerFreq / sampleRate)
	cosw0 = float32(math.Cos(w0))
	sinw0 = float32(math.Sin(w0))
	alpha = sinw0 / (2 * float32(Q))
	return
}

func applyBiquadTransfer(source source.Source, b0, b1, b2, a0, a1, a2 float32) source.Source {
	x := utils.MakeBuffer(2, 3)
	y := utils.MakeBuffer(2, 3)
	return func(step int) []float32 {
		out := utils.MakeSample(2)
		input := source(step)
		for i := range y {
			y[i][2] = y[i][1]
			y[i][1] = y[i][0]
			x[i][2] = x[i][1]
			x[i][1] = x[i][0]
			x[i][0] = input[i]
		}
		out[0] = biquadTransfer(x[0], y[0], a0, a1, a2, b0, b1, b2)
		out[1] = biquadTransfer(x[1], y[1], a0, a1, a2, b0, b1, b2)
		return out
	}
}

func biquadTransfer(x, y []float32, a0, a1, a2, b0, b1, b2 float32) float32 {
	y[0] = (b0/a0)*x[0] + (b1/a0)*x[1] + (b2/a0)*x[2] - (a1/a0)*y[1] - (a2/a0)*y[2]
	return y[0]
}
<commit_msg>Add simple comb filters These are currently untested<commit_after>package filter

import (
	"math"
	"time"

	"github.com/kwoodhouse93/audio-playground/source"
	"github.com/kwoodhouse93/audio-playground/types"
	"github.com/kwoodhouse93/audio-playground/utils"
)

// Delay stores the samples for a given duration then plays them back delayed
func Delay(src source.Source, delay time.Duration, sampleRate float64) source.Source {
	delaySamples := utils.TimeToSteps(delay, sampleRate)
	delayBuf := make([]*types.DelayLine, 2)
	delayBuf[0] = types.NewDelayLine(delaySamples)
	delayBuf[1] = types.NewDelayLine(delaySamples)
	return source.Cached(func(step int) []float32 {
		out := utils.MakeSample(2)
		s := src(step)
		out[0] = delayBuf[0].Step(s[0])
		out[1] = delayBuf[1].Step(s[1])
		return out
	})
}

// DelayFB is a delay with a feedback setting.
func DelayFB(src source.Source, delay time.Duration, feedback float32, sampleRate float64) source.Source {
	delaySamples := utils.TimeToSteps(delay, sampleRate)
	delayBuf := make([]*types.DelayLine, 2)
	delayBuf[0] = types.NewDelayLine(delaySamples)
	delayBuf[1] = types.NewDelayLine(delaySamples)
	return source.Cached(func(step int) []float32 {
		out := utils.MakeSample(2)
		s := src(step)
		out[0] = delayBuf[0].Read()
		out[1] = delayBuf[1].Read()
		delayBuf[0].Write(s[0] + (out[0] * feedback))
		delayBuf[1].Write(s[1] + (out[1] * feedback))
		return out
	})
}

// LowPass is a basic low pass filter
// H(s) = 1 / (s^2 + s/Q + 1)
func LowPass(src source.Source, cornerFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(cornerFreq, Q, sampleRate)

	b0 := (1 - cosw0) / 2
	b1 := 1 - cosw0
	b2 := (1 - cosw0) / 2
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// HighPass is a basic high pass filter
// H(s) = s^2 / (s^2 + s/Q + 1)
func HighPass(src source.Source, cornerFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(cornerFreq, Q, sampleRate)

	b0 := (1 + cosw0) / 2
	b1 := -(1 + cosw0)
	b2 := (1 + cosw0) / 2
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// BandPassPeakQ is a basic bandpass filter with constant skirt gain, and peak gain = Q
// H(s) = s / (s^2 + s/Q + 1)
func BandPassPeakQ(src source.Source, shelfFreq, Q, sampleRate float64) source.Source {
	cosw0, sinw0, alpha := calcCoefPrereqs(shelfFreq, Q, sampleRate)

	b0 := sinw0 / 2
	b1 := float32(0.0)
	b2 := -sinw0 / 2
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// BandPassPeak0 is a basic bandpass filter, iwth a constant 0 dB peak gain
// H(s) = (s/Q) / (s^2 + s/Q + 1)
func BandPassPeak0(src source.Source, shelfFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(shelfFreq, Q, sampleRate)

	b0 := alpha
	b1 := float32(0.0)
	b2 := -alpha
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// Notch is a basic notch filter
// H(s) = (s^2 + 1) / (s^2 + s/Q + 1)
func Notch(src source.Source, notchFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(notchFreq, Q, sampleRate)

	b0 := float32(1.0)
	b1 := -2 * cosw0
	b2 := float32(1.0)
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

// AllPass is a basic all pass filter
// H(s) = (s^2 - s/Q + 1) / (s^2 + s/Q + 1)
// quadFreq is the frequency at which the input and output go into quadrature
func AllPass(src source.Source, quadFreq, Q, sampleRate float64) source.Source {
	cosw0, _, alpha := calcCoefPrereqs(quadFreq, Q, sampleRate)

	b0 := 1 - alpha
	b1 := -2 * cosw0
	b2 := 1 + alpha
	a0 := 1 + alpha
	a1 := -2 * cosw0
	a2 := 1 - alpha

	return source.Cached(applyBiquadTransfer(src, b0, b1, b2, a0, a1, a2))
}

func calcCoefPrereqs(f0, Q, sampleRate float64) (cosw0, sinw0, alpha float32) {
	w0 := 2 * math.Pi * (f0 / sampleRate)
	cosw0 = float32(math.Cos(w0))
	sinw0 = float32(math.Sin(w0))
	alpha = sinw0 / (2 * float32(Q))
	return
}

func applyBiquadTransfer(src source.Source, b0, b1, b2, a0, a1, a2 float32) source.Source {
	x := utils.MakeBuffer(2, 3)
	y := utils.MakeBuffer(2, 3)
	return func(step int) []float32 {
		out := utils.MakeSample(2)
		input := src(step)
		// TODO: Can this be achieved with delay lines, or another reusable type?
		// Current approach feels a bit inelegant.
		for i := range y {
			// We're saying:
			// Shift y along by 1 (increasing indexes i.e. 0->1->2, drop 2 and input to 0)
			// Shift x along by 1
			// Put an input into x[0]
			// biquadTransfer puts a value into y[i][0]
			y[i][2] = y[i][1]
			y[i][1] = y[i][0]
			x[i][2] = x[i][1]
			x[i][1] = x[i][0]
			x[i][0] = input[i]
			out[i] = biquadTransfer(x[i], y[i], a0, a1, a2, b0, b1, b2)
		}
		return out
	}
}

func biquadTransfer(x, y []float32, a0, a1, a2, b0, b1, b2 float32) float32 {
	y[0] = (b0/a0)*x[0] + (b1/a0)*x[1] + (b2/a0)*x[2] - (a1/a0)*y[1] - (a2/a0)*y[2]
	return y[0]
}

// FeedBackComb is a simple feedback comb filter, as defined at
// https://ccrma.stanford.edu/~jos/pasp/Feedback_Comb_Filters.html
func FeedBackComb(src source.Source, inGain, backGain float32, delayLen int) source.Source {
	delayBuf := make([]*types.DelayLine, 2)
	delayBuf[0] = types.NewDelayLine(delayLen)
	delayBuf[1] = types.NewDelayLine(delayLen)
	return source.Cached(func(step int) []float32 {
		input := src(step)
		delayOut := utils.MakeSample(2)
		delayOut[0] = delayBuf[0].Read()
		delayOut[1] = delayBuf[1].Read()
		delayIn := utils.MakeSample(2)
		delayIn[0] = input[0] + (delayOut[0] * -1 * backGain)
		delayIn[1] = input[1] + (delayOut[1] * -1 * backGain)
		delayBuf[0].Write(delayIn[0])
		delayBuf[1].Write(delayIn[1])
		out := utils.MakeSample(2)
		out[0] = delayIn[0] * inGain
		out[1] = delayIn[1] * inGain
		return out
	})
}

// FeedForwardComb is a simple feedforward comb filter, as defined at
// https://ccrma.stanford.edu/~jos/pasp/Feedforward_Comb_Filters.html
func FeedForwardComb(src source.Source, inGain, outGain float32, delayLen int) source.Source {
	delayBuf := make([]*types.DelayLine, 2)
	delayBuf[0] = types.NewDelayLine(delayLen)
	delayBuf[1] = types.NewDelayLine(delayLen)
	return source.Cached(func(step int) []float32 {
		input := src(step)
		delayOut := utils.MakeSample(2)
		delayOut[0] = delayBuf[0].Step(input[0])
		delayOut[1] = delayBuf[1].Step(input[1])
		out := utils.MakeSample(2)
		out[0] = (input[0] * inGain) + (delayOut[0] * outGain)
		out[1] = (input[1] * inGain) + (delayOut[1] * outGain)
		return out
	})
}
<|endoftext|>


<commit_before>// Copyright 2017 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// DO NOT EDIT doc.go. Modify internal/doc.template, then run make -C internal.

/*
Package firestore provides a client for reading and writing to a Cloud Firestore
database.

See https://cloud.google.com/firestore/docs for an introduction
to Cloud Firestore and additional help on using the Firestore API.

See https://godoc.org/cloud.google.com/go for authentication, timeouts,
connection pooling and similar aspects of this package.

Note: you can't use both Cloud Firestore and Cloud Datastore in the same
project.

# Creating a Client

To start working with this package, create a client with a project ID:

	ctx := context.Background()
	client, err := firestore.NewClient(ctx, "projectID")
	if err != nil {
		// TODO: Handle error.
	}

# CollectionRefs and DocumentRefs

In Firestore, documents are sets of key-value pairs, and collections are groups of
documents. A Firestore database consists of a hierarchy of alternating collections
and documents, referred to by slash-separated paths like
"States/California/Cities/SanFrancisco".

This client is built around references to collections and documents. CollectionRefs
and DocumentRefs are lightweight values that refer to the corresponding database
entities. Creating a ref does not involve any network traffic.

	states := client.Collection("States")
	ny := states.Doc("NewYork")
	// Or, in a single call:
	ny = client.Doc("States/NewYork")

# Reading

Use DocumentRef.Get to read a document. The result is a DocumentSnapshot.
Call its Data method to obtain the entire document contents as a map.

	docsnap, err := ny.Get(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	dataMap := docsnap.Data()
	fmt.Println(dataMap)

You can also obtain a single field with DataAt, or extract the data into a struct
with DataTo. With the type definition

	type State struct {
		Capital    string  `firestore:"capital"`
		Population float64 `firestore:"pop"` // in millions
	}

we can extract the document's data into a value of type State:

	var nyData State
	if err := docsnap.DataTo(&nyData); err != nil {
		// TODO: Handle error.
	}

Note that this client supports struct tags beginning with "firestore:" that work like
the tags of the encoding/json package, letting you rename fields, ignore them, or
omit their values when empty.

To retrieve multiple documents from their references in a single call, use
Client.GetAll.

	docsnaps, err := client.GetAll(ctx, []*firestore.DocumentRef{
		states.Doc("Wisconsin"), states.Doc("Ohio"),
	})
	if err != nil {
		// TODO: Handle error.
	}
	for _, ds := range docsnaps {
		_ = ds // TODO: Use ds.
	}

# Writing

For writing individual documents, use the methods on DocumentReference.
Create creates a new document.

	wr, err := ny.Create(ctx, State{
		Capital:    "Albany",
		Population: 19.8,
	})
	if err != nil {
		// TODO: Handle error.
	}
	fmt.Println(wr)

The first return value is a WriteResult, which contains the time
at which the document was updated.

Create fails if the document exists. Another method, Set, either replaces an existing
document or creates a new one.

	ca := states.Doc("California")
	_, err = ca.Set(ctx, State{
		Capital:    "Sacramento",
		Population: 39.14,
	})

To update some fields of an existing document, use Update. It takes a list of
paths to update and their corresponding values.

	_, err = ca.Update(ctx, []firestore.Update{{Path: "capital", Value: "Sacramento"}})

Use DocumentRef.Delete to delete a document.

	_, err = ny.Delete(ctx)

# Preconditions

You can condition Deletes or Updates on when a document was last changed. Specify
these preconditions as an option to a Delete or Update method. The check and the
write happen atomically with a single RPC.

	docsnap, err = ca.Get(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	_, err = ca.Update(ctx,
		[]firestore.Update{{Path: "capital", Value: "Sacramento"}},
		firestore.LastUpdateTime(docsnap.UpdateTime))

Here we update a doc only if it hasn't changed since we read it.
You could also do this with a transaction.

To perform multiple writes at once, use a WriteBatch. Its methods chain
for convenience.

WriteBatch.Commit sends the collected writes to the server, where they happen
atomically.

	writeResults, err := client.Batch().
		Create(ny, State{Capital: "Albany"}).
		Update(ca, []firestore.Update{{Path: "capital", Value: "Sacramento"}}).
		Delete(client.Doc("States/WestDakota")).
		Commit(ctx)

# Queries

You can use SQL to select documents from a collection. Begin with the collection, and
build up a query using Select, Where and other methods of Query.

	q := states.Where("pop", ">", 10).OrderBy("pop", firestore.Desc)

Supported operators include '<', '<=', '>', '>=', '==', 'in', 'array-contains', and
'array-contains-any'.

Call the Query's Documents method to get an iterator, and use it like
the other Google Cloud Client iterators.

	iter := q.Documents(ctx)
	defer iter.Stop()
	for {
		doc, err := iter.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			// TODO: Handle error.
		}
		fmt.Println(doc.Data())
	}

To get all the documents in a collection, you can use the collection itself
as a query.

	iter = client.Collection("States").Documents(ctx)

# Collection Group Partition Queries

You can partition the documents of a Collection Group allowing for smaller subqueries.

	collectionGroup = client.CollectionGroup("States")
	partitions, err = collectionGroup.GetPartitionedQueries(ctx, 20)

You can also Serialize/Deserialize queries making it possible to run/stream the
queries elsewhere; another process or machine for instance.

	queryProtos := make([][]byte, 0)
	for _, query := range partitions {
		protoBytes, err := query.Serialize()
		// handle err
		queryProtos = append(queryProtos, protoBytes)
		...
	}

	for _, protoBytes := range queryProtos {
		query, err := client.CollectionGroup("").Deserialize(protoBytes)
		...
	}

# Transactions

Use a transaction to execute reads and writes atomically. All reads must happen
before any writes. Transaction creation, commit, rollback and retry are handled for
you by the Client.RunTransaction method; just provide a function and use the
read and write methods of the Transaction passed to it.

	ny := client.Doc("States/NewYork")
	err := client.RunTransaction(ctx, func(ctx context.Context, tx *firestore.Transaction) error {
		doc, err := tx.Get(ny) // tx.Get, NOT ny.Get!
		if err != nil {
			return err
		}
		pop, err := doc.DataAt("pop")
		if err != nil {
			return err
		}
		return tx.Update(ny, []firestore.Update{{Path: "pop", Value: pop.(float64) + 0.2}})
	})
	if err != nil {
		// TODO: Handle error.
	}

# Google Cloud Firestore Emulator

This package supports the Cloud Firestore emulator, which is useful for testing and
development. Environment variables are used to indicate that Firestore traffic should be
directed to the emulator instead of the production Firestore service.

To install and run the emulator and its environment variables, see the documentation
at https://cloud.google.com/sdk/gcloud/reference/beta/emulators/firestore/. Once the
emulator is running, set FIRESTORE_EMULATOR_HOST to the API endpoint.
*/
package firestore
<commit_msg>docs(firestore): adds emulator snippet (#6926)<commit_after>// Copyright 2017 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// DO NOT EDIT doc.go. Modify internal/doc.template, then run make -C internal.

/*
Package firestore provides a client for reading and writing to a Cloud Firestore
database.

See https://cloud.google.com/firestore/docs for an introduction
to Cloud Firestore and additional help on using the Firestore API.

See https://godoc.org/cloud.google.com/go for authentication, timeouts,
connection pooling and similar aspects of this package.

Note: you can't use both Cloud Firestore and Cloud Datastore in the same
project.

# Creating a Client

To start working with this package, create a client with a project ID:

	ctx := context.Background()
	client, err := firestore.NewClient(ctx, "projectID")
	if err != nil {
		// TODO: Handle error.
	}

# CollectionRefs and DocumentRefs

In Firestore, documents are sets of key-value pairs, and collections are groups of
documents. A Firestore database consists of a hierarchy of alternating collections
and documents, referred to by slash-separated paths like
"States/California/Cities/SanFrancisco".

This client is built around references to collections and documents. CollectionRefs
and DocumentRefs are lightweight values that refer to the corresponding database
entities. Creating a ref does not involve any network traffic.

	states := client.Collection("States")
	ny := states.Doc("NewYork")
	// Or, in a single call:
	ny = client.Doc("States/NewYork")

# Reading

Use DocumentRef.Get to read a document. The result is a DocumentSnapshot.
Call its Data method to obtain the entire document contents as a map.

	docsnap, err := ny.Get(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	dataMap := docsnap.Data()
	fmt.Println(dataMap)

You can also obtain a single field with DataAt, or extract the data into a struct
with DataTo. With the type definition

	type State struct {
		Capital    string  `firestore:"capital"`
		Population float64 `firestore:"pop"` // in millions
	}

we can extract the document's data into a value of type State:

	var nyData State
	if err := docsnap.DataTo(&nyData); err != nil {
		// TODO: Handle error.
	}

Note that this client supports struct tags beginning with "firestore:" that work like
the tags of the encoding/json package, letting you rename fields, ignore them, or
omit their values when empty.

To retrieve multiple documents from their references in a single call, use
Client.GetAll.

	docsnaps, err := client.GetAll(ctx, []*firestore.DocumentRef{
		states.Doc("Wisconsin"), states.Doc("Ohio"),
	})
	if err != nil {
		// TODO: Handle error.
	}
	for _, ds := range docsnaps {
		_ = ds // TODO: Use ds.
	}

# Writing

For writing individual documents, use the methods on DocumentReference.
Create creates a new document.

	wr, err := ny.Create(ctx, State{
		Capital:    "Albany",
		Population: 19.8,
	})
	if err != nil {
		// TODO: Handle error.
	}
	fmt.Println(wr)

The first return value is a WriteResult, which contains the time
at which the document was updated.

Create fails if the document exists. Another method, Set, either replaces an existing
document or creates a new one.

	ca := states.Doc("California")
	_, err = ca.Set(ctx, State{
		Capital:    "Sacramento",
		Population: 39.14,
	})

To update some fields of an existing document, use Update. It takes a list of
paths to update and their corresponding values.

	_, err = ca.Update(ctx, []firestore.Update{{Path: "capital", Value: "Sacramento"}})

Use DocumentRef.Delete to delete a document.

	_, err = ny.Delete(ctx)

# Preconditions

You can condition Deletes or Updates on when a document was last changed. Specify
these preconditions as an option to a Delete or Update method. The check and the
write happen atomically with a single RPC.

	docsnap, err = ca.Get(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	_, err = ca.Update(ctx,
		[]firestore.Update{{Path: "capital", Value: "Sacramento"}},
		firestore.LastUpdateTime(docsnap.UpdateTime))

Here we update a doc only if it hasn't changed since we read it.
You could also do this with a transaction.

To perform multiple writes at once, use a WriteBatch. Its methods chain
for convenience.

WriteBatch.Commit sends the collected writes to the server, where they happen
atomically.

	writeResults, err := client.Batch().
		Create(ny, State{Capital: "Albany"}).
		Update(ca, []firestore.Update{{Path: "capital", Value: "Sacramento"}}).
		Delete(client.Doc("States/WestDakota")).
		Commit(ctx)

# Queries

You can use SQL to select documents from a collection. Begin with the collection, and
build up a query using Select, Where and other methods of Query.

	q := states.Where("pop", ">", 10).OrderBy("pop", firestore.Desc)

Supported operators include '<', '<=', '>', '>=', '==', 'in', 'array-contains', and
'array-contains-any'.

Call the Query's Documents method to get an iterator, and use it like
the other Google Cloud Client iterators.

	iter := q.Documents(ctx)
	defer iter.Stop()
	for {
		doc, err := iter.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			// TODO: Handle error.
		}
		fmt.Println(doc.Data())
	}

To get all the documents in a collection, you can use the collection itself
as a query.

	iter = client.Collection("States").Documents(ctx)

# Collection Group Partition Queries

You can partition the documents of a Collection Group allowing for smaller subqueries.

	collectionGroup = client.CollectionGroup("States")
	partitions, err = collectionGroup.GetPartitionedQueries(ctx, 20)

You can also Serialize/Deserialize queries making it possible to run/stream the
queries elsewhere; another process or machine for instance.

	queryProtos := make([][]byte, 0)
	for _, query := range partitions {
		protoBytes, err := query.Serialize()
		// handle err
		queryProtos = append(queryProtos, protoBytes)
		...
	}

	for _, protoBytes := range queryProtos {
		query, err := client.CollectionGroup("").Deserialize(protoBytes)
		...
	}

# Transactions

Use a transaction to execute reads and writes atomically. All reads must happen
before any writes. Transaction creation, commit, rollback and retry are handled for
you by the Client.RunTransaction method; just provide a function and use the
read and write methods of the Transaction passed to it.

	ny := client.Doc("States/NewYork")
	err := client.RunTransaction(ctx, func(ctx context.Context, tx *firestore.Transaction) error {
		doc, err := tx.Get(ny) // tx.Get, NOT ny.Get!
		if err != nil {
			return err
		}
		pop, err := doc.DataAt("pop")
		if err != nil {
			return err
		}
		return tx.Update(ny, []firestore.Update{{Path: "pop", Value: pop.(float64) + 0.2}})
	})
	if err != nil {
		// TODO: Handle error.
	}

# Google Cloud Firestore Emulator

This package supports the Cloud Firestore emulator, which is useful for testing and
development. Environment variables are used to indicate that Firestore traffic should be
directed to the emulator instead of the production Firestore service.

To install and run the emulator and its environment variables, see the documentation
at https://cloud.google.com/sdk/gcloud/reference/beta/emulators/firestore/. Once the
emulator is running, set FIRESTORE_EMULATOR_HOST to the API endpoint.

	// Set FIRESTORE_EMULATOR_HOST environment variable.
	err := os.Setenv("FIRESTORE_EMULATOR_HOST", "localhost:9000")
	if err != nil {
		// TODO: Handle error.
	}
	// Create client as usual.
	client, err := firestore.NewClient(ctx, "my-project-id")
	if err != nil {
		// TODO: Handle error.
	}
	defer client.Close()
*/
package firestore
<|endoftext|>


<commit_before>package network

import (
	"net"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Network represents a LXD network.
type Network interface {
	// Load.
	init(state *state.State, id int64, projectName string, name string, netType string, description string, config map[string]string, status string)
	fillConfig(config map[string]string) error
	Info() Info

	// Config.
	ValidateName(name string) error
	Validate(config map[string]string) error
	ID() int64
	Name() string
	Type() string
	Description() string
	Status() string
	Config() map[string]string
	IsUsed() (bool, error)
	DHCPv4Subnet() *net.IPNet
	DHCPv6Subnet() *net.IPNet
	DHCPv4Ranges() []shared.IPRange
	DHCPv6Ranges() []shared.IPRange

	// Actions.
	Create(clientType cluster.ClientType) error
	Start() error
	Stop() error
	Rename(name string) error
	Update(newNetwork api.NetworkPut, targetNode string, clientType cluster.ClientType) error
	HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error
	Delete(clientType cluster.ClientType) error
}
<commit_msg>lxd/network/network/interface: Adds Type interface and moves non-DB depedent functions into it<commit_after>package network

import (
	"net"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Type represents an LXD network driver type.
type Type interface {
	fillConfig(config map[string]string) error
	Info() Info
	ValidateName(name string) error
	Type() string
}

// Network represents an instantiated LXD network.
type Network interface {
	Type

	// Load.
	init(state *state.State, id int64, projectName string, name string, netType string, description string, config map[string]string, status string)

	// Config.
	Validate(config map[string]string) error
	ID() int64
	Name() string
	Description() string
	Status() string
	Config() map[string]string
	IsUsed() (bool, error)
	DHCPv4Subnet() *net.IPNet
	DHCPv6Subnet() *net.IPNet
	DHCPv4Ranges() []shared.IPRange
	DHCPv6Ranges() []shared.IPRange

	// Actions.
	Create(clientType cluster.ClientType) error
	Start() error
	Stop() error
	Rename(name string) error
	Update(newNetwork api.NetworkPut, targetNode string, clientType cluster.ClientType) error
	HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error
	Delete(clientType cluster.ClientType) error
}
<|endoftext|>


<commit_before>package etcd

import (
	"context"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	"github.com/prometheus/common/model"

	"github.com/openshift/origin/test/extended/prometheus/client"
	exutil "github.com/openshift/origin/test/extended/util"
)

var _ = g.Describe("[sig-etcd] etcd", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("etcd-leader-change").AsAdmin()
	g.It("leader changes are not excessive", func() {
		prometheus, err := client.NewE2EPrometheusRouterClient(oc)
		o.Expect(err).ToNot(o.HaveOccurred())
		g.By("Examining the rate of increase in the number of etcd leadership changes for last five minutes")
		result, _, err := prometheus.Query(context.Background(), "increase((max by (job) (etcd_server_leader_changes_seen_total) or 0*absent(etcd_server_leader_changes_seen_total))[15m:1m])", time.Now())
		o.Expect(err).ToNot(o.HaveOccurred())
		o.Expect(result.(model.Vector)[0].Value).To(o.BeNumerically("==", 0))
	})
})
<commit_msg>test/extended/etcd: make excessive leader_change test actionable<commit_after>package etcd

import (
	"context"
	"fmt"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	"github.com/prometheus/common/model"

	"github.com/openshift/origin/test/extended/prometheus/client"
	exutil "github.com/openshift/origin/test/extended/util"
)

var _ = g.Describe("[sig-etcd] etcd", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLI("etcd-leader-change").AsAdmin()
	g.It("leader changes are not excessive", func() {
		prometheus, err := client.NewE2EPrometheusRouterClient(oc)
		o.Expect(err).ToNot(o.HaveOccurred())
		g.By("Examining the rate of increase in the number of etcd leadership changes for last five minutes")
		result, _, err := prometheus.Query(context.Background(), "increase((max by (job) (etcd_server_leader_changes_seen_total) or 0*absent(etcd_server_leader_changes_seen_total))[15m:1m])", time.Now())
		o.Expect(err).ToNot(o.HaveOccurred())
		leaderChangeLastFiveMinutes := result.(model.Vector)[0].Value
		if leaderChangeLastFiveMinutes != 0 {
			o.Expect(fmt.Errorf("Leader changes observed last 5m %q: Leader changes are a result of stopping the etcd leader process or from latency (disk or network), review etcd performance metrics", leaderChangeLastFiveMinutes)).ToNot(o.HaveOccurred())
		}
	})
})
<|endoftext|>