<commit_before>package protocol

import (
	"bytes"
	"net"
	"testing"
)

type messageTestPair struct {
	command string
	payload []byte
	message []byte
}

var messageTests = []messageTestPair{ // generated using CreatePacket in shared.py
	{"hey", nil, []byte{233, 190, 180, 217, 104, 101, 121, 0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 207, 131, 225, 53}},
	{"message", []byte("Some huge test message"), []byte{233, 190, 180, 217, 109,
		101, 115, 115, 97, 103, 101, 0, 0, 0, 0, 0, 0, 0, 0, 22, 11, 150, 64, 0, 83,
		111, 109, 101, 32, 104, 117, 103, 101, 32, 116, 101, 115, 116, 32, 109, 101,
		115, 115, 97, 103, 101}},
	{"die", []byte("you don't deserve to live, my friend"), []byte{233, 190, 180,
		217, 100, 105, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 240, 214, 59,
		80, 121, 111, 117, 32, 100, 111, 110, 39, 116, 32, 100, 101, 115, 101, 114,
		118, 101, 32, 116, 111, 32, 108, 105, 118, 101, 44, 32, 109, 121, 32, 102,
		114, 105, 101, 110, 100}},
}

func TestCreateMessage(t *testing.T) {
	for _, pair := range messageTests {
		msg := CreateMessage(pair.command, pair.payload)
		if !bytes.Equal(msg, pair.message) {
			t.Error("for command", pair.command, "payload", pair.payload, "expected",
				pair.message, "got", msg)
		}
	}
}

func TestCreateVerackMessage(t *testing.T) {
	b := CreateVerackMessage()
	msgBytes := []byte{233, 190, 180, 217, 118, 101, 114, 97, 99, 107, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 207, 131, 225, 53} // CreatePacket('verack') in shared.py
	if !bytes.Equal(b, msgBytes) {
		t.Error("invalid verack message, got", b)
	}
}

func TestUnpackMessageHeader(t *testing.T) {
	for i, pair := range messageTests {
		command, length, checksum, err := UnpackMessageHeader(pair.message)
		if err != nil {
			t.Error("got error:", err.Error())
		}
		if pair.command != command {
			t.Error("for case", i+1, "expected command", pair.command, "got", command)
		}
		if len(pair.payload) != int(length) {
			t.Error("for case", i+1, "expected payload length", len(pair.payload),
				"got", length)
		}
		// checksum is from bytes 20-24
		if !bytes.Equal(pair.message[20:24], checksum[:]) {
			t.Error("for case", i+1, "expected checksum", pair.message[20:24],
				"got", checksum[:])
		}
	}
}

// Verify the checksum
func TestVerifyMessageChecksum(t *testing.T) {
	for i, pair := range messageTests {
		// checksum is from bytes 20-24
		var checksum [4]byte
		copy(checksum[:], pair.message[20:24])

		if !VerifyMessageChecksum(pair.message[24:], checksum) {
			t.Error("for case", i+1, "checksum verification failed")
		}
	}
}

func TestVersionMessage(t *testing.T) {
	var (
		time       int64  = 1416114153
		remoteHost        = net.ParseIP("192.168.0.1")
		remotePort uint16 = 8444
		localPort  uint16 = 8444
		// Ignored by the remote host. The actual remote connected IP used.
		localHost        = net.ParseIP("127.0.0.1")
		nonce     uint64 = 54562198651689
	)

	vMsg := VersionMessage{
		Version:   3,
		Services:  1,
		Timestamp: time,
		AddrRecv: NetworkAddressShort{
			Services: 1,
			IP:       remoteHost,
			Port:     remotePort,
		},
		AddrFrom: NetworkAddressShort{
			Services: 1,
			IP:       localHost,
			Port:     localPort, // local port
		},
		Nonce:     nonce, // Random value
		UserAgent: Varstring("/BM-Go:0.0.1/"),
		Streams:   VarintList{1},
	}

	_ = vMsg.Serialize()
}

func TestAddrMessage(t *testing.T) {

}
<commit_msg>Written version message test<commit_after>package protocol

import (
	"bytes"
	"fmt"
	"net"
	"reflect"
	"testing"
)

type messageTestPair struct {
	command string
	payload []byte
	message []byte
}

var messageTests = []messageTestPair{ // generated using CreatePacket in shared.py
	{"hey", nil, []byte{233, 190, 180, 217, 104, 101, 121, 0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 207, 131, 225, 53}},
	{"message", []byte("Some huge test message"), []byte{233, 190, 180, 217, 109,
		101, 115, 115, 97, 103, 101, 0, 0, 0, 0, 0, 0, 0, 0, 22, 11, 150, 64, 0, 83,
		111, 109, 101, 32, 104, 117, 103, 101, 32, 116, 101, 115, 116, 32, 109, 101,
		115, 115, 97, 103, 101}},
	{"die", []byte("you don't deserve to live, my friend"), []byte{233, 190, 180,
		217, 100, 105, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 240, 214, 59,
		80, 121, 111, 117, 32, 100, 111, 110, 39, 116, 32, 100, 101, 115, 101, 114,
		118, 101, 32, 116, 111, 32, 108, 105, 118, 101, 44, 32, 109, 121, 32, 102,
		114, 105, 101, 110, 100}},
}

func TestCreateMessage(t *testing.T) {
	for _, pair := range messageTests {
		msg := CreateMessage(pair.command, pair.payload)
		if !bytes.Equal(msg, pair.message) {
			t.Error("for command", pair.command, "payload", pair.payload, "expected",
				pair.message, "got", msg)
		}
	}
}

func TestCreateVerackMessage(t *testing.T) {
	b := CreateVerackMessage()
	msgBytes := []byte{233, 190, 180, 217, 118, 101, 114, 97, 99, 107, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 207, 131, 225, 53} // CreatePacket('verack') in shared.py
	if !bytes.Equal(b, msgBytes) {
		t.Error("invalid verack message, got", b)
	}
}

func TestUnpackMessageHeader(t *testing.T) {
	for i, pair := range messageTests {
		command, length, checksum, err := UnpackMessageHeader(pair.message)
		if err != nil {
			t.Error("got error:", err.Error())
		}
		if pair.command != command {
			t.Error("for case", i+1, "expected command", pair.command, "got", command)
		}
		if len(pair.payload) != int(length) {
			t.Error("for case", i+1, "expected payload length", len(pair.payload),
				"got", length)
		}
		// checksum is from bytes 20-24
		if !bytes.Equal(pair.message[20:24], checksum[:]) {
			t.Error("for case", i+1, "expected checksum", pair.message[20:24],
				"got", checksum[:])
		}
	}
}

// Verify the checksum
func TestVerifyMessageChecksum(t *testing.T) {
	for i, pair := range messageTests {
		// checksum is from bytes 20-24
		var checksum [4]byte
		copy(checksum[:], pair.message[20:24])

		if !VerifyMessageChecksum(pair.message[24:], checksum) {
			t.Error("for case", i+1, "checksum verification failed")
		}
	}
}

func TestNetworkAddressShort(t *testing.T) {

}

func TestNetworkAddress(t *testing.T) {

}

func TestVersionMessage(t *testing.T) {
	var (
		time       int64  = 1416114153
		remoteHost        = net.ParseIP("192.168.0.1")
		remotePort uint16 = 8444
		localPort  uint16 = 8444
		// Ignored by the remote host. The actual remote connected IP used.
		localHost        = net.ParseIP("127.0.0.1")
		nonce     uint64 = 54562198651689
	)

	vMsg := VersionMessage{
		Version:   3,
		Services:  1,
		Timestamp: time,
		AddrRecv: NetworkAddressShort{
			Services: 1,
			IP:       remoteHost,
			Port:     remotePort,
		},
		AddrFrom: NetworkAddressShort{
			Services: 1,
			IP:       localHost,
			Port:     localPort, // local port
		},
		Nonce:     nonce, // Random value
		UserAgent: Varstring("/BM-Go:0.0.1/"),
		Streams:   VarintList{1},
	}

	testRes := vMsg.Serialize()
	res := []byte{233, 190, 180, 217, 118, 101, 114, 115, 105, 111, 110, 0, 0, 0,
		0, 0, 0, 0, 0, 96, 133, 59, 125, 112, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1,
		0, 0, 0, 0, 84, 104, 47, 233, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 255, 255, 192, 168, 0, 1, 32, 252, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 0, 0, 1, 32, 252, 0, 0, 49, 159,
		192, 120, 3, 41, 13, 47, 66, 77, 45, 71, 111, 58, 48, 46, 48, 46, 49, 47,
		1, 1} // encoded using Python

	if !bytes.Equal(testRes, res) {
		t.Error("error encoding version message")
	}

	var vMsgTest VersionMessage
	err := vMsgTest.Deserialize(res[MessageHeaderSize():]) // exclude the header
	if err != nil {
		t.Error("error decoding version message: " + err.Error())
	}

	if !reflect.DeepEqual(vMsg, vMsgTest) {
		t.Error("version message not equal to test")
		fmt.Printf("%+v\n", vMsgTest)
	}
}

func TestAddrMessage(t *testing.T) {

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"sort"
	"strings"
	"time"
)

var f1name = flag.String("f1", "", "First CSV file name to compare")
var f2name = flag.String("f2", "", "Second CSV file name to compare")
var output = flag.String("o", "", "Output CSV file for differences")
var key = flag.Int("key", 0, "Key column in input CSVs (first is 1); must be unique")
var help = flag.Bool("help", false, "Show help message")
var ondupfirst = flag.Bool("ondupFirst", false, "On duplicate key, keep first one")
var onduplast = flag.Bool("ondupLast", false, "On duplicate key, keep last  one")

var detailedHelp = `
	Detailed Help:
	Inputs:
		- a key column
		- two input filenames
		- an output filename
	There will be two input files to compare and there will be
	one output file created:
	a) The first file will be read and stored into a map
	b) The second file will be read and stored into a map
	c) It is an error if a file has the same key value on two rows.
	Keys must be unique within each file.
	Note that key column number is one based, not zero based!
	NOTE! if duplicate keys exist, then there are options to keep
	the first or to keep the last one. Default is to error out.
	d) Then all keys from both inputs are combined/deduped/sorted
	e) Then we range over the combined keyset and output a new CSV
	that has a new status column as the first column and the other columns
	from the inputs as the remaining columns.
	f) the new status column has the following values:
	- EQ meaning that the values for the key are same in both input files
	- IN=1 meaning that the key and values are only in input file #1
	- IN=2 similar for input file #2
	- DFn=x,y,..,z where n is either 1 or 2; followed by a comma delimited
	list of column numbers where the values for the key do not match.
	Note that the DF statuses always come in pairs, one for each input file.
	g) Limitations:
	- both input files must have the same number of columns
	- both must have a header row and the headers must be the same
`

func main() {
	flag.Parse()

	if *help {
		usage("")
	}

	if *key == 0 {
		usage("Key column number missing.")
	}

	if *f1name == "" {
		usage("First filename is missing.")
	}

	if *f2name == "" {
		fmt.Println()
		usage("Second filename is missing.")
	}

	if *output == "" {
		fmt.Println()
		usage("Output filename is missing.")
	}

	now := time.Now()
	log.Printf("Start: %v", now.Format(time.StampMilli))

	// open first input file stop.Format(Time.StampMilli)
	var r1 *csv.Reader
	f1, f1err := os.Open(*f1name)
	if f1err != nil {
		log.Fatal("os.Open() Error:" + f1err.Error())
	}
	r1 = csv.NewReader(f1)

	// open second input file
	var r2 *csv.Reader
	f2, f2err := os.Open(*f2name)
	if f2err != nil {
		log.Fatal("os.Open() Error:" + f2err.Error())
	}
	r2 = csv.NewReader(f2)

	/*********************************************************/
	// do a quick check on columns first
	// if not the same, then log error and exit

	// second file
	hdrs2, rerr := r2.Read()
	if rerr == io.EOF {
		log.Fatal("File 2 is empty", rerr)
	}
	if rerr != nil {
		log.Fatalf("csv.Read:\n%v\n", rerr)
	}
	numcols2 := len(hdrs2)

	// first file
	hdrs1, rerr := r1.Read()
	if rerr == io.EOF {
		log.Fatal("File 1 is empty", rerr)
	}
	if rerr != nil {
		log.Fatalf("csv.Read:\n%v\n", rerr)
	}
	numcols1 := len(hdrs1)

	if numcols1 != numcols2 {
		log.Fatalf("Different number of columns:%v vs. %v",
			numcols1, numcols2)
	}

	// check that headers are the same
	for i := range hdrs1 {
		if hdrs1[i] == hdrs2[i] {
			continue
		}
		log.Fatal("Headers are not the same on input files")
	}

	// set expectations of fields per row
	r1.FieldsPerRecord = numcols1
	r2.FieldsPerRecord = numcols1

	// open output file
	var wf1 *csv.Writer
	wf1o, wf1oerr := os.Create(*output)
	if wf1oerr != nil {
		log.Fatal("os.Create() Error:" + wf1oerr.Error())
	}
	defer wf1o.Close()
	wf1 = csv.NewWriter(wf1o)
	hdrOutput := make([]string, 0)
	hdrOutput = append(hdrOutput, "STATUS")
	hdrOutput = append(hdrOutput, hdrs1...)
	err := wf1.Write(hdrOutput)
	if err != nil {
		log.Fatalf("Output Error:\n%v\n", err)
	}

	log.Printf("Processing input #1:%v\n", *f1name)
	f1map := make(map[string][]string)
	// read first file
	rows := 0
	for {
		// read the csv file
		cells, rerr := r1.Read()
		if rerr == io.EOF {
			break
		}
		if rerr != nil {
			log.Fatalf("csv.Read:\n%v\n", rerr)
		}
		rows++
		keyv := cells[*key-1]
		if _, ok := f1map[keyv]; ok {
			if *onduplast {
				log.Printf("Replacing non-unique key: %v on row %v\n", keyv, rows+1)
			} else if *ondupfirst {
				log.Printf("Skipping non-unique key: %v on row %v\n", keyv, rows+1)
				continue
			} else {
				log.Fatalf("Key value not unique: %v on row %v\n", keyv, rows+1)
			}
		}
		f1map[keyv] = cells
	}
	log.Printf("Number of rows in file %v:%v\n", *f1name, rows)
	f1.Close()

	log.Printf("Processing input #2:%v\n", *f2name)
	f2map := make(map[string][]string)
	// read second file
	rows = 0
	for {
		// read the csv file
		cells, rerr := r2.Read()
		if rerr == io.EOF {
			break
		}
		if rerr != nil {
			log.Fatalf("csv.Read:\n%v\n", rerr)
		}
		rows++
		keyv := cells[*key-1]
		if _, ok := f2map[keyv]; ok {
			if *onduplast {
				log.Printf("Replacing non-unique key: %v on row %v\n", keyv, rows+1)
			} else if *ondupfirst {
				log.Printf("Skipping non-unique key: %v on row %v\n", keyv, rows+1)
				continue
			} else {
				log.Fatalf("Key value not unique: %v on row %v\n", keyv, rows+1)
			}
		}
		f2map[keyv] = cells
	}
	log.Printf("Number of rows in file %v:%v\n", *f2name, rows)
	f2.Close()

	//
	// Get a combined set of keys
	//
	uniqkeyset := make(map[string]struct{})
	for k := range f1map {
		uniqkeyset[k] = struct{}{}
	}
	for k := range f2map {
		uniqkeyset[k] = struct{}{}
	}
	keySliceSize := len(uniqkeyset)
	keys := make([]string, keySliceSize)
	slot := 0
	for k := range uniqkeyset {
		keys[slot] = k
		slot++
	}
	log.Printf("Number of combined unique keys:%v\n", keySliceSize)

	// sort them
	sort.Slice(keys, func(i, j int) bool {
		return keys[i] < keys[j]
	})

	// Now range of combined unique keys
	for n := range keys {
		val := keys[n]
		row1, ok1 := f1map[val]
		row2, ok2 := f2map[val]
		if ok1 && ok2 {
			// are all the row values the same?
			diffList := make([]int, 0)
			for i := range row1 {
				if row1[i] == row2[i] {
					continue
				}
				f := i - 1
				diffList = append(diffList, f)
			}
			if len(diffList) == 0 {
				outrow1 := make([]string, 0)
				outrow1 = append(outrow1, "EQ")
				outrow1 = append(outrow1, row1...)
				err := wf1.Write(outrow1)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
			} else {
				diffs := ""
				for i := range diffList {
					diffs += fmt.Sprintf("%v,", diffList[i]+2)
				}
				diffs = strings.TrimRight(diffs, ",")
				outrow1 := make([]string, 0)
				outrow1 = append(outrow1, fmt.Sprintf("DF1=%v", diffs))
				outrow1 = append(outrow1, row1...)
				err := wf1.Write(outrow1)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
				outrow2 := make([]string, 0)
				outrow2 = append(outrow2, fmt.Sprintf("DF2=%v", diffs))
				outrow2 = append(outrow2, row2...)
				err = wf1.Write(outrow2)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
			}
		} else {
			if !ok1 {
				outrow := make([]string, 0)
				outrow = append(outrow, "IN=2")
				outrow = append(outrow, row2...)
				err := wf1.Write(outrow)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
			} else {
				outrow := make([]string, 0)
				outrow = append(outrow, "IN=1")
				outrow = append(outrow, row1...)
				err := wf1.Write(outrow)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
			}
		}

	}
	wf1.Flush()

	// wrapup
	stop := time.Now()
	elapsed := time.Since(now)
	log.Printf("End: %v", stop.Format(time.StampMilli))
	log.Printf("Elapsed time %v", elapsed)

}

func usage(msg string) {
	fmt.Println(msg)
	fmt.Print("Usage: diffcsv [options]\n")
	flag.PrintDefaults()
	if msg == "" {
		fmt.Println(detailedHelp)
	}
	os.Exit(0)
}
<commit_msg>Error to use both on-dup options<commit_after>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"sort"
	"strings"
	"time"
)

var f1name = flag.String("f1", "", "First CSV file name to compare")
var f2name = flag.String("f2", "", "Second CSV file name to compare")
var output = flag.String("o", "", "Output CSV file for differences")
var key = flag.Int("key", 0, "Key column in input CSVs (first is 1); must be unique")
var help = flag.Bool("help", false, "Show help message")
var ondupfirst = flag.Bool("ondupFirst", false, "On duplicate key, keep first one")
var onduplast = flag.Bool("ondupLast", false, "On duplicate key, keep last  one")

var detailedHelp = `
	Detailed Help:
	Inputs:
		- a key column
		- two input filenames
		- an output filename
	There will be two input files to compare and there will be
	one output file created:
	a) The first file will be read and stored into a map
	b) The second file will be read and stored into a map
	c) It is an error if a file has the same key value on two rows.
	Keys must be unique within each file.
	Note that key column number is one based, not zero based!
	NOTE! if duplicate keys exist, then there are options to keep
	the first or to keep the last one. Default is to error out.
	d) Then all keys from both inputs are combined/deduped/sorted
	e) Then we range over the combined keyset and output a new CSV
	that has a new status column as the first column and the other columns
	from the inputs as the remaining columns.
	f) the new status column has the following values:
	- EQ meaning that the values for the key are same in both input files
	- IN=1 meaning that the key and values are only in input file #1
	- IN=2 similar for input file #2
	- DFn=x,y,..,z where n is either 1 or 2; followed by a comma delimited
	list of column numbers where the values for the key do not match.
	Note that the DF statuses always come in pairs, one for each input file.
	g) Limitations:
	- both input files must have the same number of columns
	- both must have a header row and the headers must be the same
`

func main() {
	flag.Parse()

	if *help {
		usage("")
	}

	if *key == 0 {
		usage("Key column number missing.")
	}

	if *f1name == "" {
		usage("First filename is missing.")
	}

	if *f2name == "" {
		fmt.Println()
		usage("Second filename is missing.")
	}

	if *output == "" {
		fmt.Println()
		usage("Output filename is missing.")
	}

	if *ondupFirst && *ondupLast {
		fmt.Println()
		usage("Cannot use both on-dup options")
	}

	now := time.Now()
	log.Printf("Start: %v", now.Format(time.StampMilli))

	// open first input file stop.Format(Time.StampMilli)
	var r1 *csv.Reader
	f1, f1err := os.Open(*f1name)
	if f1err != nil {
		log.Fatal("os.Open() Error:" + f1err.Error())
	}
	r1 = csv.NewReader(f1)

	// open second input file
	var r2 *csv.Reader
	f2, f2err := os.Open(*f2name)
	if f2err != nil {
		log.Fatal("os.Open() Error:" + f2err.Error())
	}
	r2 = csv.NewReader(f2)

	/*********************************************************/
	// do a quick check on columns first
	// if not the same, then log error and exit

	// second file
	hdrs2, rerr := r2.Read()
	if rerr == io.EOF {
		log.Fatal("File 2 is empty", rerr)
	}
	if rerr != nil {
		log.Fatalf("csv.Read:\n%v\n", rerr)
	}
	numcols2 := len(hdrs2)

	// first file
	hdrs1, rerr := r1.Read()
	if rerr == io.EOF {
		log.Fatal("File 1 is empty", rerr)
	}
	if rerr != nil {
		log.Fatalf("csv.Read:\n%v\n", rerr)
	}
	numcols1 := len(hdrs1)

	if numcols1 != numcols2 {
		log.Fatalf("Different number of columns:%v vs. %v",
			numcols1, numcols2)
	}

	// check that headers are the same
	for i := range hdrs1 {
		if hdrs1[i] == hdrs2[i] {
			continue
		}
		log.Fatal("Headers are not the same on input files")
	}

	// set expectations of fields per row
	r1.FieldsPerRecord = numcols1
	r2.FieldsPerRecord = numcols1

	// open output file
	var wf1 *csv.Writer
	wf1o, wf1oerr := os.Create(*output)
	if wf1oerr != nil {
		log.Fatal("os.Create() Error:" + wf1oerr.Error())
	}
	defer wf1o.Close()
	wf1 = csv.NewWriter(wf1o)
	hdrOutput := make([]string, 0)
	hdrOutput = append(hdrOutput, "STATUS")
	hdrOutput = append(hdrOutput, hdrs1...)
	err := wf1.Write(hdrOutput)
	if err != nil {
		log.Fatalf("Output Error:\n%v\n", err)
	}

	log.Printf("Processing input #1:%v\n", *f1name)
	f1map := make(map[string][]string)
	// read first file
	rows := 0
	for {
		// read the csv file
		cells, rerr := r1.Read()
		if rerr == io.EOF {
			break
		}
		if rerr != nil {
			log.Fatalf("csv.Read:\n%v\n", rerr)
		}
		rows++
		keyv := cells[*key-1]
		if _, ok := f1map[keyv]; ok {
			if *onduplast {
				log.Printf("Replacing non-unique key: %v on row %v\n", keyv, rows+1)
			} else if *ondupfirst {
				log.Printf("Skipping non-unique key: %v on row %v\n", keyv, rows+1)
				continue
			} else {
				log.Fatalf("Key value not unique: %v on row %v\n", keyv, rows+1)
			}
		}
		f1map[keyv] = cells
	}
	log.Printf("Number of rows in file %v:%v\n", *f1name, rows)
	f1.Close()

	log.Printf("Processing input #2:%v\n", *f2name)
	f2map := make(map[string][]string)
	// read second file
	rows = 0
	for {
		// read the csv file
		cells, rerr := r2.Read()
		if rerr == io.EOF {
			break
		}
		if rerr != nil {
			log.Fatalf("csv.Read:\n%v\n", rerr)
		}
		rows++
		keyv := cells[*key-1]
		if _, ok := f2map[keyv]; ok {
			if *onduplast {
				log.Printf("Replacing non-unique key: %v on row %v\n", keyv, rows+1)
			} else if *ondupfirst {
				log.Printf("Skipping non-unique key: %v on row %v\n", keyv, rows+1)
				continue
			} else {
				log.Fatalf("Key value not unique: %v on row %v\n", keyv, rows+1)
			}
		}
		f2map[keyv] = cells
	}
	log.Printf("Number of rows in file %v:%v\n", *f2name, rows)
	f2.Close()

	//
	// Get a combined set of keys
	//
	uniqkeyset := make(map[string]struct{})
	for k := range f1map {
		uniqkeyset[k] = struct{}{}
	}
	for k := range f2map {
		uniqkeyset[k] = struct{}{}
	}
	keySliceSize := len(uniqkeyset)
	keys := make([]string, keySliceSize)
	slot := 0
	for k := range uniqkeyset {
		keys[slot] = k
		slot++
	}
	log.Printf("Number of combined unique keys:%v\n", keySliceSize)

	// sort them
	sort.Slice(keys, func(i, j int) bool {
		return keys[i] < keys[j]
	})

	// Now range of combined unique keys
	for n := range keys {
		val := keys[n]
		row1, ok1 := f1map[val]
		row2, ok2 := f2map[val]
		if ok1 && ok2 {
			// are all the row values the same?
			diffList := make([]int, 0)
			for i := range row1 {
				if row1[i] == row2[i] {
					continue
				}
				f := i - 1
				diffList = append(diffList, f)
			}
			if len(diffList) == 0 {
				outrow1 := make([]string, 0)
				outrow1 = append(outrow1, "EQ")
				outrow1 = append(outrow1, row1...)
				err := wf1.Write(outrow1)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
			} else {
				diffs := ""
				for i := range diffList {
					diffs += fmt.Sprintf("%v,", diffList[i]+2)
				}
				diffs = strings.TrimRight(diffs, ",")
				outrow1 := make([]string, 0)
				outrow1 = append(outrow1, fmt.Sprintf("DF1=%v", diffs))
				outrow1 = append(outrow1, row1...)
				err := wf1.Write(outrow1)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
				outrow2 := make([]string, 0)
				outrow2 = append(outrow2, fmt.Sprintf("DF2=%v", diffs))
				outrow2 = append(outrow2, row2...)
				err = wf1.Write(outrow2)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
			}
		} else {
			if !ok1 {
				outrow := make([]string, 0)
				outrow = append(outrow, "IN=2")
				outrow = append(outrow, row2...)
				err := wf1.Write(outrow)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
			} else {
				outrow := make([]string, 0)
				outrow = append(outrow, "IN=1")
				outrow = append(outrow, row1...)
				err := wf1.Write(outrow)
				if err != nil {
					log.Fatalf("Output Write() Error: %v\n", err)
				}
			}
		}

	}
	wf1.Flush()

	// wrapup
	stop := time.Now()
	elapsed := time.Since(now)
	log.Printf("End: %v", stop.Format(time.StampMilli))
	log.Printf("Elapsed time %v", elapsed)

}

func usage(msg string) {
	fmt.Println(msg)
	fmt.Print("Usage: diffcsv [options]\n")
	flag.PrintDefaults()
	if msg == "" {
		fmt.Println(detailedHelp)
	}
	os.Exit(0)
}
<|endoftext|>


<commit_before>package handler

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/lalog"
	"github.com/HouzuoGuo/laitos/toolbox"
)

// UplinkMessageMetadataGateway describes a gateway that received an uplink message.
type UplinkMessageMetadataGateway struct {
	ID        string  `json:"gtw_id"`
	Timestamp int     `json:"timestamp"`
	Time      string  `json:"time"`
	Channel   int     `json:"channel"`
	RSSI      float64 `json:"rssi"`
	SNR       float64 `json:"snr"`
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
	Altitude  float64 `json:"altitude"`
}

// UplinkMessageMetadata is the metadata part of an unlink message that describes the transmission and recipient quality.
type UplinkMessageMetadata struct {
	Time                     string                         `json:"time"`
	Frequency                float64                        `json:"frequency"`
	Modulation               string                         `json:"modulation"`
	SpreadingFactorBandwidth string                         `json:"data_rate"`
	BitRate                  float64                        `json:"bit_rate"`
	CodingRate               string                         `json:"coding_rate"`
	Gateways                 []UplinkMessageMetadataGateway `json:"gateways"`
	Latitude                 float64                        `json:"latitude"`
	Longitude                float64                        `json:"longitude"`
	Altitude                 float64                        `json:"altitude"`
}

// TTNMapperPayload is TTN-Mapper compatible payload fields embedded into an uplink message.
type TTNMapperPayload struct {
	Altitude  float64 `json:"altitude"`
	HDOP      float64 `json:"hdop"`
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
}

// UplinkMessage is an uplink, TTN-Mapper compatible message transmitted by LoRA device, arrived via TTN HTTP integration.
type UplinkMessage struct {
	AppID            string                `json:"app_id"`
	DeviceID         string                `json:"dev_id"`
	DeviceEUISerial  string                `json:"hardware_serial"`
	Port             int                   `json:"port"`
	Counter          int                   `json:"counter"`
	RawPayloadBase64 string                `json:"payload_raw"`
	TTNMapperPayload TTNMapperPayload      `json:"payload_fields"`
	Metadata         UplinkMessageMetadata `json:"metadata"`
	DownlinkURL      string                `json:"downlink_url"`
}

// ReceptionComment describes a reception of TTN packet/message, the description describes the transmitter and gateway, and will be
// stored by store&forward message processor in-memory.
type ReceptionComment struct {
	DeviceID                            string
	UplinkSequenceNum                   int
	UplinkPort                          int
	Latitude, Longitude, Altitude, HDOP float64
	Frequency                           float64
	Modulation                          string
	SpreadingFactorBandwidth            string
	CodingRate                          string
	NumGateway                          int
	GatewayID                           string
	GWLatitude, GWLongitude, GWAltitude float64
	RSSI                                float64
	SNR                                 float64
	PayloadLen                          int
	Channel                             int
	TimeAtReception                     string
}

/*
HandleTheThingsNetworkHTTPIntegration collects an uplink message from TheThingsNetwork HTTP integration endpoint,
if the message carries an app command, the command will be executed by store&forward command processor, and the result
will be delivered as a downlink message.
*/
type HandleTheThingsNetworkHTTPIntegration struct {
	cmdProc *toolbox.CommandProcessor
	logger  lalog.Logger
}

func (hand *HandleTheThingsNetworkHTTPIntegration) Initialise(logger lalog.Logger, cmdProc *toolbox.CommandProcessor, _ string) error {
	if cmdProc == nil {
		return errors.New("HandleTheThingsNetworkHTTPIntegration.Initialise: command processor must not be nil")
	}
	if errs := cmdProc.IsSaneForInternet(); len(errs) > 0 {
		return fmt.Errorf("HandleTheThingsNetworkHTTPIntegration.Initialise: %+v", errs)
	}
	hand.cmdProc = cmdProc
	hand.logger = logger
	return nil
}

// DownlinkMessage is made in reply to an UplinkMessage and will be schedule for transmission to LoRA device by a gateway.
type DownlinkMessage struct {
	DeviceID         string `json:"dev_id"`
	Port             int    `json:"port"`
	Confirmed        bool   `json:"confirmed"`
	RawPayloadBase64 string `json:"payload_raw"`
}

func (msg DownlinkMessage) ToJSONString() string {
	b, err := json.Marshal(msg)
	if err != nil {
		lalog.DefaultLogger.Warning("DownlinkMessage.ToJSONString", "", err, "failed to marshal message")
		return ""
	}
	return string(b)
}

func (hand *HandleTheThingsNetworkHTTPIntegration) Handle(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return
	}
	defer func() {
		_ = r.Body.Close()
	}()
	var msg UplinkMessage
	if err := json.Unmarshal(body, &msg); err != nil || msg.AppID == "" || msg.DeviceID == "" {
		http.Error(w, "failed to decode uplink message", http.StatusInternalServerError)
		return
	}
	// Decode the raw payload sent by transmitter
	payloadBytes, err := base64.StdEncoding.DecodeString(msg.RawPayloadBase64)
	if err != nil {
		http.Error(w, "failed to decode uplink message payload", http.StatusInternalServerError)
		return
	}
	// Construct a report to save in message processor
	var firstGW UplinkMessageMetadataGateway
	if len(msg.Metadata.Gateways) > 0 {
		firstGW = msg.Metadata.Gateways[0]
	}
	hand.logger.Info("Handle", msg.DeviceEUISerial, nil,
		"received transmission from device %s, packet #%d on port %d, located at %f, %f (TTN Mapper %f, %f), received by gateway %s located at %f %f, payload size %d bytes.",
		msg.DeviceID, msg.Counter, msg.Port, msg.Metadata.Latitude, msg.Metadata.Longitude,
		msg.TTNMapperPayload.Latitude, msg.TTNMapperPayload.Longitude,
		firstGW.ID, firstGW.Latitude, firstGW.Longitude, len(payloadBytes))

	comment := ReceptionComment{
		DeviceID:                 msg.DeviceID,
		UplinkSequenceNum:        msg.Counter,
		UplinkPort:               msg.Port,
		Latitude:                 msg.TTNMapperPayload.Latitude,
		Longitude:                msg.TTNMapperPayload.Longitude,
		Altitude:                 msg.TTNMapperPayload.Altitude,
		HDOP:                     msg.TTNMapperPayload.HDOP,
		Frequency:                msg.Metadata.Frequency,
		Modulation:               msg.Metadata.Modulation,
		SpreadingFactorBandwidth: msg.Metadata.SpreadingFactorBandwidth,
		CodingRate:               msg.Metadata.CodingRate,
		NumGateway:               len(msg.Metadata.Gateways),
		GatewayID:                firstGW.ID,
		GWLatitude:               firstGW.Latitude,
		GWLongitude:              firstGW.Longitude,
		GWAltitude:               firstGW.Altitude,
		RSSI:                     firstGW.RSSI,
		SNR:                      firstGW.SNR,
		Channel:                  firstGW.Channel,
		TimeAtReception:          firstGW.Time,
	}
	report := toolbox.SubjectReportRequest{
		SubjectIP:       msg.DeviceEUISerial,
		SubjectHostName: msg.DeviceID,
		SubjectPlatform: msg.AppID,
		SubjectComment:  comment,
	}
	/*
		The first 10 bytes are decoded like this:
		(from https://github.com/kizniche/ttgo-tbeam-ttn-tracker)
		function Decoder(bytes, port) {
				var decoded = {};
				decoded.latitude = ((bytes[0]<<16)>>>0) + ((bytes[1]<<8)>>>0) + bytes[2];
				decoded.latitude = (decoded.latitude / 16777215.0 * 180) - 90;
				decoded.longitude = ((bytes[3]<<16)>>>0) + ((bytes[4]<<8)>>>0) + bytes[5];
				decoded.longitude = (decoded.longitude / 16777215.0 * 360) - 180;
				var altValue = ((bytes[6]<<8)>>>0) + bytes[7];
				var sign = bytes[6] & (1 << 7);
				if(sign) decoded.altitude = 0xFFFF0000 | altValue;
				else decoded.altitude = altValue;
				decoded.hdop = bytes[8] / 10.0;
				decoded.sats = bytes[9];
				return decoded;
		}
		After the 10th byte there comes the app command.
	*/
	if len(payloadBytes) > 10 {
		// There is an app command carried in the payload, ask store&forward message processor to execute it.
		report.CommandRequest.Command = strings.TrimSpace(string(bytes.TrimLeft(bytes.TrimRight(payloadBytes[10:], "\x00"), "\x00")))
	}
	cmdResp := hand.cmdProc.Features.MessageProcessor.StoreReport(r.Context(), report, msg.DeviceEUISerial, "httpd")
	/*
		Assume that LoRAWAN transmitter operates at SF8/125kHz (or better), at which the maximum payload size is 133 bytes across all regions.
		Among the payload, TTN uses "at least 13 bytes" for its own overhead.
		References:
		- https://docs.exploratory.engineering/lora/dr_sf/
		- https://www.thethingsnetwork.org/forum/t/limitations-data-rate-packet-size-30-seconds-uplink-and-10-messages-downlink-per-day-fair-access-policy-guidelines/1300
		Therefore, limit the downlink payload to 110 bytes, leaving 10 bytes of buffer just in case.
		Limiting command result size is usually carried out with LintText, but in this case with TTN there is an application constraint.
		Make sure the downstream message never exceeds 110 bytes, otherwise the LoRA transceiver may not get anything back.
	*/
	if result := cmdResp.CommandResponse.Result; len(result) > 110 {
		cmdResp.CommandResponse.Result = result[:110]
	}
	// Reply with app command execution result
	if len(report.CommandRequest.Command) > 10 {
		downlinkResp, err := inet.DoHTTP(r.Context(), inet.HTTPRequest{
			Method:      http.MethodPost,
			ContentType: "application/json",
			Body: strings.NewReader(DownlinkMessage{
				DeviceID:         msg.DeviceID,
				Port:             msg.Port,
				Confirmed:        false,
				RawPayloadBase64: base64.StdEncoding.EncodeToString([]byte(cmdResp.CommandResponse.Result)),
			}.ToJSONString()),
		}, strings.Replace(msg.DownlinkURL, "%", "%%", -1))
		if err != nil {
			err = downlinkResp.Non2xxToError()
		}
		if err != nil {
			hand.logger.Warning("HandleTheThingsNetworkHTTPIntegration.Handler", GetRealClientIP(r), err, "failed to send downlink reply message")
		}
	}
	w.WriteHeader(http.StatusOK)
}

func (_ *HandleTheThingsNetworkHTTPIntegration) GetRateLimitFactor() int {
	return 6
}

func (_ *HandleTheThingsNetworkHTTPIntegration) SelfTest() error {
	return nil
}
<commit_msg>remove unused attribute PayloadLen from struct ReceptionComment<commit_after>package handler

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/lalog"
	"github.com/HouzuoGuo/laitos/toolbox"
)

// UplinkMessageMetadataGateway describes a gateway that received an uplink message.
type UplinkMessageMetadataGateway struct {
	ID        string  `json:"gtw_id"`
	Timestamp int     `json:"timestamp"`
	Time      string  `json:"time"`
	Channel   int     `json:"channel"`
	RSSI      float64 `json:"rssi"`
	SNR       float64 `json:"snr"`
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
	Altitude  float64 `json:"altitude"`
}

// UplinkMessageMetadata is the metadata part of an unlink message that describes the transmission and recipient quality.
type UplinkMessageMetadata struct {
	Time                     string                         `json:"time"`
	Frequency                float64                        `json:"frequency"`
	Modulation               string                         `json:"modulation"`
	SpreadingFactorBandwidth string                         `json:"data_rate"`
	BitRate                  float64                        `json:"bit_rate"`
	CodingRate               string                         `json:"coding_rate"`
	Gateways                 []UplinkMessageMetadataGateway `json:"gateways"`
	Latitude                 float64                        `json:"latitude"`
	Longitude                float64                        `json:"longitude"`
	Altitude                 float64                        `json:"altitude"`
}

// TTNMapperPayload is TTN-Mapper compatible payload fields embedded into an uplink message.
type TTNMapperPayload struct {
	Altitude  float64 `json:"altitude"`
	HDOP      float64 `json:"hdop"`
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
}

// UplinkMessage is an uplink, TTN-Mapper compatible message transmitted by LoRA device, arrived via TTN HTTP integration.
type UplinkMessage struct {
	AppID            string                `json:"app_id"`
	DeviceID         string                `json:"dev_id"`
	DeviceEUISerial  string                `json:"hardware_serial"`
	Port             int                   `json:"port"`
	Counter          int                   `json:"counter"`
	RawPayloadBase64 string                `json:"payload_raw"`
	TTNMapperPayload TTNMapperPayload      `json:"payload_fields"`
	Metadata         UplinkMessageMetadata `json:"metadata"`
	DownlinkURL      string                `json:"downlink_url"`
}

// ReceptionComment describes a reception of TTN packet/message, the description describes the transmitter and gateway, and will be
// stored by store&forward message processor in-memory.
type ReceptionComment struct {
	DeviceID                            string
	UplinkSequenceNum                   int
	UplinkPort                          int
	Latitude, Longitude, Altitude, HDOP float64
	Frequency                           float64
	Modulation                          string
	SpreadingFactorBandwidth            string
	CodingRate                          string
	NumGateway                          int
	GatewayID                           string
	GWLatitude, GWLongitude, GWAltitude float64
	RSSI                                float64
	SNR                                 float64
	Channel                             int
	TimeAtReception                     string
}

/*
HandleTheThingsNetworkHTTPIntegration collects an uplink message from TheThingsNetwork HTTP integration endpoint,
if the message carries an app command, the command will be executed by store&forward command processor, and the result
will be delivered as a downlink message.
*/
type HandleTheThingsNetworkHTTPIntegration struct {
	cmdProc *toolbox.CommandProcessor
	logger  lalog.Logger
}

func (hand *HandleTheThingsNetworkHTTPIntegration) Initialise(logger lalog.Logger, cmdProc *toolbox.CommandProcessor, _ string) error {
	if cmdProc == nil {
		return errors.New("HandleTheThingsNetworkHTTPIntegration.Initialise: command processor must not be nil")
	}
	if errs := cmdProc.IsSaneForInternet(); len(errs) > 0 {
		return fmt.Errorf("HandleTheThingsNetworkHTTPIntegration.Initialise: %+v", errs)
	}
	hand.cmdProc = cmdProc
	hand.logger = logger
	return nil
}

// DownlinkMessage is made in reply to an UplinkMessage and will be schedule for transmission to LoRA device by a gateway.
type DownlinkMessage struct {
	DeviceID         string `json:"dev_id"`
	Port             int    `json:"port"`
	Confirmed        bool   `json:"confirmed"`
	RawPayloadBase64 string `json:"payload_raw"`
}

func (msg DownlinkMessage) ToJSONString() string {
	b, err := json.Marshal(msg)
	if err != nil {
		lalog.DefaultLogger.Warning("DownlinkMessage.ToJSONString", "", err, "failed to marshal message")
		return ""
	}
	return string(b)
}

func (hand *HandleTheThingsNetworkHTTPIntegration) Handle(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return
	}
	defer func() {
		_ = r.Body.Close()
	}()
	var msg UplinkMessage
	if err := json.Unmarshal(body, &msg); err != nil || msg.AppID == "" || msg.DeviceID == "" {
		http.Error(w, "failed to decode uplink message", http.StatusInternalServerError)
		return
	}
	// Decode the raw payload sent by transmitter
	payloadBytes, err := base64.StdEncoding.DecodeString(msg.RawPayloadBase64)
	if err != nil {
		http.Error(w, "failed to decode uplink message payload", http.StatusInternalServerError)
		return
	}
	// Construct a report to save in message processor
	var firstGW UplinkMessageMetadataGateway
	if len(msg.Metadata.Gateways) > 0 {
		firstGW = msg.Metadata.Gateways[0]
	}
	hand.logger.Info("Handle", msg.DeviceEUISerial, nil,
		"received transmission from device %s, packet #%d on port %d, located at %f, %f (TTN Mapper %f, %f), received by gateway %s located at %f %f, payload size %d bytes.",
		msg.DeviceID, msg.Counter, msg.Port, msg.Metadata.Latitude, msg.Metadata.Longitude,
		msg.TTNMapperPayload.Latitude, msg.TTNMapperPayload.Longitude,
		firstGW.ID, firstGW.Latitude, firstGW.Longitude, len(payloadBytes))

	comment := ReceptionComment{
		DeviceID:                 msg.DeviceID,
		UplinkSequenceNum:        msg.Counter,
		UplinkPort:               msg.Port,
		Latitude:                 msg.TTNMapperPayload.Latitude,
		Longitude:                msg.TTNMapperPayload.Longitude,
		Altitude:                 msg.TTNMapperPayload.Altitude,
		HDOP:                     msg.TTNMapperPayload.HDOP,
		Frequency:                msg.Metadata.Frequency,
		Modulation:               msg.Metadata.Modulation,
		SpreadingFactorBandwidth: msg.Metadata.SpreadingFactorBandwidth,
		CodingRate:               msg.Metadata.CodingRate,
		NumGateway:               len(msg.Metadata.Gateways),
		GatewayID:                firstGW.ID,
		GWLatitude:               firstGW.Latitude,
		GWLongitude:              firstGW.Longitude,
		GWAltitude:               firstGW.Altitude,
		RSSI:                     firstGW.RSSI,
		SNR:                      firstGW.SNR,
		Channel:                  firstGW.Channel,
		TimeAtReception:          firstGW.Time,
	}
	report := toolbox.SubjectReportRequest{
		SubjectIP:       msg.DeviceEUISerial,
		SubjectHostName: msg.DeviceID,
		SubjectPlatform: msg.AppID,
		SubjectComment:  comment,
	}
	/*
		The first 10 bytes are decoded like this:
		(from https://github.com/kizniche/ttgo-tbeam-ttn-tracker)
		function Decoder(bytes, port) {
				var decoded = {};
				decoded.latitude = ((bytes[0]<<16)>>>0) + ((bytes[1]<<8)>>>0) + bytes[2];
				decoded.latitude = (decoded.latitude / 16777215.0 * 180) - 90;
				decoded.longitude = ((bytes[3]<<16)>>>0) + ((bytes[4]<<8)>>>0) + bytes[5];
				decoded.longitude = (decoded.longitude / 16777215.0 * 360) - 180;
				var altValue = ((bytes[6]<<8)>>>0) + bytes[7];
				var sign = bytes[6] & (1 << 7);
				if(sign) decoded.altitude = 0xFFFF0000 | altValue;
				else decoded.altitude = altValue;
				decoded.hdop = bytes[8] / 10.0;
				decoded.sats = bytes[9];
				return decoded;
		}
		After the 10th byte there comes the app command.
	*/
	if len(payloadBytes) > 10 {
		// There is an app command carried in the payload, ask store&forward message processor to execute it.
		report.CommandRequest.Command = strings.TrimSpace(string(bytes.TrimLeft(bytes.TrimRight(payloadBytes[10:], "\x00"), "\x00")))
	}
	cmdResp := hand.cmdProc.Features.MessageProcessor.StoreReport(r.Context(), report, msg.DeviceEUISerial, "httpd")
	/*
		Assume that LoRAWAN transmitter operates at SF8/125kHz (or better), at which the maximum payload size is 133 bytes across all regions.
		Among the payload, TTN uses "at least 13 bytes" for its own overhead.
		References:
		- https://docs.exploratory.engineering/lora/dr_sf/
		- https://www.thethingsnetwork.org/forum/t/limitations-data-rate-packet-size-30-seconds-uplink-and-10-messages-downlink-per-day-fair-access-policy-guidelines/1300
		Therefore, limit the downlink payload to 110 bytes, leaving 10 bytes of buffer just in case.
		Limiting command result size is usually carried out with LintText, but in this case with TTN there is an application constraint.
		Make sure the downstream message never exceeds 110 bytes, otherwise the LoRA transceiver may not get anything back.
	*/
	if result := cmdResp.CommandResponse.Result; len(result) > 110 {
		cmdResp.CommandResponse.Result = result[:110]
	}
	// Reply with app command execution result
	if len(report.CommandRequest.Command) > 10 {
		downlinkResp, err := inet.DoHTTP(r.Context(), inet.HTTPRequest{
			Method:      http.MethodPost,
			ContentType: "application/json",
			Body: strings.NewReader(DownlinkMessage{
				DeviceID:         msg.DeviceID,
				Port:             msg.Port,
				Confirmed:        false,
				RawPayloadBase64: base64.StdEncoding.EncodeToString([]byte(cmdResp.CommandResponse.Result)),
			}.ToJSONString()),
		}, strings.Replace(msg.DownlinkURL, "%", "%%", -1))
		if err != nil {
			err = downlinkResp.Non2xxToError()
		}
		if err != nil {
			hand.logger.Warning("HandleTheThingsNetworkHTTPIntegration.Handler", GetRealClientIP(r), err, "failed to send downlink reply message")
		}
	}
	w.WriteHeader(http.StatusOK)
}

func (_ *HandleTheThingsNetworkHTTPIntegration) GetRateLimitFactor() int {
	return 6
}

func (_ *HandleTheThingsNetworkHTTPIntegration) SelfTest() error {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"github.com/google/jsonapi"
)

// Blog is a model representing a blog site
type Blog struct {
	ID            int       `jsonapi:"primary,blogs"`
	Title         string    `jsonapi:"attr,title"`
	Posts         []*Post   `jsonapi:"relation,posts"`
	CurrentPost   *Post     `jsonapi:"relation,current_post"`
	CurrentPostID int       `jsonapi:"attr,current_post_id"`
	CreatedAt     time.Time `jsonapi:"attr,created_at"`
	ViewCount     int       `jsonapi:"attr,view_count"`
}

// Post is a model representing a post on a blog
type Post struct {
	ID       int        `jsonapi:"primary,posts"`
	BlogID   int        `jsonapi:"attr,blog_id"`
	Title    string     `jsonapi:"attr,title"`
	Body     string     `jsonapi:"attr,body"`
	Comments []*Comment `jsonapi:"relation,comments"`
}

// Comment is a model representing a user submitted comment
type Comment struct {
	ID     int    `jsonapi:"primary,comments"`
	PostID int    `jsonapi:"attr,post_id"`
	Body   string `jsonapi:"attr,body"`
}

// JSONAPILinks implements the Linkable interface for a blog
func (blog Blog) JSONAPILinks() *jsonapi.Links {
	return &jsonapi.Links{
		"self": fmt.Sprintf("https://example.com/blogs/%d", blog.ID),
	}
}

// JSONAPIRelationshipLinks implements the RelationshipLinkable interface for a blog
func (blog Blog) JSONAPIRelationshipLinks(relation string) *jsonapi.Links {
	if relation == "posts" {
		return &jsonapi.Links{
			"related": fmt.Sprintf("https://example.com/blogs/%d/posts", blog.ID),
		}
	}
	if relation == "current_post" {
		return &jsonapi.Links{
			"related": fmt.Sprintf("https://example.com/blogs/%d/current_post", blog.ID),
		}
	}
	return nil
}

// JSONAPIMeta implements the Metable interface for a blog
func (blog Blog) JSONAPIMeta() *jsonapi.Meta {
	return &jsonapi.Meta{
		"detail": "extra details regarding the blog",
	}
}

// JSONAPIRelationshipLinks implements the RelationshipMetable interface for a blog
func (blog Blog) JSONAPIRelationshipMeta(relation string) *jsonapi.Meta {
	if relation == "posts" {
		return &jsonapi.Meta{
			"detail": "posts meta information",
		}
	}
	if relation == "current_post" {
		return &jsonapi.Meta{
			"detail": "current post meta information",
		}
	}
	return nil
}
<commit_msg>Fix comment.<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/google/jsonapi"
)

// Blog is a model representing a blog site
type Blog struct {
	ID            int       `jsonapi:"primary,blogs"`
	Title         string    `jsonapi:"attr,title"`
	Posts         []*Post   `jsonapi:"relation,posts"`
	CurrentPost   *Post     `jsonapi:"relation,current_post"`
	CurrentPostID int       `jsonapi:"attr,current_post_id"`
	CreatedAt     time.Time `jsonapi:"attr,created_at"`
	ViewCount     int       `jsonapi:"attr,view_count"`
}

// Post is a model representing a post on a blog
type Post struct {
	ID       int        `jsonapi:"primary,posts"`
	BlogID   int        `jsonapi:"attr,blog_id"`
	Title    string     `jsonapi:"attr,title"`
	Body     string     `jsonapi:"attr,body"`
	Comments []*Comment `jsonapi:"relation,comments"`
}

// Comment is a model representing a user submitted comment
type Comment struct {
	ID     int    `jsonapi:"primary,comments"`
	PostID int    `jsonapi:"attr,post_id"`
	Body   string `jsonapi:"attr,body"`
}

// JSONAPILinks implements the Linkable interface for a blog
func (blog Blog) JSONAPILinks() *jsonapi.Links {
	return &jsonapi.Links{
		"self": fmt.Sprintf("https://example.com/blogs/%d", blog.ID),
	}
}

// JSONAPIRelationshipLinks implements the RelationshipLinkable interface for a blog
func (blog Blog) JSONAPIRelationshipLinks(relation string) *jsonapi.Links {
	if relation == "posts" {
		return &jsonapi.Links{
			"related": fmt.Sprintf("https://example.com/blogs/%d/posts", blog.ID),
		}
	}
	if relation == "current_post" {
		return &jsonapi.Links{
			"related": fmt.Sprintf("https://example.com/blogs/%d/current_post", blog.ID),
		}
	}
	return nil
}

// JSONAPIMeta implements the Metable interface for a blog
func (blog Blog) JSONAPIMeta() *jsonapi.Meta {
	return &jsonapi.Meta{
		"detail": "extra details regarding the blog",
	}
}

// JSONAPIRelationshipMeta implements the RelationshipMetable interface for a blog
func (blog Blog) JSONAPIRelationshipMeta(relation string) *jsonapi.Meta {
	if relation == "posts" {
		return &jsonapi.Meta{
			"detail": "posts meta information",
		}
	}
	if relation == "current_post" {
		return &jsonapi.Meta{
			"detail": "current post meta information",
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package maintenance

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sync"
)

// Values for HTTP Contet-Type header.
var (
	HTMLContentType = "text/html; charset=utf-8"
	TextContentType = "text/text; charset=utf-8"
	JSONContentType = "application/json; charset=utf-8"
)

// Logger defines methods required for logging.
type Logger interface {
	Infof(format string, a ...interface{})
	Errorf(format string, a ...interface{})
}

// stdLogger is a simple implementation of Logger interface
// that uses log package for logging messages.
type stdLogger struct{}

func (l stdLogger) Infof(format string, a ...interface{}) {
	log.Printf("INFO "+format, a...)
}

func (l stdLogger) Errorf(format string, a ...interface{}) {
	log.Printf("ERROR "+format, a...)
}

// Store defines methods that are reqired to check, set and remove
// information wheather the maintenance is on of off.
// Usually only one boolean value is needed to be stored
type Store interface {
	// Return true if maintenance is enabled.
	Status() (on bool, err error)
	// Enable maintenance and returns true if the state has changed.
	On() (changed bool, err error)
	// Disables maintenance and returns true if the state has changed.
	Off() (changed bool, err error)
}

// MemoryStore implements Store that keeps data in memory.
type MemoryStore struct {
	on bool
	mu sync.Mutex
}

// NewMemoryStore creates a new instance of MemoryStore.
func NewMemoryStore() *MemoryStore {
	return &MemoryStore{}
}

// Status returns true if maintenance is enabled.
func (s *MemoryStore) Status() (on bool, err error) {
	s.mu.Lock()
	on = s.on
	s.mu.Unlock()
	return
}

// On enables maintenance.
func (s *MemoryStore) On() (changed bool, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.on {
		return
	}
	s.on = true
	changed = true
	return
}

// Off disables maintenance.
func (s *MemoryStore) Off() (changed bool, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if !s.on {
		return
	}
	s.on = false
	changed = true
	return
}

// FileStore implements Store that manages maintenance
// status by existence of a specific file. If file exists
// maintenance is enabled, otherwise is disabled.
// This store persists maintenance state and provides
// a simple way to set maintenance on local filesystem
// with external tools.
type FileStore struct {
	filename string
}

// NewFileStore creates a new instance of FileStore.
func NewFileStore(filename string) *FileStore {
	return &FileStore{
		filename: filename,
	}
}

// Status returns true if maintenance is enabled.
func (s *FileStore) Status() (on bool, err error) {
	_, err = os.Stat(s.filename)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// On enables maintenance.
func (s *FileStore) On() (changed bool, err error) {
	if _, err = os.Stat(s.filename); err == nil {
		return
	}
	err = os.MkdirAll(filepath.Dir(s.filename), 0777)
	if err != nil {
		return
	}
	f, err := os.Create(s.filename)
	if err != nil {
		return
	}
	f.Close()
	changed = true
	return
}

// Off disables maintenance.
func (s *FileStore) Off() (changed bool, err error) {
	_, err = os.Stat(s.filename)
	if os.IsNotExist(err) {
		err = nil
		return
	}
	if err != nil {
		return
	}
	if err = os.Remove(s.filename); err != nil {
		return
	}
	changed = true
	return
}

// Response holds configuration for HTTP response
// during maintenance mode.
type Response struct {
	// Body will be returned if Handler is nil.
	Body    string
	Handler http.Handler
}

// Service implements http.Service interface to write a custom
// HTTP response during maintenance mode.
// It also provides JSON API handlers that can be used to
// check, set and remove maintenance mode.
type Service struct {
	HTML Response
	JSON Response
	Text Response

	store  Store
	logger Logger
}

// Option is a function that sets optional parameters to the Handler.
type Option func(*Service)

// WithStore sets Store to the Handler. If this option
// is not used, handler defaults to MemoryStore.
func WithStore(store Store) Option { return func(o *Service) { o.store = store } }

// WithLogger sets the function that will perform message logging.
// Default is log.Printf.
func WithLogger(logger Logger) Option { return func(o *Service) { o.logger = logger } }

// New creates a new instance of Handler.
// The first argument is the handler that will be executed
// when maintenance mode is off.
func New(options ...Option) (s *Service) {
	s = &Service{
		logger: stdLogger{},
	}
	for _, option := range options {
		option(s)
	}
	if s.store == nil {
		s.store = NewMemoryStore()
	}
	return
}

// HTMLHandler is a HTTP middleware that should be used
// alongide HTML pages.
func (s Service) HTMLHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		on, err := s.store.Status()
		if err != nil {
			s.logger.Errorf("maintenance status: %v", err)
		}
		if on || err != nil {
			if s.HTML.Handler != nil {
				s.HTML.Handler.ServeHTTP(w, r)
				return
			}
			w.Header().Set("Content-Type", HTMLContentType)
			w.WriteHeader(http.StatusServiceUnavailable)
			fmt.Fprintln(w, s.HTML.Body)
			return
		}
		h.ServeHTTP(w, r)
	})
}

// JSONHandler is a HTTP middleware that should be used
// alongide JSON-encoded responses.
func (s Service) JSONHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		on, err := s.store.Status()
		if err != nil {
			s.logger.Errorf("maintenance status: %v", err)
		}
		if on || err != nil {
			if s.JSON.Handler != nil {
				s.JSON.Handler.ServeHTTP(w, r)
				return
			}
			w.Header().Set("Content-Type", JSONContentType)
			w.WriteHeader(http.StatusServiceUnavailable)
			fmt.Fprintln(w, s.JSON.Body)
			return
		}
		h.ServeHTTP(w, r)
	})
}

// TextHandler is a HTTP middleware that should be used
// alongide plaintext responses.
func (s Service) TextHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		on, err := s.store.Status()
		if err != nil {
			s.logger.Errorf("maintenance status: %v", err)
		}
		if on || err != nil {
			if s.Text.Handler != nil {
				s.Text.Handler.ServeHTTP(w, r)
				return
			}
			w.Header().Set("Content-Type", TextContentType)
			w.WriteHeader(http.StatusServiceUnavailable)
			fmt.Fprintln(w, s.Text.Body)
			return
		}
		h.ServeHTTP(w, r)
	})
}

// StatusHandler can be used in JSON-encoded HTTP API
// to check the status of maintenance.
func (s Service) StatusHandler(w http.ResponseWriter, r *http.Request) {
	on, err := s.store.Status()
	if err != nil {
		s.logger.Errorf("maintenance status: %s", err)
		jsonInternalServerErrorResponse(w)
		return
	}
	jsonStatusResponse(w, on)
}

// OnHandler can be used in JSON-encoded HTTP API to enable maintenance.
// It returns HTTP Status Created if the maintenance is enabled.
// If the maintenance is already enabled, it returns HTTP Status OK.
func (s Service) OnHandler(w http.ResponseWriter, r *http.Request) {
	changed, err := s.store.On()
	if err != nil {
		s.logger.Errorf("maintenance on: %s", err)
		jsonInternalServerErrorResponse(w)
		return
	}
	if changed {
		s.logger.Infof("maintenance on")
		jsonCreatedResponse(w)
		return
	}
	jsonOKResponse(w)
}

// OffHandler can be used in JSON-encoded HTTP API to disable maintenance.
func (s Service) OffHandler(w http.ResponseWriter, r *http.Request) {
	changed, err := s.store.Off()
	if err != nil {
		s.logger.Errorf("maintenance off: %s", err)
		jsonInternalServerErrorResponse(w)
		return
	}
	if changed {
		s.logger.Infof("maintenance off")
	}
	jsonOKResponse(w)
}

func jsonOKResponse(w http.ResponseWriter) {
	w.Header().Set("Content-Type", JSONContentType)
	w.WriteHeader(http.StatusOK)
	fmt.Fprintln(w, `{"message":"OK","code":200}`)
}

func jsonCreatedResponse(w http.ResponseWriter) {
	w.Header().Set("Content-Type", JSONContentType)
	w.WriteHeader(http.StatusCreated)
	fmt.Fprintln(w, `{"message":"Created","code":201}`)
}

func jsonInternalServerErrorResponse(w http.ResponseWriter) {
	w.Header().Set("Content-Type", JSONContentType)
	w.WriteHeader(http.StatusInternalServerError)
	fmt.Fprintln(w, `{"message":"Internal Server Error","code":500}`)
}

func jsonStatusResponse(w http.ResponseWriter, on bool) {
	w.Header().Set("Content-Type", JSONContentType)
	w.WriteHeader(http.StatusOK)
	if on {
		fmt.Fprintln(w, `{"status":"on"}`)
	} else {
		fmt.Fprintln(w, `{"status":"off"}`)
	}
}
<commit_msg>maintenance: expose the Status method<commit_after>// Copyright (c) 2017, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package maintenance

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sync"
)

// Values for HTTP Contet-Type header.
var (
	HTMLContentType = "text/html; charset=utf-8"
	TextContentType = "text/text; charset=utf-8"
	JSONContentType = "application/json; charset=utf-8"
)

// Logger defines methods required for logging.
type Logger interface {
	Infof(format string, a ...interface{})
	Errorf(format string, a ...interface{})
}

// stdLogger is a simple implementation of Logger interface
// that uses log package for logging messages.
type stdLogger struct{}

func (l stdLogger) Infof(format string, a ...interface{}) {
	log.Printf("INFO "+format, a...)
}

func (l stdLogger) Errorf(format string, a ...interface{}) {
	log.Printf("ERROR "+format, a...)
}

// Store defines methods that are reqired to check, set and remove
// information wheather the maintenance is on of off.
// Usually only one boolean value is needed to be stored
type Store interface {
	// Return true if maintenance is enabled.
	Status() (on bool, err error)
	// Enable maintenance and returns true if the state has changed.
	On() (changed bool, err error)
	// Disables maintenance and returns true if the state has changed.
	Off() (changed bool, err error)
}

// MemoryStore implements Store that keeps data in memory.
type MemoryStore struct {
	on bool
	mu sync.Mutex
}

// NewMemoryStore creates a new instance of MemoryStore.
func NewMemoryStore() *MemoryStore {
	return &MemoryStore{}
}

// Status returns true if maintenance is enabled.
func (s *MemoryStore) Status() (on bool, err error) {
	s.mu.Lock()
	on = s.on
	s.mu.Unlock()
	return
}

// On enables maintenance.
func (s *MemoryStore) On() (changed bool, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.on {
		return
	}
	s.on = true
	changed = true
	return
}

// Off disables maintenance.
func (s *MemoryStore) Off() (changed bool, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if !s.on {
		return
	}
	s.on = false
	changed = true
	return
}

// FileStore implements Store that manages maintenance
// status by existence of a specific file. If file exists
// maintenance is enabled, otherwise is disabled.
// This store persists maintenance state and provides
// a simple way to set maintenance on local filesystem
// with external tools.
type FileStore struct {
	filename string
}

// NewFileStore creates a new instance of FileStore.
func NewFileStore(filename string) *FileStore {
	return &FileStore{
		filename: filename,
	}
}

// Status returns true if maintenance is enabled.
func (s *FileStore) Status() (on bool, err error) {
	_, err = os.Stat(s.filename)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// On enables maintenance.
func (s *FileStore) On() (changed bool, err error) {
	if _, err = os.Stat(s.filename); err == nil {
		return
	}
	err = os.MkdirAll(filepath.Dir(s.filename), 0777)
	if err != nil {
		return
	}
	f, err := os.Create(s.filename)
	if err != nil {
		return
	}
	f.Close()
	changed = true
	return
}

// Off disables maintenance.
func (s *FileStore) Off() (changed bool, err error) {
	_, err = os.Stat(s.filename)
	if os.IsNotExist(err) {
		err = nil
		return
	}
	if err != nil {
		return
	}
	if err = os.Remove(s.filename); err != nil {
		return
	}
	changed = true
	return
}

// Response holds configuration for HTTP response
// during maintenance mode.
type Response struct {
	// Body will be returned if Handler is nil.
	Body    string
	Handler http.Handler
}

// Service implements http.Service interface to write a custom
// HTTP response during maintenance mode.
// It also provides JSON API handlers that can be used to
// check, set and remove maintenance mode.
type Service struct {
	HTML Response
	JSON Response
	Text Response

	store  Store
	logger Logger
}

// Option is a function that sets optional parameters to the Handler.
type Option func(*Service)

// WithStore sets Store to the Handler. If this option
// is not used, handler defaults to MemoryStore.
func WithStore(store Store) Option { return func(o *Service) { o.store = store } }

// WithLogger sets the function that will perform message logging.
// Default is log.Printf.
func WithLogger(logger Logger) Option { return func(o *Service) { o.logger = logger } }

// New creates a new instance of Handler.
// The first argument is the handler that will be executed
// when maintenance mode is off.
func New(options ...Option) (s *Service) {
	s = &Service{
		logger: stdLogger{},
	}
	for _, option := range options {
		option(s)
	}
	if s.store == nil {
		s.store = NewMemoryStore()
	}
	return
}

// HTMLHandler is a HTTP middleware that should be used
// alongide HTML pages.
func (s Service) HTMLHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		on, err := s.store.Status()
		if err != nil {
			s.logger.Errorf("maintenance status: %v", err)
		}
		if on || err != nil {
			if s.HTML.Handler != nil {
				s.HTML.Handler.ServeHTTP(w, r)
				return
			}
			w.Header().Set("Content-Type", HTMLContentType)
			w.WriteHeader(http.StatusServiceUnavailable)
			fmt.Fprintln(w, s.HTML.Body)
			return
		}
		h.ServeHTTP(w, r)
	})
}

// JSONHandler is a HTTP middleware that should be used
// alongide JSON-encoded responses.
func (s Service) JSONHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		on, err := s.store.Status()
		if err != nil {
			s.logger.Errorf("maintenance status: %v", err)
		}
		if on || err != nil {
			if s.JSON.Handler != nil {
				s.JSON.Handler.ServeHTTP(w, r)
				return
			}
			w.Header().Set("Content-Type", JSONContentType)
			w.WriteHeader(http.StatusServiceUnavailable)
			fmt.Fprintln(w, s.JSON.Body)
			return
		}
		h.ServeHTTP(w, r)
	})
}

// TextHandler is a HTTP middleware that should be used
// alongide plaintext responses.
func (s Service) TextHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		on, err := s.store.Status()
		if err != nil {
			s.logger.Errorf("maintenance status: %v", err)
		}
		if on || err != nil {
			if s.Text.Handler != nil {
				s.Text.Handler.ServeHTTP(w, r)
				return
			}
			w.Header().Set("Content-Type", TextContentType)
			w.WriteHeader(http.StatusServiceUnavailable)
			fmt.Fprintln(w, s.Text.Body)
			return
		}
		h.ServeHTTP(w, r)
	})
}

// Status returns whether the maintenance mode is enabled.
func (s Service) Status() (on bool, err error) {
	return s.store.Status()
}

// StatusHandler can be used in JSON-encoded HTTP API
// to check the status of maintenance.
func (s Service) StatusHandler(w http.ResponseWriter, r *http.Request) {
	on, err := s.store.Status()
	if err != nil {
		s.logger.Errorf("maintenance status: %s", err)
		jsonInternalServerErrorResponse(w)
		return
	}
	jsonStatusResponse(w, on)
}

// OnHandler can be used in JSON-encoded HTTP API to enable maintenance.
// It returns HTTP Status Created if the maintenance is enabled.
// If the maintenance is already enabled, it returns HTTP Status OK.
func (s Service) OnHandler(w http.ResponseWriter, r *http.Request) {
	changed, err := s.store.On()
	if err != nil {
		s.logger.Errorf("maintenance on: %s", err)
		jsonInternalServerErrorResponse(w)
		return
	}
	if changed {
		s.logger.Infof("maintenance on")
		jsonCreatedResponse(w)
		return
	}
	jsonOKResponse(w)
}

// OffHandler can be used in JSON-encoded HTTP API to disable maintenance.
func (s Service) OffHandler(w http.ResponseWriter, r *http.Request) {
	changed, err := s.store.Off()
	if err != nil {
		s.logger.Errorf("maintenance off: %s", err)
		jsonInternalServerErrorResponse(w)
		return
	}
	if changed {
		s.logger.Infof("maintenance off")
	}
	jsonOKResponse(w)
}

func jsonOKResponse(w http.ResponseWriter) {
	w.Header().Set("Content-Type", JSONContentType)
	w.WriteHeader(http.StatusOK)
	fmt.Fprintln(w, `{"message":"OK","code":200}`)
}

func jsonCreatedResponse(w http.ResponseWriter) {
	w.Header().Set("Content-Type", JSONContentType)
	w.WriteHeader(http.StatusCreated)
	fmt.Fprintln(w, `{"message":"Created","code":201}`)
}

func jsonInternalServerErrorResponse(w http.ResponseWriter) {
	w.Header().Set("Content-Type", JSONContentType)
	w.WriteHeader(http.StatusInternalServerError)
	fmt.Fprintln(w, `{"message":"Internal Server Error","code":500}`)
}

func jsonStatusResponse(w http.ResponseWriter, on bool) {
	w.Header().Set("Content-Type", JSONContentType)
	w.WriteHeader(http.StatusOK)
	if on {
		fmt.Fprintln(w, `{"status":"on"}`)
	} else {
		fmt.Fprintln(w, `{"status":"off"}`)
	}
}
<|endoftext|>


<commit_before>package masterapi

import (
	"encoding/json"
	"fmt"
	"github.com/KIT-MAMID/mamid/model"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"strconv"
)

type Slave struct {
	ID                   uint   `json:"id"`
	Hostname             string `json:"hostname"`
	Port                 uint   `json:"slave_port"`
	MongodPortRangeBegin uint   `json:"mongod_port_range_begin"` //inclusive
	MongodPortRangeEnd   uint   `json:"mongod_port_range_end"`   //exclusive
	PersistentStorage    bool   `json:"persistent_storage"`
	ConfiguredState      string `json:"state"`
}

func (m *MasterAPI) SlaveIndex(w http.ResponseWriter, r *http.Request) {

	var slaves []model.Slave
	err := m.DB.Find(&slaves).Error
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}
	json.NewEncoder(w).Encode(slaves)
}

func (m *MasterAPI) SlaveById(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id64, err := strconv.ParseUint(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	id := uint(id64)

	var slaves []model.Slave
	err = m.DB.Find(&slaves, &model.Slave{ID: id}).Error

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}
	if len(slaves) == 0 { // Not found?
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if len(slaves) > 1 {
		log.Printf("inconsistency: multiple slaves for slave.ID = %d found in database", len(slaves))
	}
	json.NewEncoder(w).Encode(ProjectModelSlaveToSlave(&slaves[0]))
	return
}

func (m *MasterAPI) SlavePut(w http.ResponseWriter, r *http.Request) {
	var postSlave Slave
	err := json.NewDecoder(r.Body).Decode(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "cannot parse object (%s)", err.Error())
		return
	}

	// Validation

	if postSlave.ID != 0 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not change the slave ID in PUT request")
		return
	}

	modelSlave, err := ProjectSlaveToModelSlave(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, err.Error())
		return
	}

	// Persist to database

	err = m.DB.Create(modelSlave).Error
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}

	// TODO set location header. Would it be better to return the ID? YES.

	return
}

func (m *MasterAPI) SlaveUpdate(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id64, err := strconv.ParseUint(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	id := uint(id64)

	var postSlave Slave
	err = json.NewDecoder(r.Body).Decode(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "cannot parse object (%s)", err.Error())
		return
	}

	// Validation

	if postSlave.ID != id {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not change the id of an object")
		return
	}

	if err = postSlave.assertNoZeroFieldsSet(); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not POST JSON with zero values in any field: %s", err.Error())
		return
	}

	modelSlave, err := ProjectSlaveToModelSlave(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, err)
		return
	}

	// Persist to database

	m.DB.Model(&modelSlave).Updates(&modelSlave)
}

func (m *MasterAPI) SlaveDelete(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id64, err := strconv.ParseUint(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	id := uint(id64)

	s := m.DB.Delete(&model.Slave{ID: id})
	if s.Error != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}
	if s.RowsAffected == 0 {
		w.WriteHeader(http.StatusNotFound)
	}

	if s.RowsAffected > 1 {
		log.Printf("inconsistency: slave DELETE affected more than one row. Slave.ID = %v", id)
	}
}
<commit_msg>FIX: masterapi: order slaves by id.<commit_after>package masterapi

import (
	"encoding/json"
	"fmt"
	"github.com/KIT-MAMID/mamid/model"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"strconv"
)

type Slave struct {
	ID                   uint   `json:"id"`
	Hostname             string `json:"hostname"`
	Port                 uint   `json:"slave_port"`
	MongodPortRangeBegin uint   `json:"mongod_port_range_begin"` //inclusive
	MongodPortRangeEnd   uint   `json:"mongod_port_range_end"`   //exclusive
	PersistentStorage    bool   `json:"persistent_storage"`
	ConfiguredState      string `json:"state"`
}

func (m *MasterAPI) SlaveIndex(w http.ResponseWriter, r *http.Request) {

	var slaves []model.Slave
	err := m.DB.Order("id", false).Find(&slaves).Error
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}
	json.NewEncoder(w).Encode(slaves)
}

func (m *MasterAPI) SlaveById(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id64, err := strconv.ParseUint(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	id := uint(id64)

	var slaves []model.Slave
	err = m.DB.Find(&slaves, &model.Slave{ID: id}).Error

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}
	if len(slaves) == 0 { // Not found?
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if len(slaves) > 1 {
		log.Printf("inconsistency: multiple slaves for slave.ID = %d found in database", len(slaves))
	}
	json.NewEncoder(w).Encode(ProjectModelSlaveToSlave(&slaves[0]))
	return
}

func (m *MasterAPI) SlavePut(w http.ResponseWriter, r *http.Request) {
	var postSlave Slave
	err := json.NewDecoder(r.Body).Decode(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "cannot parse object (%s)", err.Error())
		return
	}

	// Validation

	if postSlave.ID != 0 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not change the slave ID in PUT request")
		return
	}

	modelSlave, err := ProjectSlaveToModelSlave(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, err.Error())
		return
	}

	// Persist to database

	err = m.DB.Create(modelSlave).Error
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}

	// TODO set location header. Would it be better to return the ID? YES.

	return
}

func (m *MasterAPI) SlaveUpdate(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id64, err := strconv.ParseUint(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	id := uint(id64)

	var postSlave Slave
	err = json.NewDecoder(r.Body).Decode(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "cannot parse object (%s)", err.Error())
		return
	}

	// Validation

	if postSlave.ID != id {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not change the id of an object")
		return
	}

	if err = postSlave.assertNoZeroFieldsSet(); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "must not POST JSON with zero values in any field: %s", err.Error())
		return
	}

	modelSlave, err := ProjectSlaveToModelSlave(&postSlave)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprint(w, err)
		return
	}

	// Persist to database

	m.DB.Model(&modelSlave).Updates(&modelSlave)
}

func (m *MasterAPI) SlaveDelete(w http.ResponseWriter, r *http.Request) {
	idStr := mux.Vars(r)["slaveId"]
	id64, err := strconv.ParseUint(idStr, 10, 0)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	id := uint(id64)

	s := m.DB.Delete(&model.Slave{ID: id})
	if s.Error != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}
	if s.RowsAffected == 0 {
		w.WriteHeader(http.StatusNotFound)
	}

	if s.RowsAffected > 1 {
		log.Printf("inconsistency: slave DELETE affected more than one row. Slave.ID = %v", id)
	}
}
<|endoftext|>


<commit_before>/*
 Copyright 2020 The GoPlus Authors (goplus.org)

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package golang

import (
	"go/ast"
	"go/token"
	"reflect"
	"strings"

	"github.com/goplus/gop/exec.spec"
	"github.com/qiniu/x/log"
)

// -----------------------------------------------------------------------------

// Var represents a variable.
type Var struct {
	typ   reflect.Type
	name  string
	where *scopeCtx
}

// NewVar creates a variable instance.
func NewVar(typ reflect.Type, name string) *Var {
	c := name[0]
	if c >= '0' && c <= '9' {
		name = "_ret_" + name
	}
	return &Var{typ: typ, name: name}
}

// Type returns variable's type.
func (p *Var) Type() reflect.Type {
	return p.typ
}

// Name returns variable's name.
func (p *Var) Name() string {
	return p.name
}

// IsUnnamedOut returns if variable unnamed or not.
func (p *Var) IsUnnamedOut() bool {
	return strings.HasPrefix(p.name, "_ret_")
}

func (p *Var) setScope(where *scopeCtx) {
	if p.where != nil {
		panic("Var.setScope: variable already defined")
	}
	p.where = where
}

// -----------------------------------------------------------------------------

type scopeCtx struct {
	parentCtx *scopeCtx
	vlist     []exec.Var
	stmts     []ast.Stmt
	labels    []*Label // labels of current statement
}

func (p *scopeCtx) addVar(vars ...exec.Var) {
	for _, v := range vars {
		v.(*Var).setScope(p)
	}
	p.vlist = append(p.vlist, vars...)
}

func (p *scopeCtx) toGenDecl(b *Builder) *ast.GenDecl {
	n := len(p.vlist)
	if n == 0 {
		return nil
	}
	specs := make([]ast.Spec, 0, n)
	for _, item := range p.vlist {
		v := item.(*Var)
		spec := &ast.ValueSpec{
			Names: []*ast.Ident{Ident(v.name)},
			Type:  Type(b, v.typ),
		}
		specs = append(specs, spec)
	}
	return &ast.GenDecl{
		Tok:   token.VAR,
		Specs: specs,
	}
}

func (p *scopeCtx) getStmts(b *Builder) []ast.Stmt {
	if decl := p.toGenDecl(b); decl != nil {
		p.stmts[0] = &ast.DeclStmt{Decl: decl}
		return p.stmts
	}
	return p.stmts[1:]
}

func (p *scopeCtx) initStmts() {
	p.stmts = make([]ast.Stmt, 1, 8)
}

// -----------------------------------------------------------------------------

// DefineVar defines variables.
func (p *Builder) DefineVar(vars ...exec.Var) *Builder {
	var vlist []exec.Var
	for _, v := range vars {
		if pkgPath := v.Type().PkgPath(); pkgPath != "" {
			p.Import(pkgPath)
		} else if v.Name() == "_" {
			continue
		}
		pv := v.(*Var)
		if strings.HasPrefix(pv.name, "_") {
			pv.name = "_q" + pv.name
		}
		vlist = append(vlist, v)
	}
	p.addVar(vlist...)
	return p
}

// InCurrentCtx returns if a variable is in current context or not.
func (p *Builder) InCurrentCtx(v exec.Var) bool {
	return p.scopeCtx == v.(*Var).where
}

// Load instr
func (p *Builder) Load(idx int32) *Builder {
	p.rhs.Push(p.argIdent(idx))
	return p
}

// Addr instr
func (p *Builder) Addr(idx int32) *Builder {
	p.rhs.Push(p.argIdent(idx))
	return p
}

// Store instr
func (p *Builder) Store(idx int32) *Builder {
	p.lhs.Push(p.argIdent(idx))
	return p
}

func (p *Builder) argIdent(idx int32) *ast.Ident {
	i := len(p.cfun.in) + int(idx)
	if i == -1 {
		return Ident("_recv")
	}
	return Ident(toArg(i))
}

// LoadVar instr
func (p *Builder) LoadVar(v exec.Var) *Builder {
	p.rhs.Push(Ident(v.(*Var).name))
	return p
}

// StoreVar instr
func (p *Builder) StoreVar(v exec.Var) *Builder {
	p.lhs.Push(Ident(v.(*Var).name))
	return p
}

// AddrVar instr
func (p *Builder) AddrVar(v exec.Var) *Builder {
	p.rhs.Push(&ast.UnaryExpr{
		Op: token.AND,
		X:  Ident(v.(*Var).name),
	})
	return p
}

func (p *Builder) bigAddrOp(kind exec.Kind, op exec.AddrOperator) *Builder {
	if op == exec.OpAddrVal {
		return p
	}
	method := addropMethods[op]
	if method == "" {
		log.Panicln("bigAddrOp: unknown op -", op)
	}
	var expr ast.Expr
	var x = p.rhs.Pop()
	var val = p.rhs.Pop().(ast.Expr)
	if op == exec.OpInc || op == exec.OpDec {
		pkg := p.Import("math/big")
		var fnName string
		switch kind {
		case exec.BigInt:
			fnName = "NewInt"
		case exec.BigRat:
			fnName = "NewRat"
		case exec.BigFloat:
			fnName = "NewFloat"
		}
		val = &ast.CallExpr{
			Fun: &ast.SelectorExpr{
				&ast.Ident{Name: pkg},
				&ast.Ident{Name: fnName},
			},
			Args: []ast.Expr{&ast.Ident{Name: "1"}},
		}
	}
	switch v := x.(type) {
	case *ast.UnaryExpr:
		if v.Op != token.AND {
			log.Panicln("bigAddrOp: unknown x expr -", reflect.TypeOf(x))
		}
		bigOp := &ast.SelectorExpr{X: v.X, Sel: Ident(method)}
		expr = &ast.CallExpr{Fun: bigOp, Args: []ast.Expr{v.X, val}}
	default:
		log.Panicln("bigAddrOp: todo")
	}
	p.rhs.Push(&ast.ExprStmt{X: expr})
	return p
}

var addropMethods = [...]string{
	exec.OpAddAssign:    "Add",
	exec.OpSubAssign:    "Sub",
	exec.OpMulAssign:    "Mul",
	exec.OpQuoAssign:    "Quo",
	exec.OpModAssign:    "Mod",
	exec.OpAndAssign:    "And",
	exec.OpOrAssign:     "Or",
	exec.OpXorAssign:    "Xor",
	exec.OpAndNotAssign: "AndNot",
	exec.OpLshAssign:    "Lsh",
	exec.OpRshAssign:    "Rsh",
	exec.OpInc:          "Add",
	exec.OpDec:          "Sub",
}

// AddrOp instr
func (p *Builder) AddrOp(kind exec.Kind, op exec.AddrOperator) *Builder {
	if kind >= exec.BigInt {
		return p.bigAddrOp(kind, op)
	}
	if op == exec.OpAddrVal {
		p.rhs.Push(&ast.StarExpr{
			X: p.rhs.Pop().(ast.Expr),
		})
		return p
	}
	if op == exec.OpAssign {
		p.emitStmt(&ast.AssignStmt{
			Lhs: []ast.Expr{&ast.StarExpr{
				X: p.rhs.Pop().(ast.Expr),
			}},
			Tok: token.ASSIGN,
			Rhs: []ast.Expr{p.rhs.Pop().(ast.Expr)},
		})
		return p
	}
	var stmt ast.Stmt
	var x = p.rhs.Pop()
	var val = p.rhs.Pop().(ast.Expr)
	switch v := x.(type) {
	case *ast.UnaryExpr:
		if v.Op != token.AND {
			log.Panicln("AddrOp: unknown x expr -", reflect.TypeOf(x))
		}
		if op == exec.OpInc || op == exec.OpDec {
			stmt = &ast.IncDecStmt{X: v.X, TokPos: v.OpPos, Tok: addropTokens[op]}
		} else {
			stmt = &ast.AssignStmt{Lhs: []ast.Expr{v.X}, Tok: addropTokens[op], Rhs: []ast.Expr{val}}
		}
	default:
		log.Panicln("AddrOp: todo")
	}
	p.rhs.Push(stmt)
	return p
}

var addropTokens = [...]token.Token{
	exec.OpAddAssign:    token.ADD_ASSIGN,
	exec.OpSubAssign:    token.SUB_ASSIGN,
	exec.OpMulAssign:    token.MUL_ASSIGN,
	exec.OpQuoAssign:    token.QUO_ASSIGN,
	exec.OpModAssign:    token.REM_ASSIGN,
	exec.OpAndAssign:    token.AND_ASSIGN,
	exec.OpOrAssign:     token.OR_ASSIGN,
	exec.OpXorAssign:    token.XOR_ASSIGN,
	exec.OpAndNotAssign: token.AND_NOT_ASSIGN,
	exec.OpLshAssign:    token.SHL_ASSIGN,
	exec.OpRshAssign:    token.SHR_ASSIGN,
	exec.OpInc:          token.INC,
	exec.OpDec:          token.DEC,
}

// -----------------------------------------------------------------------------
<commit_msg>golang: big inc/dec simply code<commit_after>/*
 Copyright 2020 The GoPlus Authors (goplus.org)

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package golang

import (
	"go/ast"
	"go/token"
	"reflect"
	"strings"

	"github.com/goplus/gop/exec.spec"
	"github.com/qiniu/x/log"
)

// -----------------------------------------------------------------------------

// Var represents a variable.
type Var struct {
	typ   reflect.Type
	name  string
	where *scopeCtx
}

// NewVar creates a variable instance.
func NewVar(typ reflect.Type, name string) *Var {
	c := name[0]
	if c >= '0' && c <= '9' {
		name = "_ret_" + name
	}
	return &Var{typ: typ, name: name}
}

// Type returns variable's type.
func (p *Var) Type() reflect.Type {
	return p.typ
}

// Name returns variable's name.
func (p *Var) Name() string {
	return p.name
}

// IsUnnamedOut returns if variable unnamed or not.
func (p *Var) IsUnnamedOut() bool {
	return strings.HasPrefix(p.name, "_ret_")
}

func (p *Var) setScope(where *scopeCtx) {
	if p.where != nil {
		panic("Var.setScope: variable already defined")
	}
	p.where = where
}

// -----------------------------------------------------------------------------

type scopeCtx struct {
	parentCtx *scopeCtx
	vlist     []exec.Var
	stmts     []ast.Stmt
	labels    []*Label // labels of current statement
}

func (p *scopeCtx) addVar(vars ...exec.Var) {
	for _, v := range vars {
		v.(*Var).setScope(p)
	}
	p.vlist = append(p.vlist, vars...)
}

func (p *scopeCtx) toGenDecl(b *Builder) *ast.GenDecl {
	n := len(p.vlist)
	if n == 0 {
		return nil
	}
	specs := make([]ast.Spec, 0, n)
	for _, item := range p.vlist {
		v := item.(*Var)
		spec := &ast.ValueSpec{
			Names: []*ast.Ident{Ident(v.name)},
			Type:  Type(b, v.typ),
		}
		specs = append(specs, spec)
	}
	return &ast.GenDecl{
		Tok:   token.VAR,
		Specs: specs,
	}
}

func (p *scopeCtx) getStmts(b *Builder) []ast.Stmt {
	if decl := p.toGenDecl(b); decl != nil {
		p.stmts[0] = &ast.DeclStmt{Decl: decl}
		return p.stmts
	}
	return p.stmts[1:]
}

func (p *scopeCtx) initStmts() {
	p.stmts = make([]ast.Stmt, 1, 8)
}

// -----------------------------------------------------------------------------

// DefineVar defines variables.
func (p *Builder) DefineVar(vars ...exec.Var) *Builder {
	var vlist []exec.Var
	for _, v := range vars {
		if pkgPath := v.Type().PkgPath(); pkgPath != "" {
			p.Import(pkgPath)
		} else if v.Name() == "_" {
			continue
		}
		pv := v.(*Var)
		if strings.HasPrefix(pv.name, "_") {
			pv.name = "_q" + pv.name
		}
		vlist = append(vlist, v)
	}
	p.addVar(vlist...)
	return p
}

// InCurrentCtx returns if a variable is in current context or not.
func (p *Builder) InCurrentCtx(v exec.Var) bool {
	return p.scopeCtx == v.(*Var).where
}

// Load instr
func (p *Builder) Load(idx int32) *Builder {
	p.rhs.Push(p.argIdent(idx))
	return p
}

// Addr instr
func (p *Builder) Addr(idx int32) *Builder {
	p.rhs.Push(p.argIdent(idx))
	return p
}

// Store instr
func (p *Builder) Store(idx int32) *Builder {
	p.lhs.Push(p.argIdent(idx))
	return p
}

func (p *Builder) argIdent(idx int32) *ast.Ident {
	i := len(p.cfun.in) + int(idx)
	if i == -1 {
		return Ident("_recv")
	}
	return Ident(toArg(i))
}

// LoadVar instr
func (p *Builder) LoadVar(v exec.Var) *Builder {
	p.rhs.Push(Ident(v.(*Var).name))
	return p
}

// StoreVar instr
func (p *Builder) StoreVar(v exec.Var) *Builder {
	p.lhs.Push(Ident(v.(*Var).name))
	return p
}

// AddrVar instr
func (p *Builder) AddrVar(v exec.Var) *Builder {
	p.rhs.Push(&ast.UnaryExpr{
		Op: token.AND,
		X:  Ident(v.(*Var).name),
	})
	return p
}

func (p *Builder) bigAddrOp(kind exec.Kind, op exec.AddrOperator) *Builder {
	if op == exec.OpAddrVal {
		return p
	}
	method := addropMethods[op]
	if method == "" {
		log.Panicln("bigAddrOp: unknown op -", op)
	}
	var expr ast.Expr
	var x = p.rhs.Pop()
	var val = p.rhs.Pop().(ast.Expr)
	if op == exec.OpInc || op == exec.OpDec {
		var fnName string
		switch kind {
		case exec.BigInt:
			fnName = "NewInt"
		case exec.BigRat:
			fnName = "NewRat"
		case exec.BigFloat:
			fnName = "NewFloat"
		}
		val = &ast.CallExpr{
			Fun:  p.GoSymIdent(p.Import("math/big"), fnName),
			Args: []ast.Expr{&ast.Ident{Name: "1"}},
		}
	}
	switch v := x.(type) {
	case *ast.UnaryExpr:
		if v.Op != token.AND {
			log.Panicln("bigAddrOp: unknown x expr -", reflect.TypeOf(x))
		}
		bigOp := &ast.SelectorExpr{X: v.X, Sel: Ident(method)}
		expr = &ast.CallExpr{Fun: bigOp, Args: []ast.Expr{v.X, val}}
	default:
		log.Panicln("bigAddrOp: todo")
	}
	p.rhs.Push(&ast.ExprStmt{X: expr})
	return p
}

var addropMethods = [...]string{
	exec.OpAddAssign:    "Add",
	exec.OpSubAssign:    "Sub",
	exec.OpMulAssign:    "Mul",
	exec.OpQuoAssign:    "Quo",
	exec.OpModAssign:    "Mod",
	exec.OpAndAssign:    "And",
	exec.OpOrAssign:     "Or",
	exec.OpXorAssign:    "Xor",
	exec.OpAndNotAssign: "AndNot",
	exec.OpLshAssign:    "Lsh",
	exec.OpRshAssign:    "Rsh",
	exec.OpInc:          "Add",
	exec.OpDec:          "Sub",
}

// AddrOp instr
func (p *Builder) AddrOp(kind exec.Kind, op exec.AddrOperator) *Builder {
	if kind >= exec.BigInt {
		return p.bigAddrOp(kind, op)
	}
	if op == exec.OpAddrVal {
		p.rhs.Push(&ast.StarExpr{
			X: p.rhs.Pop().(ast.Expr),
		})
		return p
	}
	if op == exec.OpAssign {
		p.emitStmt(&ast.AssignStmt{
			Lhs: []ast.Expr{&ast.StarExpr{
				X: p.rhs.Pop().(ast.Expr),
			}},
			Tok: token.ASSIGN,
			Rhs: []ast.Expr{p.rhs.Pop().(ast.Expr)},
		})
		return p
	}
	var stmt ast.Stmt
	var x = p.rhs.Pop()
	var val = p.rhs.Pop().(ast.Expr)
	switch v := x.(type) {
	case *ast.UnaryExpr:
		if v.Op != token.AND {
			log.Panicln("AddrOp: unknown x expr -", reflect.TypeOf(x))
		}
		if op == exec.OpInc || op == exec.OpDec {
			stmt = &ast.IncDecStmt{X: v.X, TokPos: v.OpPos, Tok: addropTokens[op]}
		} else {
			stmt = &ast.AssignStmt{Lhs: []ast.Expr{v.X}, Tok: addropTokens[op], Rhs: []ast.Expr{val}}
		}
	default:
		log.Panicln("AddrOp: todo")
	}
	p.rhs.Push(stmt)
	return p
}

var addropTokens = [...]token.Token{
	exec.OpAddAssign:    token.ADD_ASSIGN,
	exec.OpSubAssign:    token.SUB_ASSIGN,
	exec.OpMulAssign:    token.MUL_ASSIGN,
	exec.OpQuoAssign:    token.QUO_ASSIGN,
	exec.OpModAssign:    token.REM_ASSIGN,
	exec.OpAndAssign:    token.AND_ASSIGN,
	exec.OpOrAssign:     token.OR_ASSIGN,
	exec.OpXorAssign:    token.XOR_ASSIGN,
	exec.OpAndNotAssign: token.AND_NOT_ASSIGN,
	exec.OpLshAssign:    token.SHL_ASSIGN,
	exec.OpRshAssign:    token.SHR_ASSIGN,
	exec.OpInc:          token.INC,
	exec.OpDec:          token.DEC,
}

// -----------------------------------------------------------------------------
<|endoftext|>


<commit_before>package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

type downloadResult struct {
	Page  []byte
	Error error
}

type releaseView struct {
	DownloadUrl string
	Version     string
	Md5         string
	Path        string
}

var rePackageUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)#md5=.+?\">(?P<filename>.+?)</a>`)
var reDownloadUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)\"\s+rel=\".+\">(?P<version>.+?) download_url</a>`)

func simpleIndexHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(w, "index", nil)
}

func getPage(url string) downloadResult {
	log.Println("Downloading page: " + url)
	res, err := http.Get(url)
	if err != nil {
		return downloadResult{Page: []byte(""), Error: err}
	}
	page, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return downloadResult{Page: []byte(""), Error: err}
	}
	return downloadResult{Page: page, Error: err}
}

func updateProxyCache(pkg Package) error {
	log.Println("Updating proxy cache for: " + pkg.Name)
	url := Config.PypiMirror + "/simple/" + pkg.Name + "/"

	result := getPage(url)

	if result.Error != nil {
		// We'll try next time
		return result.Error
	}

	finalizeCache(pkg, result.Page)
	return nil
}

func finalizeCache(pkg Package, data []byte) {
	log.Println("Finalizing cache for: " + pkg.Name)
	returnData := string(data)
	// Replace the local package links with links to a local proxy
	// so we can cache that result as well.
	packageUris := rePackageUrl.FindAllSubmatch(data, -1)
	for _, line := range packageUris {
		uri := line[1]
		filename := line[2]
		// TODO: This most certainly has edge cases that aren't addressed here.
		// Specifically: ".tar.gz" can't be the only filetype uploaded...
		versionSplit := strings.Split(string(uri), "-")
		almostVersion := versionSplit[len(versionSplit)-1]
		version := strings.Replace(almostVersion, ".tar.gz", "", -1)
		quoteduri := url.QueryEscape(Config.PypiMirror + "/a/b/" + string(uri))
		replaceuri := "/fondu/cached-file/" + string(filename) + "?package=" + pkg.Name + "&release=" + version + "&original=" + quoteduri + "&name=" + url.QueryEscape(string(filename))
		returnData = strings.Replace(returnData, string(uri), replaceuri, -1)
	}

	// Replace the download links with links to a local proxy so that
	// we can cache the downloads as well.
	downloadUrls := reDownloadUrl.FindAllSubmatch(data, -1)
	for _, line := range downloadUrls {
		uri := line[1]
		version := line[2]
		filename := pkg.Name + "-" + string(version) + ".tar.gz"
		quotedUri := url.QueryEscape(string(uri))
		replaceUri := "/fondu/cached-file/" + filename + "?package=" + pkg.Name + "&release=" + string(version) + "&original=" + quotedUri + "&name=" + url.QueryEscape(filename)
		returnData = strings.Replace(returnData, string(uri), replaceUri, -1)
	}

	pkg.SetProxy([]byte(returnData))
}

func renderProxy(w http.ResponseWriter, pkg Package) {
	log.Println("Rendering proxy for: " + pkg.Name)
	io.WriteString(w, string(pkg.ProxyData()))
}

func buildReleaseMap(pkg Package) []releaseView {
	releaseMap := []releaseView{}
	for _, rel := range pkg.Releases() {
		metadata, err := rel.Metadata()
		if err != nil {
			break
		}
		md5Json, err := metadata.Get("md5_digest").Array()
		if err != nil {
			continue
		}
		md5 := md5Json[0]

		releaseMap = append(releaseMap, releaseView{
			DownloadUrl: rel.DownloadUrl(),
			Version:     rel.Version,
			Md5:         md5.(string),
			Path:        rel.Path(),
		})
	}
	return releaseMap
}

func simpleHandler(w http.ResponseWriter, r *http.Request) {
	paths := strings.Split(r.URL.Path, "/")
	name := paths[len(paths)-2]
	pkg := Package{Name: name, DataDir: Config.DataDir}

	// The package is ours, so we serve it ourselves.
	if pkg.Exists() && !pkg.Proxied() {
		log.Print("Private package: " + name + ". Serving it.")
		releaseMap := buildReleaseMap(pkg)
		renderTemplate(w, "single", &releaseMap)
		return
	}

	// Public package, so just render the proxy
	if pkg.Proxied() {
		log.Print("Proxied package: " + name + ". Sending cached data.")
		go updateProxyCache(pkg)
		renderProxy(w, pkg)
		return
	}

	if err := updateProxyCache(pkg); err != nil {
		http.Error(w, "Unable to update proxy", http.StatusInternalServerError)
		return
	}

	renderProxy(w, pkg)
}
<commit_msg>Disable background cache updates for now<commit_after>package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

type downloadResult struct {
	Page  []byte
	Error error
}

type releaseView struct {
	DownloadUrl string
	Version     string
	Md5         string
	Path        string
}

var rePackageUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)#md5=.+?\">(?P<filename>.+?)</a>`)
var reDownloadUrl = regexp.MustCompile(`(?i)<a href=\"(?P<url>.+?)\"\s+rel=\".+\">(?P<version>.+?) download_url</a>`)

func simpleIndexHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(w, "index", nil)
}

func getPage(url string) downloadResult {
	log.Println("Downloading page: " + url)
	res, err := http.Get(url)
	if err != nil {
		return downloadResult{Page: []byte(""), Error: err}
	}
	page, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return downloadResult{Page: []byte(""), Error: err}
	}
	return downloadResult{Page: page, Error: err}
}

func updateProxyCache(pkg Package) error {
	log.Println("Updating proxy cache for: " + pkg.Name)
	url := Config.PypiMirror + "/simple/" + pkg.Name + "/"

	result := getPage(url)

	if result.Error != nil {
		// We'll try next time
		return result.Error
	}

	finalizeCache(pkg, result.Page)
	return nil
}

func finalizeCache(pkg Package, data []byte) {
	log.Println("Finalizing cache for: " + pkg.Name)
	returnData := string(data)
	// Replace the local package links with links to a local proxy
	// so we can cache that result as well.
	packageUris := rePackageUrl.FindAllSubmatch(data, -1)
	for _, line := range packageUris {
		uri := line[1]
		filename := line[2]
		// TODO: This most certainly has edge cases that aren't addressed here.
		// Specifically: ".tar.gz" can't be the only filetype uploaded...
		versionSplit := strings.Split(string(uri), "-")
		almostVersion := versionSplit[len(versionSplit)-1]
		version := strings.Replace(almostVersion, ".tar.gz", "", -1)
		quoteduri := url.QueryEscape(Config.PypiMirror + "/a/b/" + string(uri))
		replaceuri := "/fondu/cached-file/" + string(filename) + "?package=" + pkg.Name + "&release=" + version + "&original=" + quoteduri + "&name=" + url.QueryEscape(string(filename))
		returnData = strings.Replace(returnData, string(uri), replaceuri, -1)
	}

	// Replace the download links with links to a local proxy so that
	// we can cache the downloads as well.
	downloadUrls := reDownloadUrl.FindAllSubmatch(data, -1)
	for _, line := range downloadUrls {
		uri := line[1]
		version := line[2]
		filename := pkg.Name + "-" + string(version) + ".tar.gz"
		quotedUri := url.QueryEscape(string(uri))
		replaceUri := "/fondu/cached-file/" + filename + "?package=" + pkg.Name + "&release=" + string(version) + "&original=" + quotedUri + "&name=" + url.QueryEscape(filename)
		returnData = strings.Replace(returnData, string(uri), replaceUri, -1)
	}

	pkg.SetProxy([]byte(returnData))
}

func renderProxy(w http.ResponseWriter, pkg Package) {
	log.Println("Rendering proxy for: " + pkg.Name)
	io.WriteString(w, string(pkg.ProxyData()))
}

func buildReleaseMap(pkg Package) []releaseView {
	releaseMap := []releaseView{}
	for _, rel := range pkg.Releases() {
		metadata, err := rel.Metadata()
		if err != nil {
			break
		}
		md5Json, err := metadata.Get("md5_digest").Array()
		if err != nil {
			continue
		}
		md5 := md5Json[0]

		releaseMap = append(releaseMap, releaseView{
			DownloadUrl: rel.DownloadUrl(),
			Version:     rel.Version,
			Md5:         md5.(string),
			Path:        rel.Path(),
		})
	}
	return releaseMap
}

func simpleHandler(w http.ResponseWriter, r *http.Request) {
	paths := strings.Split(r.URL.Path, "/")
	name := paths[len(paths)-2]
	pkg := Package{Name: name, DataDir: Config.DataDir}

	// The package is ours, so we serve it ourselves.
	if pkg.Exists() && !pkg.Proxied() {
		log.Print("Private package: " + name + ". Serving it.")
		releaseMap := buildReleaseMap(pkg)
		renderTemplate(w, "single", &releaseMap)
		return
	}

	// Public package, so just render the proxy
	if pkg.Proxied() {
		log.Print("Proxied package: " + name + ". Sending cached data.")
		// go updateProxyCache(pkg)
		renderProxy(w, pkg)
		return
	}

	if err := updateProxyCache(pkg); err != nil {
		http.Error(w, "Unable to update proxy", http.StatusInternalServerError)
		return
	}

	renderProxy(w, pkg)
}
<|endoftext|>