<commit_before>package ftp

import (
	"net"
	"fmt"
	"bufio"
	"strings"
	"strconv"
)

// FTPClient is a representation of a client connected to the FTP server. It contains all data necessary for the entire browsing session.
type FTPClient struct {
	server 			*FTPServer
	conn 			net.Conn // connection instance
	writer 			*bufio.Writer
	scanner 		*bufio.Scanner
	authenticated   bool
	user 			string
	password 		string
	dir				string
	relativedir		string
	transferType 	string
	Mode			int
	dataSocket 		net.Conn // data socket
}

// HandleRequest is used to handle a command sent by the client to the server. It takes a string as a paramater and does not return any data.
func (this *FTPClient) HandleRequest(req string) {
	// get COMMAND, then MESSAGE
	request := strings.SplitAfterN(req, ` `, 2)
	command := strings.Trim(request[0], ` `)

	// did they even send a message?
	if len(request) > 1 {
		message := strings.Trim(request[1], ` `)

		fmt.Println("Command: " + command + ", message: " + message)

		// lets assign the command
		switch command {
		case "USER":
			this.USER(message)
		case "PASS":
			this.PASS(message)
		case "TYPE":
			this.TYPE(message)
		default:
			this.NOTIMP()
		}

	// there was no message
	} else {
		fmt.Println("Command: " + command)

		// handle
		switch command {
		case "PASV":
			this.PASV()
		case "QUIT":
			this.QUIT()
		case "SYST":
			this.SYST()
		case "FEAT":
			this.FEAT()
		case "PWD":
			this.PWD()
		case "LIST":
			this.LIST()
		default:
			this.NOTIMP()
		}
	}
}

// Send a message to the FTP Client.
func (this *FTPClient) SendMessage(code int) {
	message := GetMessages()[code]
	completeMsg := strconv.Itoa(code) + " " + message

	this.Write(completeMsg)
	fmt.Println(completeMsg)
}

// Send a message to the FTP Client, with an injectable.
func (this *FTPClient) SendMessageWithInjectable(code int, injectable string) {
	message := GetMessages()[code]
	completeMsg := strconv.Itoa(code) + " " + message
	completeMsg = strings.Replace(completeMsg, "%s", injectable, -1)

	this.Write(completeMsg)
	fmt.Println(completeMsg)
}

// Write a string to the client.
func (this *FTPClient) Write(message string) {
	_, err := this.writer.WriteString(message + "\n")
	if err != nil {
		fmt.Println("Error occurred writing to connection: " + err.Error())
		return
	}
	err = this.writer.Flush()
	if err != nil {
		fmt.Println("Error occurred flushing data stream: " + err.Error())
	}
}

// Write a string to the client's data socket.
func (this *FTPClient) WriteDataSocket(message string) {
	writer := bufio.NewWriter(this.dataSocket)
	_, err := writer.WriteString(message + "\n")
	if err != nil {
		fmt.Println("Error occurred writing to data socket connection: " + err.Error())
		return
	}
	err = writer.Flush()
	if err != nil {
		fmt.Println("Error occurred flushing data socket stream: " + err.Error())
	}
}

// Closes the client.
func (this *FTPClient) Close() {
	this.conn.Close()
}<commit_msg>Adds ability to close data socket<commit_after>package ftp

import (
	"net"
	"fmt"
	"bufio"
	"strings"
	"strconv"
)

// FTPClient is a representation of a client connected to the FTP server. It contains all data necessary for the entire browsing session.
type FTPClient struct {
	server 			*FTPServer
	conn 			net.Conn // connection instance
	writer 			*bufio.Writer
	scanner 		*bufio.Scanner
	authenticated   bool
	user 			string
	password 		string
	dir				string
	relativedir		string
	transferType 	string
	Mode			int
	dataSocket 		net.Conn // data socket
}

// HandleRequest is used to handle a command sent by the client to the server. It takes a string as a paramater and does not return any data.
func (this *FTPClient) HandleRequest(req string) {
	// get COMMAND, then MESSAGE
	request := strings.SplitAfterN(req, ` `, 2)
	command := strings.Trim(request[0], ` `)

	// did they even send a message?
	if len(request) > 1 {
		message := strings.Trim(request[1], ` `)

		fmt.Println("Command: " + command + ", message: " + message)

		// lets assign the command
		switch command {
		case "USER":
			this.USER(message)
		case "PASS":
			this.PASS(message)
		case "TYPE":
			this.TYPE(message)
		default:
			this.NOTIMP()
		}

	// there was no message
	} else {
		fmt.Println("Command: " + command)

		// handle
		switch command {
		case "PASV":
			this.PASV()
		case "QUIT":
			this.QUIT()
		case "SYST":
			this.SYST()
		case "FEAT":
			this.FEAT()
		case "PWD":
			this.PWD()
		case "LIST":
			this.LIST()
		default:
			this.NOTIMP()
		}
	}
}

// Send a message to the FTP Client.
func (this *FTPClient) SendMessage(code int) {
	message := GetMessages()[code]
	completeMsg := strconv.Itoa(code) + " " + message

	this.Write(completeMsg)
	fmt.Println(completeMsg)
}

// Send a message to the FTP Client, with an injectable.
func (this *FTPClient) SendMessageWithInjectable(code int, injectable string) {
	message := GetMessages()[code]
	completeMsg := strconv.Itoa(code) + " " + message
	completeMsg = strings.Replace(completeMsg, "%s", injectable, -1)

	this.Write(completeMsg)
	fmt.Println(completeMsg)
}

// Write a string to the client.
func (this *FTPClient) Write(message string) {
	_, err := this.writer.WriteString(message + "\n")
	if err != nil {
		fmt.Println("Error occurred writing to connection: " + err.Error())
		return
	}
	err = this.writer.Flush()
	if err != nil {
		fmt.Println("Error occurred flushing data stream: " + err.Error())
	}
}

// Write a string to the client's data socket.
func (this *FTPClient) WriteDataSocket(message string) {
	writer := bufio.NewWriter(this.dataSocket)
	_, err := writer.WriteString(message + "\n")
	if err != nil {
		fmt.Println("Error occurred writing to data socket connection: " + err.Error())
		return
	}
	err = writer.Flush()
	if err != nil {
		fmt.Println("Error occurred flushing data socket stream: " + err.Error())
	}
}

// Closes the data socket.
func (this *FTPClient) CloseDataSocket() {
	this.dataSocket.Close()
}

// Closes the client.
func (this *FTPClient) Close() {
	this.conn.Close()
}<|endoftext|>


<commit_before>package main

import (
	"zvr/utils"
	"time"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"encoding/json"
	"github.com/pkg/errors"
	"fmt"
	"zvr/server"
	"strings"
	"os"
)

const (
	VIRTIO_PORT_PATH = "/dev/virtio-ports/applianceVm.vport"
	BOOTSTRAP_INFO_CACHE = "/home/vyos/zvr/bootstrap-info.json"
	TMP_LOCATION_FOR_ESX = "/tmp/bootstrap-info.json"
)

type nic struct {
	mac string
	ip string
	name string
	netmask string
	isDefaultRoute bool
	gateway string
}

var bootstrapInfo map[string]interface{} = make(map[string]interface{})
var nics map[string]*nic = make(map[string]*nic)

func waitIptablesServiceOnline()  {
	bash := utils.Bash{
		Command: "/sbin/iptables-save",
	}

	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		err := bash.Run()
		if err != nil {
			log.Debugf("iptables service seems not ready, %v", err)
		}
		return err == nil
	}, time.Duration(120)*time.Second, time.Duration(500)*time.Millisecond)
}

func waitVirtioPortOnline() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		ok, err := utils.PathExists(VIRTIO_PORT_PATH); utils.PanicOnError(err)
		if !ok {
			log.Debugf("%s doesn't not exist, wait it ...", VIRTIO_PORT_PATH)
		}
		return ok
	}, time.Duration(120)*time.Second, time.Duration(500)*time.Millisecond)
}

func isOnVMwareHypervisor() bool {
	bash := utils.Bash{
		Command: "dmesg | grep -q 'Hypervisor.*VMware'",
	}

	if ret, _, _, err := bash.RunWithReturn(); ret == 0 && err == nil {
		return true
	}

	return false
}

func parseEsxBootInfo() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		if _, err := os.Stat(TMP_LOCATION_FOR_ESX); os.IsNotExist(err) {
			log.Debugf("bootstrap info not ready, waiting ...")
			return false
		}

		content, err := ioutil.ReadFile(TMP_LOCATION_FOR_ESX); utils.PanicOnError(err)
		if err = json.Unmarshal(content, &bootstrapInfo); err != nil {
			panic(errors.Wrap(err, fmt.Sprintf("unable to JSON parse:\n %s", string(content))))
		}

		err = utils.MkdirForFile(BOOTSTRAP_INFO_CACHE, 0666); utils.PanicOnError(err)
		err = os.Rename(TMP_LOCATION_FOR_ESX, BOOTSTRAP_INFO_CACHE); utils.PanicOnError(err)
		err = os.Chmod(BOOTSTRAP_INFO_CACHE, 0777); utils.PanicOnError(err)
		log.Debugf("recieved bootstrap info:\n%s", string(content))
		return true
	}, time.Duration(300)*time.Second, time.Duration(1)*time.Second)
}

func parseKvmBootInfo() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		content, err := ioutil.ReadFile(VIRTIO_PORT_PATH); utils.PanicOnError(err)
		if len(content) == 0 {
			log.Debugf("no content in %s, it may not be ready, wait it ...", VIRTIO_PORT_PATH)
			return false
		}

		if err := json.Unmarshal(content, &bootstrapInfo); err != nil {
			panic(errors.Wrap(err, fmt.Sprintf("unable to JSON parse:\n %s", string(content))))
		}

		err = utils.MkdirForFile(BOOTSTRAP_INFO_CACHE, 0666); utils.PanicOnError(err)
		err = ioutil.WriteFile(BOOTSTRAP_INFO_CACHE, content, 0666); utils.PanicOnError(err)
		err = os.Chmod(BOOTSTRAP_INFO_CACHE, 0777); utils.PanicOnError(err)
		log.Debugf("recieved bootstrap info:\n%s", string(content))
		return true
	}, time.Duration(300)*time.Second, time.Duration(1)*time.Second)
}

func resetVyos()  {
	// clear all configuration in case someone runs 'save' command manually before,
	// to keep the vyos must be stateless

	// delete all interfaces
	tree := server.NewParserFromShowConfiguration().Tree
	tree.Delete("interfaces ethernet")
	tree.Apply(true)

	// reload default configuration
	server.RunVyosScriptAsUserVyos("load /opt/vyatta/etc/config.boot.default\nsave")
}

func configureVyos()  {
	resetVyos()

	mgmtNic := bootstrapInfo["managementNic"].(map[string]interface{})
	if mgmtNic == nil {
		panic(errors.New("no field 'managementNic' in bootstrap info"))
	}

	eth0 := &nic{ name: "eth0" }
	var ok bool
	eth0.mac, ok = mgmtNic["mac"].(string); utils.PanicIfError(ok, errors.New("cannot find 'mac' field for the management nic"))
	eth0.netmask, ok = mgmtNic["netmask"].(string); utils.PanicIfError(ok, errors.New("cannot find 'netmask' field for the management nic"))
	eth0.ip, ok = mgmtNic["ip"].(string); utils.PanicIfError(ok, errors.New("cannot find 'ip' field for the management nic"))
	eth0.isDefaultRoute = mgmtNic["isDefaultRoute"].(bool)
	eth0.gateway = mgmtNic["gateway"].(string)
	nics[eth0.name] = eth0

	otherNics := bootstrapInfo["additionalNics"].([]interface{})
	if otherNics != nil {
		for _, o := range otherNics {
			onic := o.(map[string]interface{})
			n := &nic{}
			n.name, ok = onic["deviceName"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'deviceName' field for the nic"))
			n.mac, ok = onic["mac"].(string); utils.PanicIfError(ok, errors.New("cannot find 'mac' field for the nic"))
			n.netmask, ok = onic["netmask"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'netmask' field for the nic[name:%s]", n.name))
			n.ip, ok = onic["ip"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'ip' field for the nic[name:%s]", n.name))
			n.gateway = onic["gateway"].(string)
			n.isDefaultRoute = onic["isDefaultRoute"].(bool)
			nics[n.name] = n
		}
	}

	type deviceName struct {
		expected string
		actual string
		swap string
	}

	devNames := make([]*deviceName, 0)

	// check integrity of nics
	for _, nic := range nics {
		utils.Assertf(nic.name != "", "name cannot be empty[mac:%s]", nic.mac)
		utils.Assertf(nic.ip != "", "ip cannot be empty[nicname: %s]", nic.name)
		utils.Assertf(nic.gateway != "", "gateway cannot be empty[nicname:%s]", nic.name)
		utils.Assertf(nic.netmask != "", "netmask cannot be empty[nicname:%s]", nic.name)
		utils.Assertf(nic.mac != "", "mac cannot be empty[nicname:%s]", nic.name)

		nicname, err := utils.GetNicNameByMac(nic.mac); utils.PanicOnError(err)
		if nicname != nic.name {
			devNames = append(devNames, &deviceName{
				expected: nic.name,
				actual: nicname,
			})
		}
	}

	if len(devNames) != 0 {
		// shutdown links and change to temporary names
		cmds := make([]string, 0)
		for i, devname := range devNames {
			devnum := 1000 + i

			devname.swap = fmt.Sprintf("eth%v", devnum)
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v down", devname.actual))
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v name %v", devname.actual, devname.swap))
		}

		b := utils.Bash{
			Command: strings.Join(cmds, "\n"),
		}

		b.Run()
		b.PanicIfError()

		// change temporary names to real names and bring up links
		cmds = make([]string, 0)
		for _, devname := range devNames {
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v name %v", devname.swap, devname.expected))
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v up", devname.expected))
		}

		b = utils.Bash{
			Command: strings.Join(cmds, "\n"),
		}

		b.Run()
		b.PanicIfError()
	}

	vyos := server.NewParserFromShowConfiguration()
	tree := vyos.Tree

	sshkey := bootstrapInfo["publicKey"].(string)
	utils.Assert(sshkey != "", "cannot find 'publicKey' in bootstrap info")
	sshkeyparts := strings.Split(sshkey, " ")
	sshtype := sshkeyparts[0]
	key := sshkeyparts[1]
	id := sshkeyparts[2]

	tree.Setf("system login user vyos authentication public-keys %s key %s", id, key)
	tree.Setf("system login user vyos authentication public-keys %s type %s", id, sshtype)

	setNic := func(nic *nic) {
		cidr, err := utils.NetmaskToCIDR(nic.netmask); utils.PanicOnError(err)
		//tree.Setf("interfaces ethernet %s hw-id %s", nic.name, nic.mac)
		tree.Setf("interfaces ethernet %s address %s", nic.name, fmt.Sprintf("%v/%v", nic.ip, cidr))
		tree.Setf("interfaces ethernet %s duplex auto", nic.name)
		tree.Setf("interfaces ethernet %s smp_affinity auto", nic.name)
		tree.Setf("interfaces ethernet %s speed auto", nic.name)
		if nic.isDefaultRoute {
			tree.Setf("system gateway-address %v", nic.gateway)
		}
	}

	sshport := bootstrapInfo["sshPort"].(float64)
	utils.Assert(sshport != 0, "sshport not found in bootstrap info")
	tree.Setf("service ssh port %v", int(sshport))
	tree.Setf("service ssh listen-address %v", eth0.ip)

	// configure firewall
	for _, nic := range nics {
		setNic(nic)

		tree.SetFirewallOnInterface(nic.name, "local",
			"action accept",
			"state established enable",
			"state related enable",
			fmt.Sprintf("destination address %v", nic.ip),
		)
		tree.SetFirewallOnInterface(nic.name, "local",
			"action accept",
			"protocol icmp",
			fmt.Sprintf("destination address %v", nic.ip),
		)

		if !nic.isDefaultRoute && nic.name != "eth0" {
			// for nic connecting to the guest network
			// make its FORWARDING chain open
			// here in = "-A FORWARDING -i nic.name -j ACCEPT"
			tree.SetFirewallOnInterface(nic.name, "in",
				"action accept",
				"state established enable",
				"state related enable",
				"state new enable",
			)
		} else {
			tree.SetFirewallOnInterface(nic.name, "in",
				"action accept",
				"state established enable",
				"state related enable",
			)
		}

		tree.SetFirewallOnInterface(nic.name, "in",
			"action accept",
			"protocol icmp",
		)

		// only allow ssh traffic on eth0, disable on others
		if nic.name == "eth0" {
			tree.SetFirewallOnInterface(nic.name, "local",
				fmt.Sprintf("destination port %v", int(sshport)),
				fmt.Sprintf("destination address %v", nic.ip),
				"protocol tcp",
				"action accept",
			)
		} else {
			tree.SetFirewallOnInterface(nic.name, "local",
				fmt.Sprintf("destination port %v", int(sshport)),
				fmt.Sprintf("destination address %v", nic.ip),
				"protocol tcp",
				"action reject",
			)
		}

		tree.SetFirewallDefaultAction(nic.name, "local", "reject")
		tree.SetFirewallDefaultAction(nic.name, "in", "reject")

		tree.AttachFirewallToInterface(nic.name, "local")
		tree.AttachFirewallToInterface(nic.name, "in")
	}

	tree.Set("system time-zone Asia/Shanghai")

	password := bootstrapInfo["vyosPassword"]; utils.Assert(password != "", "vyosPassword cannot be empty")
	tree.Setf("system login user vyos authentication plaintext-password %v", password)

	tree.Apply(true)

	arping := func(nicname, ip, gateway string) {
		b := utils.Bash{ Command: fmt.Sprintf("arping -A -U -c 1 -I %s -s %s %s", nicname, ip, gateway) }
		b.Run()
	}

	// arping to advocate our mac addresses
	arping("eth0", eth0.ip, eth0.gateway)
	for _, nic := range nics {
		arping(nic.name, nic.ip, nic.gateway)
	}
}

func startZvr()  {
	b := utils.Bash{
		Command: "/etc/init.d/zstack-virtualrouteragent restart",
	}
	b.Run()
	b.PanicIfError()
}

func main() {
	utils.InitLog("/home/vyos/zvr/zvrboot.log", false)
	waitIptablesServiceOnline()
	if isOnVMwareHypervisor() {
		parseEsxBootInfo()
	} else {
		waitVirtioPortOnline()
		parseKvmBootInfo()
	}
	configureVyos()
	startZvr()
	log.Debugf("successfully configured the sysmtem and bootstrap the zstack virtual router agents")
}
<commit_msg>Do not change the vrouter password on ESX<commit_after>package main

import (
	"zvr/utils"
	"time"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"encoding/json"
	"github.com/pkg/errors"
	"fmt"
	"zvr/server"
	"strings"
	"os"
)

const (
	VIRTIO_PORT_PATH = "/dev/virtio-ports/applianceVm.vport"
	BOOTSTRAP_INFO_CACHE = "/home/vyos/zvr/bootstrap-info.json"
	TMP_LOCATION_FOR_ESX = "/tmp/bootstrap-info.json"
)

type nic struct {
	mac string
	ip string
	name string
	netmask string
	isDefaultRoute bool
	gateway string
}

var bootstrapInfo map[string]interface{} = make(map[string]interface{})
var nics map[string]*nic = make(map[string]*nic)

func waitIptablesServiceOnline()  {
	bash := utils.Bash{
		Command: "/sbin/iptables-save",
	}

	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		err := bash.Run()
		if err != nil {
			log.Debugf("iptables service seems not ready, %v", err)
		}
		return err == nil
	}, time.Duration(120)*time.Second, time.Duration(500)*time.Millisecond)
}

func waitVirtioPortOnline() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		ok, err := utils.PathExists(VIRTIO_PORT_PATH); utils.PanicOnError(err)
		if !ok {
			log.Debugf("%s doesn't not exist, wait it ...", VIRTIO_PORT_PATH)
		}
		return ok
	}, time.Duration(120)*time.Second, time.Duration(500)*time.Millisecond)
}

func isOnVMwareHypervisor() bool {
	bash := utils.Bash{
		Command: "dmesg | grep -q 'Hypervisor.*VMware'",
	}

	if ret, _, _, err := bash.RunWithReturn(); ret == 0 && err == nil {
		return true
	}

	return false
}

func parseEsxBootInfo() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		if _, err := os.Stat(TMP_LOCATION_FOR_ESX); os.IsNotExist(err) {
			log.Debugf("bootstrap info not ready, waiting ...")
			return false
		}

		content, err := ioutil.ReadFile(TMP_LOCATION_FOR_ESX); utils.PanicOnError(err)
		if err = json.Unmarshal(content, &bootstrapInfo); err != nil {
			panic(errors.Wrap(err, fmt.Sprintf("unable to JSON parse:\n %s", string(content))))
		}

		err = utils.MkdirForFile(BOOTSTRAP_INFO_CACHE, 0666); utils.PanicOnError(err)
		err = os.Rename(TMP_LOCATION_FOR_ESX, BOOTSTRAP_INFO_CACHE); utils.PanicOnError(err)
		err = os.Chmod(BOOTSTRAP_INFO_CACHE, 0777); utils.PanicOnError(err)
		log.Debugf("recieved bootstrap info:\n%s", string(content))
		return true
	}, time.Duration(300)*time.Second, time.Duration(1)*time.Second)
}

func parseKvmBootInfo() {
	utils.LoopRunUntilSuccessOrTimeout(func() bool {
		content, err := ioutil.ReadFile(VIRTIO_PORT_PATH); utils.PanicOnError(err)
		if len(content) == 0 {
			log.Debugf("no content in %s, it may not be ready, wait it ...", VIRTIO_PORT_PATH)
			return false
		}

		if err := json.Unmarshal(content, &bootstrapInfo); err != nil {
			panic(errors.Wrap(err, fmt.Sprintf("unable to JSON parse:\n %s", string(content))))
		}

		err = utils.MkdirForFile(BOOTSTRAP_INFO_CACHE, 0666); utils.PanicOnError(err)
		err = ioutil.WriteFile(BOOTSTRAP_INFO_CACHE, content, 0666); utils.PanicOnError(err)
		err = os.Chmod(BOOTSTRAP_INFO_CACHE, 0777); utils.PanicOnError(err)
		log.Debugf("recieved bootstrap info:\n%s", string(content))
		return true
	}, time.Duration(300)*time.Second, time.Duration(1)*time.Second)
}

func resetVyos()  {
	// clear all configuration in case someone runs 'save' command manually before,
	// to keep the vyos must be stateless

	// delete all interfaces
	tree := server.NewParserFromShowConfiguration().Tree
	tree.Delete("interfaces ethernet")
	tree.Apply(true)

	// reload default configuration
	server.RunVyosScriptAsUserVyos("load /opt/vyatta/etc/config.boot.default\nsave")
}

func configureVyos()  {
	resetVyos()

	mgmtNic := bootstrapInfo["managementNic"].(map[string]interface{})
	if mgmtNic == nil {
		panic(errors.New("no field 'managementNic' in bootstrap info"))
	}

	eth0 := &nic{ name: "eth0" }
	var ok bool
	eth0.mac, ok = mgmtNic["mac"].(string); utils.PanicIfError(ok, errors.New("cannot find 'mac' field for the management nic"))
	eth0.netmask, ok = mgmtNic["netmask"].(string); utils.PanicIfError(ok, errors.New("cannot find 'netmask' field for the management nic"))
	eth0.ip, ok = mgmtNic["ip"].(string); utils.PanicIfError(ok, errors.New("cannot find 'ip' field for the management nic"))
	eth0.isDefaultRoute = mgmtNic["isDefaultRoute"].(bool)
	eth0.gateway = mgmtNic["gateway"].(string)
	nics[eth0.name] = eth0

	otherNics := bootstrapInfo["additionalNics"].([]interface{})
	if otherNics != nil {
		for _, o := range otherNics {
			onic := o.(map[string]interface{})
			n := &nic{}
			n.name, ok = onic["deviceName"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'deviceName' field for the nic"))
			n.mac, ok = onic["mac"].(string); utils.PanicIfError(ok, errors.New("cannot find 'mac' field for the nic"))
			n.netmask, ok = onic["netmask"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'netmask' field for the nic[name:%s]", n.name))
			n.ip, ok = onic["ip"].(string); utils.PanicIfError(ok, fmt.Errorf("cannot find 'ip' field for the nic[name:%s]", n.name))
			n.gateway = onic["gateway"].(string)
			n.isDefaultRoute = onic["isDefaultRoute"].(bool)
			nics[n.name] = n
		}
	}

	type deviceName struct {
		expected string
		actual string
		swap string
	}

	devNames := make([]*deviceName, 0)

	// check integrity of nics
	for _, nic := range nics {
		utils.Assertf(nic.name != "", "name cannot be empty[mac:%s]", nic.mac)
		utils.Assertf(nic.ip != "", "ip cannot be empty[nicname: %s]", nic.name)
		utils.Assertf(nic.gateway != "", "gateway cannot be empty[nicname:%s]", nic.name)
		utils.Assertf(nic.netmask != "", "netmask cannot be empty[nicname:%s]", nic.name)
		utils.Assertf(nic.mac != "", "mac cannot be empty[nicname:%s]", nic.name)

		nicname, err := utils.GetNicNameByMac(nic.mac); utils.PanicOnError(err)
		if nicname != nic.name {
			devNames = append(devNames, &deviceName{
				expected: nic.name,
				actual: nicname,
			})
		}
	}

	if len(devNames) != 0 {
		// shutdown links and change to temporary names
		cmds := make([]string, 0)
		for i, devname := range devNames {
			devnum := 1000 + i

			devname.swap = fmt.Sprintf("eth%v", devnum)
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v down", devname.actual))
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v name %v", devname.actual, devname.swap))
		}

		b := utils.Bash{
			Command: strings.Join(cmds, "\n"),
		}

		b.Run()
		b.PanicIfError()

		// change temporary names to real names and bring up links
		cmds = make([]string, 0)
		for _, devname := range devNames {
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v name %v", devname.swap, devname.expected))
			cmds = append(cmds, fmt.Sprintf("ip link set dev %v up", devname.expected))
		}

		b = utils.Bash{
			Command: strings.Join(cmds, "\n"),
		}

		b.Run()
		b.PanicIfError()
	}

	vyos := server.NewParserFromShowConfiguration()
	tree := vyos.Tree

	sshkey := bootstrapInfo["publicKey"].(string)
	utils.Assert(sshkey != "", "cannot find 'publicKey' in bootstrap info")
	sshkeyparts := strings.Split(sshkey, " ")
	sshtype := sshkeyparts[0]
	key := sshkeyparts[1]
	id := sshkeyparts[2]

	tree.Setf("system login user vyos authentication public-keys %s key %s", id, key)
	tree.Setf("system login user vyos authentication public-keys %s type %s", id, sshtype)

	setNic := func(nic *nic) {
		cidr, err := utils.NetmaskToCIDR(nic.netmask); utils.PanicOnError(err)
		//tree.Setf("interfaces ethernet %s hw-id %s", nic.name, nic.mac)
		tree.Setf("interfaces ethernet %s address %s", nic.name, fmt.Sprintf("%v/%v", nic.ip, cidr))
		tree.Setf("interfaces ethernet %s duplex auto", nic.name)
		tree.Setf("interfaces ethernet %s smp_affinity auto", nic.name)
		tree.Setf("interfaces ethernet %s speed auto", nic.name)
		if nic.isDefaultRoute {
			tree.Setf("system gateway-address %v", nic.gateway)
		}
	}

	sshport := bootstrapInfo["sshPort"].(float64)
	utils.Assert(sshport != 0, "sshport not found in bootstrap info")
	tree.Setf("service ssh port %v", int(sshport))
	tree.Setf("service ssh listen-address %v", eth0.ip)

	// configure firewall
	for _, nic := range nics {
		setNic(nic)

		tree.SetFirewallOnInterface(nic.name, "local",
			"action accept",
			"state established enable",
			"state related enable",
			fmt.Sprintf("destination address %v", nic.ip),
		)
		tree.SetFirewallOnInterface(nic.name, "local",
			"action accept",
			"protocol icmp",
			fmt.Sprintf("destination address %v", nic.ip),
		)

		if !nic.isDefaultRoute && nic.name != "eth0" {
			// for nic connecting to the guest network
			// make its FORWARDING chain open
			// here in = "-A FORWARDING -i nic.name -j ACCEPT"
			tree.SetFirewallOnInterface(nic.name, "in",
				"action accept",
				"state established enable",
				"state related enable",
				"state new enable",
			)
		} else {
			tree.SetFirewallOnInterface(nic.name, "in",
				"action accept",
				"state established enable",
				"state related enable",
			)
		}

		tree.SetFirewallOnInterface(nic.name, "in",
			"action accept",
			"protocol icmp",
		)

		// only allow ssh traffic on eth0, disable on others
		if nic.name == "eth0" {
			tree.SetFirewallOnInterface(nic.name, "local",
				fmt.Sprintf("destination port %v", int(sshport)),
				fmt.Sprintf("destination address %v", nic.ip),
				"protocol tcp",
				"action accept",
			)
		} else {
			tree.SetFirewallOnInterface(nic.name, "local",
				fmt.Sprintf("destination port %v", int(sshport)),
				fmt.Sprintf("destination address %v", nic.ip),
				"protocol tcp",
				"action reject",
			)
		}

		tree.SetFirewallDefaultAction(nic.name, "local", "reject")
		tree.SetFirewallDefaultAction(nic.name, "in", "reject")

		tree.AttachFirewallToInterface(nic.name, "local")
		tree.AttachFirewallToInterface(nic.name, "in")
	}

	tree.Set("system time-zone Asia/Shanghai")

	password := bootstrapInfo["vyosPassword"]; utils.Assert(password != "", "vyosPassword cannot be empty")
	if !isOnVMwareHypervisor() {
		tree.Setf("system login user vyos authentication plaintext-password %v", password)
	}

	tree.Apply(true)

	arping := func(nicname, ip, gateway string) {
		b := utils.Bash{ Command: fmt.Sprintf("arping -A -U -c 1 -I %s -s %s %s", nicname, ip, gateway) }
		b.Run()
	}

	// arping to advocate our mac addresses
	arping("eth0", eth0.ip, eth0.gateway)
	for _, nic := range nics {
		arping(nic.name, nic.ip, nic.gateway)
	}
}

func startZvr()  {
	b := utils.Bash{
		Command: "/etc/init.d/zstack-virtualrouteragent restart",
	}
	b.Run()
	b.PanicIfError()
}

func main() {
	utils.InitLog("/home/vyos/zvr/zvrboot.log", false)
	waitIptablesServiceOnline()
	if isOnVMwareHypervisor() {
		parseEsxBootInfo()
	} else {
		waitVirtioPortOnline()
		parseKvmBootInfo()
	}
	configureVyos()
	startZvr()
	log.Debugf("successfully configured the sysmtem and bootstrap the zstack virtual router agents")
}
<|endoftext|>


<commit_before>package main

/*
#cgo CFLAGS: -I/usr/include
#cgo LDFLAGS: -lcwiid -Lcwiid/libcwiid/libcwiid.a
#include "racerwiigo.h"
#include <stdlib.h>
#include <cwiid.h>
#include <time.h>
#include <bluetooth/bluetooth.h>
*/
import "C"

import (
	"encoding/csv"
	"fmt"
	"html/template"
	"net/http"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unsafe"
)

var buttons = []_Ctype_uint16_t{ // only HOME and A buttons are used for this program
	C.CWIID_BTN_A,
	//C.CWIID_BTN_B,
	//C.CWIID_BTN_1,
	//C.CWIID_BTN_2,
	//C.CWIID_BTN_MINUS,
	C.CWIID_BTN_HOME,
	//C.CWIID_BTN_LEFT,
	//C.CWIID_BTN_RIGHT,
	//C.CWIID_BTN_DOWN,
	//C.CWIID_BTN_UP,
	//C.CWIID_BTN_PLUS,
}

var buttonStatus []bool

var buttonChan chan _Ctype_uint16_t
var exit chan bool
var callback = goCwiidCallback // so it's not garbage collected
var errCallback = goErrCallback
var start *time.Time
var entries map[uint]*Entry // map of Bib #s
var results []*Result
var raceResultsTemplate *template.Template
var useWiimote = false

type HumanDuration time.Duration

type Entry struct {
	Bib    uint
	Fname  string
	Lname  string
	Male   bool
	Age    uint
	Result *Result
}

type Result struct {
	Time  HumanDuration
	Place uint
	Entry *Entry
}

func (hd HumanDuration) String() string {
	seconds := time.Duration(hd).Seconds()
	seconds -= float64(time.Duration(hd) / time.Minute * 60)
	return fmt.Sprintf("%#02d:%#02d:%05.2f", time.Duration(hd)/time.Hour, time.Duration(hd)/time.Minute%60, seconds)
}

func (hd HumanDuration) Clock() string {
	return fmt.Sprintf("%#02d:%#02d:%02d", time.Duration(hd)/time.Hour, time.Duration(hd)/time.Minute%60, time.Duration(hd)/time.Second%60)
}

//export goCwiidCallback
func goCwiidCallback(wm unsafe.Pointer, a int, mesg *C.struct_cwiid_btn_mesg, tp unsafe.Pointer) {
	//defer C.free(unsafe.Pointer(mesg))
	var messages []C.struct_cwiid_btn_mesg
	sliceHeader := (*reflect.SliceHeader)((unsafe.Pointer(&messages)))
	sliceHeader.Cap = a
	sliceHeader.Len = a
	sliceHeader.Data = uintptr(unsafe.Pointer(mesg))
	for _, m := range messages {
		if m._type != C.CWIID_MESG_BTN {
			exit <- true
			continue
		}
		//fmt.Printf("Received message - %#v\n", m)
		for x, button := range buttons {
			if m.buttons&button == button {
				if !buttonStatus[x] {
					buttonChan <- button
					buttonStatus[x] = true
				}
			} else {
				buttonStatus[x] = false
			}
		}
	}
}

//export goErrCallback
func goErrCallback(wm unsafe.Pointer, char *C.char, ap unsafe.Pointer) {
	//func goErrCallback(wm *C.cwiid_wiimote_t, char *C.char, ap C.va_list) {s
	str := C.GoString(char)
	switch str {
	case "No Bluetooth interface found":
		fallthrough
	case "no such device":
		fmt.Printf("No Bluetooth device found\n")
		os.Exit(1)
	case "Socket connect error (control channel)":
		fallthrough
	case "No wiimotes found":
		exit <- true
	default:
		fmt.Printf("Inside error calback - %s\n", str)
	}
}

func uploadRacers(w http.ResponseWriter, r *http.Request) {
	reader, err := r.MultipartReader()
	if err != nil {
		fmt.Fprintf(w, "Error getting Reader - %s", err)
		return
	}
	part, err := reader.NextPart()
	if err != nil {
		fmt.Fprintf(w, "Error getting Part - %s", err)
		return
	}
	csvIn := csv.NewReader(part)
	rawEntries, err := csvIn.ReadAll()
	if err != nil {
		fmt.Fprintf(w, "Error Reading CSV file - %s", err)
		return
	}
	if len(rawEntries) <= 1 {
		fmt.Fprintf(w, "Either blank file or only supplied the header row")
		return
	}
	// make the map and unlink all previous relationships (if any)
	entries = make(map[uint]*Entry)
	for _, result := range results {
		result.Entry = nil
	}
	for row := 1; row < len(rawEntries); row++ {
		entry := new(Entry)
		for col := range rawEntries[row] {
			switch rawEntries[0][col] {
			case "Fname":
				entry.Fname = rawEntries[row][col]
			case "Lname":
				entry.Lname = rawEntries[row][col]
			case "Age":
				tmpAge, _ := strconv.Atoi(rawEntries[row][col])
				entry.Age = uint(tmpAge)
			case "Gender":
				entry.Male = (rawEntries[row][col] == "M")
			case "Bib":
				tmpBib, _ := strconv.Atoi(rawEntries[row][col])
				entry.Bib = uint(tmpBib)
			default:
				fmt.Printf("Field %s not imported, dropping\n", rawEntries[0][col])
			}
		}
		fmt.Printf("Read entry - %v", entry)
		fmt.Printf("CSV Input = %v", rawEntries[row])
		if entry.Bib != 0 {
			entries[entry.Bib] = entry
		} else {
			fmt.Printf("Skipping due to no bib assigned - %v", entry)
		}
	}
	http.Redirect(w, r, "/admin", 301)
	return
}

func bibHandler(w http.ResponseWriter, r *http.Request) {
	next, err := strconv.Atoi(r.FormValue("next"))
	if err != nil || next > len(results) {
		fmt.Fprintf(w, "Error %s getting next", err)
		return
	}
	tempBib, err := strconv.Atoi(r.FormValue("bib"))
	if tempBib < 0 {
		fmt.Fprintf(w, "Cannot assign a negative bib number of %d", tempBib)
		return
	}
	bib := uint(tempBib)
	if err == nil {
		if _, ok := entries[bib]; ok {
			results[next-1].Entry = entries[bib]
			entries[bib].Result = results[next-1]
			fmt.Printf("Set bib for place %d to %d", next, bib)
		} else {
			fmt.Fprintf(w, "Bib number %d was not assigned to anyone.", bib)
			return
		}
	} else {
		fmt.Printf("Error %s setting bib for place %d to %d", err, next, bib)
	}
	http.Redirect(w, r, "/admin", 301)
	return
}

func handler(w http.ResponseWriter, r *http.Request) {
	data := map[string]interface{}{"Racers": results}
	if start != nil {
		diff := time.Since(*start)
		data["Start"] = start.Format("3:04:05")
		data["Time"] = HumanDuration(diff).Clock()
		data["Seconds"] = fmt.Sprintf("%.0f", diff.Seconds())
		data["NextUpdate"] = diff / time.Millisecond % 1000
		if strings.HasSuffix(r.RequestURI, "admin") {
			data["Admin"] = true
			for x := range results {
				if results[x].Entry == nil {
					data["Next"] = results[x].Place
					break
				}
			}
		}
	}
	raceResultsTemplate, err := template.ParseFiles("raceResults.template")
	if err != nil {
		fmt.Fprintf(w, "Error parsing template - %s", err)
	} else {
		err = raceResultsTemplate.Execute(w, data)
		if err != nil {
			fmt.Fprintf(w, "Error executing template - %s", err)
		}
	}
}

func main() {
	var err error
	raceResultsTemplate, err = template.ParseFiles("raceResults.template")
	if err != nil {
		fmt.Printf("Error parsing template! - %s\n", err)
		return
	}
	go raceFunc()
	http.HandleFunc("www.raceresults.org/", handler)
	http.HandleFunc("www.raceresults.org/admin", handler)
	http.HandleFunc("www.raceresults.org/bib", bibHandler)
	http.HandleFunc("www.raceresults.org/uploadRacers", uploadRacers)
	http.Handle("/", http.RedirectHandler("http://www.raceresults.org/", 307))
	err = http.ListenAndServe(":80", nil)
	if err != nil {
		fmt.Printf("Error starting http server! - %s\n", err)
		err = http.ListenAndServe(":8080", nil)
		if err != nil {
			fmt.Printf("Error starting http server! - %s\n", err)
			return
		}
	}
}

func raceFunc() {
	buttonStatus = make([]bool, len(buttons))
	var bdaddr C.bdaddr_t
	var wm *C.struct_cwiid_wiimote_t
	buttonChan = make(chan _Ctype_uint16_t, 1)
	exit = make(chan bool, 1)
	ticker := time.NewTicker(time.Second)
	results = make([]*Result, 0, 1024)
	var err error
	if useWiimote {
		val, err := C.cwiid_set_err(C.getErrCallback())
		if val != 0 || err != nil {
			fmt.Printf("Error setting the callback to catch errors - %d - %v", val, err)
			os.Exit(1)
		}
	}
	for {
	outer:
		for {
			// clear both channels
			select {
			case <-buttonChan:
			case <-exit:
			default:
				break outer
			}
		}
		fmt.Println("Press 1&2 on the Wiimote now")
		if useWiimote {
			wm, err = C.cwiid_open(&bdaddr, 0)
			if err != nil {
				fmt.Errorf("cwiid_open: %v\n", err)
				continue
			}

			res, err := C.cwiid_command(wm, C.CWIID_CMD_RPT_MODE, C.CWIID_RPT_BTN)
			if res != 0 || err != nil {
				fmt.Printf("Result of command = %d - %v\n", res, err)
			}

			res, err = C.cwiid_set_mesg_callback(wm, C.getCwiidCallback())
			if res != 0 || err != nil {
				fmt.Printf("Result of callback = %d - %v\n", res, err)
			}
			res, err = C.cwiid_enable(wm, C.CWIID_FLAG_MESG_IFC)
			if res != 0 || err != nil {
				fmt.Printf("Result of enable = %d - %v\n", res, err)
			}

			res, err = C.cwiid_set_led(wm, C.CWIID_LED4_ON)
			if res != 0 || err != nil {
				fmt.Printf("Set led result = %d\n", res)
				fmt.Errorf("Err = %v", err)
			}
		} else { // simulate the pressing of the wiimote A button for testing
			go func() {
				simulButton := time.NewTicker(time.Second * 10)
				buttonChan <- C.CWIID_BTN_A // start race immediately
				//buttonChan <- C.CWIID_BTN_A // first runner! :)
				for {
					select {
					case <-simulButton.C:
						buttonChan <- C.CWIID_BTN_A
					}
				}
			}()
		}
	loop:
		for {
			select {
			case <-exit:
				fmt.Println("Wiimote lost connection!")
				break loop
			case button := <-buttonChan:
				switch button {
				case C.CWIID_BTN_A:
					if start == nil {
						start = new(time.Time)
						*start = time.Now()
						fmt.Printf("Race started @ %s\n", start.Format("3:04:05"))
						results = results[:0]
					} else {
						place := len(results)
						results = append(results, &Result{Place: uint(place + 1), Time: HumanDuration(time.Now().Sub(*start))})
						fmt.Printf("#%d - %s\n", results[place].Place, results[place].Time)
					}
				//case C.CWIID_BTN_B:
				//	fmt.Println("B")
				//case C.CWIID_BTN_1:
				//	fmt.Println("1")
				//case C.CWIID_BTN_2:
				//	fmt.Println("2")
				//case C.CWIID_BTN_MINUS:
				//	fmt.Println("Minus")
				case C.CWIID_BTN_HOME:
					fmt.Println("Race finished!")
					return
					//case C.CWIID_BTN_LEFT:
					//	fmt.Println("Left")
					//case C.CWIID_BTN_RIGHT:
					//	fmt.Println("Right")
					//case C.CWIID_BTN_DOWN:
					//	fmt.Println("Down")
					//case C.CWIID_BTN_UP:
					//	fmt.Println("Up")
					//case C.CWIID_BTN_PLUS:
					//	fmt.Println("Plus")
				}
			case now := <-ticker.C:
				if start != nil {
					diff := HumanDuration(now.Sub(*start))
					fmt.Println(diff)
				}
				// update the clock
			}
		}
	}
}
<commit_msg>making the captive portal url (http://raceresults/) better suited for an internal network - I didn't realize DefaultServeMux would work without a "proper" dns address<commit_after>package main

/*
#cgo CFLAGS: -I/usr/include
#cgo LDFLAGS: -lcwiid -Lcwiid/libcwiid/libcwiid.a
#include "racerwiigo.h"
#include <stdlib.h>
#include <cwiid.h>
#include <time.h>
#include <bluetooth/bluetooth.h>
*/
import "C"

import (
	"encoding/csv"
	"fmt"
	"html/template"
	"net/http"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unsafe"
)

var buttons = []_Ctype_uint16_t{ // only HOME and A buttons are used for this program
	C.CWIID_BTN_A,
	//C.CWIID_BTN_B,
	//C.CWIID_BTN_1,
	//C.CWIID_BTN_2,
	//C.CWIID_BTN_MINUS,
	C.CWIID_BTN_HOME,
	//C.CWIID_BTN_LEFT,
	//C.CWIID_BTN_RIGHT,
	//C.CWIID_BTN_DOWN,
	//C.CWIID_BTN_UP,
	//C.CWIID_BTN_PLUS,
}

var buttonStatus []bool

var buttonChan chan _Ctype_uint16_t
var exit chan bool
var callback = goCwiidCallback // so it's not garbage collected
var errCallback = goErrCallback
var start *time.Time
var entries map[uint]*Entry // map of Bib #s
var results []*Result
var raceResultsTemplate *template.Template
var useWiimote = false

type HumanDuration time.Duration

type Entry struct {
	Bib    uint
	Fname  string
	Lname  string
	Male   bool
	Age    uint
	Result *Result
}

type Result struct {
	Time  HumanDuration
	Place uint
	Entry *Entry
}

func (hd HumanDuration) String() string {
	seconds := time.Duration(hd).Seconds()
	seconds -= float64(time.Duration(hd) / time.Minute * 60)
	return fmt.Sprintf("%#02d:%#02d:%05.2f", time.Duration(hd)/time.Hour, time.Duration(hd)/time.Minute%60, seconds)
}

func (hd HumanDuration) Clock() string {
	return fmt.Sprintf("%#02d:%#02d:%02d", time.Duration(hd)/time.Hour, time.Duration(hd)/time.Minute%60, time.Duration(hd)/time.Second%60)
}

//export goCwiidCallback
func goCwiidCallback(wm unsafe.Pointer, a int, mesg *C.struct_cwiid_btn_mesg, tp unsafe.Pointer) {
	//defer C.free(unsafe.Pointer(mesg))
	var messages []C.struct_cwiid_btn_mesg
	sliceHeader := (*reflect.SliceHeader)((unsafe.Pointer(&messages)))
	sliceHeader.Cap = a
	sliceHeader.Len = a
	sliceHeader.Data = uintptr(unsafe.Pointer(mesg))
	for _, m := range messages {
		if m._type != C.CWIID_MESG_BTN {
			exit <- true
			continue
		}
		//fmt.Printf("Received message - %#v\n", m)
		for x, button := range buttons {
			if m.buttons&button == button {
				if !buttonStatus[x] {
					buttonChan <- button
					buttonStatus[x] = true
				}
			} else {
				buttonStatus[x] = false
			}
		}
	}
}

//export goErrCallback
func goErrCallback(wm unsafe.Pointer, char *C.char, ap unsafe.Pointer) {
	//func goErrCallback(wm *C.cwiid_wiimote_t, char *C.char, ap C.va_list) {s
	str := C.GoString(char)
	switch str {
	case "No Bluetooth interface found":
		fallthrough
	case "no such device":
		fmt.Printf("No Bluetooth device found\n")
		os.Exit(1)
	case "Socket connect error (control channel)":
		fallthrough
	case "No wiimotes found":
		exit <- true
	default:
		fmt.Printf("Inside error calback - %s\n", str)
	}
}

func uploadRacers(w http.ResponseWriter, r *http.Request) {
	reader, err := r.MultipartReader()
	if err != nil {
		fmt.Fprintf(w, "Error getting Reader - %s", err)
		return
	}
	part, err := reader.NextPart()
	if err != nil {
		fmt.Fprintf(w, "Error getting Part - %s", err)
		return
	}
	csvIn := csv.NewReader(part)
	rawEntries, err := csvIn.ReadAll()
	if err != nil {
		fmt.Fprintf(w, "Error Reading CSV file - %s", err)
		return
	}
	if len(rawEntries) <= 1 {
		fmt.Fprintf(w, "Either blank file or only supplied the header row")
		return
	}
	// make the map and unlink all previous relationships (if any)
	entries = make(map[uint]*Entry)
	for _, result := range results {
		result.Entry = nil
	}
	for row := 1; row < len(rawEntries); row++ {
		entry := new(Entry)
		for col := range rawEntries[row] {
			switch rawEntries[0][col] {
			case "Fname":
				entry.Fname = rawEntries[row][col]
			case "Lname":
				entry.Lname = rawEntries[row][col]
			case "Age":
				tmpAge, _ := strconv.Atoi(rawEntries[row][col])
				entry.Age = uint(tmpAge)
			case "Gender":
				entry.Male = (rawEntries[row][col] == "M")
			case "Bib":
				tmpBib, _ := strconv.Atoi(rawEntries[row][col])
				entry.Bib = uint(tmpBib)
			default:
				fmt.Printf("Field %s not imported, dropping\n", rawEntries[0][col])
			}
		}
		fmt.Printf("Read entry - %v", entry)
		fmt.Printf("CSV Input = %v", rawEntries[row])
		if entry.Bib != 0 {
			entries[entry.Bib] = entry
		} else {
			fmt.Printf("Skipping due to no bib assigned - %v", entry)
		}
	}
	http.Redirect(w, r, "/admin", 301)
	return
}

func bibHandler(w http.ResponseWriter, r *http.Request) {
	next, err := strconv.Atoi(r.FormValue("next"))
	if err != nil || next > len(results) {
		fmt.Fprintf(w, "Error %s getting next", err)
		return
	}
	tempBib, err := strconv.Atoi(r.FormValue("bib"))
	if tempBib < 0 {
		fmt.Fprintf(w, "Cannot assign a negative bib number of %d", tempBib)
		return
	}
	bib := uint(tempBib)
	if err == nil {
		if _, ok := entries[bib]; ok {
			results[next-1].Entry = entries[bib]
			entries[bib].Result = results[next-1]
			fmt.Printf("Set bib for place %d to %d", next, bib)
		} else {
			fmt.Fprintf(w, "Bib number %d was not assigned to anyone.", bib)
			return
		}
	} else {
		fmt.Printf("Error %s setting bib for place %d to %d", err, next, bib)
	}
	http.Redirect(w, r, "/admin", 301)
	return
}

func handler(w http.ResponseWriter, r *http.Request) {
	data := map[string]interface{}{"Racers": results}
	if start != nil {
		diff := time.Since(*start)
		data["Start"] = start.Format("3:04:05")
		data["Time"] = HumanDuration(diff).Clock()
		data["Seconds"] = fmt.Sprintf("%.0f", diff.Seconds())
		data["NextUpdate"] = diff / time.Millisecond % 1000
		if strings.HasSuffix(r.RequestURI, "admin") {
			data["Admin"] = true
			for x := range results {
				if results[x].Entry == nil {
					data["Next"] = results[x].Place
					break
				}
			}
		}
	}
	raceResultsTemplate, err := template.ParseFiles("raceResults.template")
	if err != nil {
		fmt.Fprintf(w, "Error parsing template - %s", err)
	} else {
		err = raceResultsTemplate.Execute(w, data)
		if err != nil {
			fmt.Fprintf(w, "Error executing template - %s", err)
		}
	}
}

func main() {
	var err error
	raceResultsTemplate, err = template.ParseFiles("raceResults.template")
	if err != nil {
		fmt.Printf("Error parsing template! - %s\n", err)
		return
	}
	go raceFunc()
	http.HandleFunc("raceresults/", handler)
	http.HandleFunc("raceresults/admin", handler)
	http.HandleFunc("raceresults/bib", bibHandler)
	http.HandleFunc("raceresults/uploadRacers", uploadRacers)
	http.Handle("/", http.RedirectHandler("http://raceresults/", 307))
	err = http.ListenAndServe(":80", nil)
	if err != nil {
		fmt.Printf("Error starting http server! - %s\n", err)
		err = http.ListenAndServe(":8080", nil)
		if err != nil {
			fmt.Printf("Error starting http server! - %s\n", err)
			return
		}
	}
}

func raceFunc() {
	buttonStatus = make([]bool, len(buttons))
	var bdaddr C.bdaddr_t
	var wm *C.struct_cwiid_wiimote_t
	buttonChan = make(chan _Ctype_uint16_t, 1)
	exit = make(chan bool, 1)
	ticker := time.NewTicker(time.Second)
	results = make([]*Result, 0, 1024)
	var err error
	if useWiimote {
		val, err := C.cwiid_set_err(C.getErrCallback())
		if val != 0 || err != nil {
			fmt.Printf("Error setting the callback to catch errors - %d - %v", val, err)
			os.Exit(1)
		}
	}
	for {
	outer:
		for {
			// clear both channels
			select {
			case <-buttonChan:
			case <-exit:
			default:
				break outer
			}
		}
		fmt.Println("Press 1&2 on the Wiimote now")
		if useWiimote {
			wm, err = C.cwiid_open(&bdaddr, 0)
			if err != nil {
				fmt.Errorf("cwiid_open: %v\n", err)
				continue
			}

			res, err := C.cwiid_command(wm, C.CWIID_CMD_RPT_MODE, C.CWIID_RPT_BTN)
			if res != 0 || err != nil {
				fmt.Printf("Result of command = %d - %v\n", res, err)
			}

			res, err = C.cwiid_set_mesg_callback(wm, C.getCwiidCallback())
			if res != 0 || err != nil {
				fmt.Printf("Result of callback = %d - %v\n", res, err)
			}
			res, err = C.cwiid_enable(wm, C.CWIID_FLAG_MESG_IFC)
			if res != 0 || err != nil {
				fmt.Printf("Result of enable = %d - %v\n", res, err)
			}

			res, err = C.cwiid_set_led(wm, C.CWIID_LED4_ON)
			if res != 0 || err != nil {
				fmt.Printf("Set led result = %d\n", res)
				fmt.Errorf("Err = %v", err)
			}
		} else { // simulate the pressing of the wiimote A button for testing
			go func() {
				simulButton := time.NewTicker(time.Second * 10)
				buttonChan <- C.CWIID_BTN_A // start race immediately
				//buttonChan <- C.CWIID_BTN_A // first runner! :)
				for {
					select {
					case <-simulButton.C:
						buttonChan <- C.CWIID_BTN_A
					}
				}
			}()
		}
	loop:
		for {
			select {
			case <-exit:
				fmt.Println("Wiimote lost connection!")
				break loop
			case button := <-buttonChan:
				switch button {
				case C.CWIID_BTN_A:
					if start == nil {
						start = new(time.Time)
						*start = time.Now()
						fmt.Printf("Race started @ %s\n", start.Format("3:04:05"))
						results = results[:0]
					} else {
						place := len(results)
						results = append(results, &Result{Place: uint(place + 1), Time: HumanDuration(time.Now().Sub(*start))})
						fmt.Printf("#%d - %s\n", results[place].Place, results[place].Time)
					}
				//case C.CWIID_BTN_B:
				//	fmt.Println("B")
				//case C.CWIID_BTN_1:
				//	fmt.Println("1")
				//case C.CWIID_BTN_2:
				//	fmt.Println("2")
				//case C.CWIID_BTN_MINUS:
				//	fmt.Println("Minus")
				case C.CWIID_BTN_HOME:
					fmt.Println("Race finished!")
					return
					//case C.CWIID_BTN_LEFT:
					//	fmt.Println("Left")
					//case C.CWIID_BTN_RIGHT:
					//	fmt.Println("Right")
					//case C.CWIID_BTN_DOWN:
					//	fmt.Println("Down")
					//case C.CWIID_BTN_UP:
					//	fmt.Println("Up")
					//case C.CWIID_BTN_PLUS:
					//	fmt.Println("Plus")
				}
			case now := <-ticker.C:
				if start != nil {
					diff := HumanDuration(now.Sub(*start))
					fmt.Println(diff)
				}
				// update the clock
			}
		}
	}
}
<|endoftext|>


<commit_before>// Package power provides interface to power managemnt peripheral.
package power

import (
	"mmio"
	"unsafe"

	"nrf5/hal/internal/mmap"
	"nrf5/hal/te"
)

type Periph struct {
	te.Regs

	resetreas mmio.U32     // 0x400
	_         [9]mmio.U32  //
	ramstatus mmio.U32     // 0x428
	_         [53]mmio.U32 //
	systemoff mmio.U32     // 0x500
	_         [3]mmio.U32  //
	pofcon    mmio.U32     // 0x510
	_         [2]mmio.U32  //
	gpregret  [2]mmio.U32  // 0x51C
	ramon     mmio.U32     // 0x524
	_         [7]mmio.U32  //
	reset     mmio.U32     // 0x544
	_         [3]mmio.U32  //
	ramonb    mmio.U32     // 0x554
	_         [8]mmio.U32  //
	dcdcen    mmio.U32     // 0x578
	_         [225]mmio.U32
	ram       [8]struct{ power, powerset, powerclr mmio.U32 }
}

//emgo:const
var POWER = (*Periph)(unsafe.Pointer(mmap.APB_BASE))

type Task byte

const (
	CONSTLAT Task = 0x78 // Enable constant latency mode.
	LOWPWR   Task = 0x7C // Enable low power mode (variable latency).
)

type Event byte

const (
	POFWARN    Event = 0x08 // Power failure warning.
	SLEEPENTER Event = 0x14 // CPU entered WFI/WFE sleep (nRF52).
	SLEEPEXIT  Event = 0x18 // CPU exited WFI/WFE sleep (nRF52).
)

func (p *Periph) Task(t Task) *te.Task    { return p.Regs.Task(int(t)) }
func (p *Periph) Event(e Event) *te.Event { return p.Regs.Event(int(e)) }

// ResetReas is a bitfield that describes reset reason.
type ResetReas uint32

const (
	RESETPIN ResetReas = 1 << 0  // Reset from pin-reset.
	DOG      ResetReas = 1 << 1  // Reset from watchdog.
	SREQ     ResetReas = 1 << 2  // Reset from AIRCR.SYSRESETREQ.
	LOCKUP   ResetReas = 1 << 3  // Reset from CPU lock-up.
	OFF      ResetReas = 1 << 16 // Wake up from OFF mode by GPIO DETECT.
	LPCOMP   ResetReas = 1 << 17 // Wake up from OFF mode by LPCOMP ANADETECT.
	DIF      ResetReas = 1 << 18 // Wake up from OFF mode by debug interface.
	NFC      ResetReas = 1 << 19 // Wake up from OFF mode by NFC.
)

// LoadRESETREAS returns reset reason bits.
func (p *Periph) LoadRESETREAS() ResetReas {
	return ResetReas(p.resetreas.Load())
}

// ClearRESETREAS clears reset reason bits specified by mask.
func (p *Periph) ClearRESETREAS(mask ResetReas) {
	p.resetreas.Store(uint32(mask))
}

// RAMStatus is a bitfield that describes RAM blocks status.
type RAMStatus byte

const (
	RAMBLOCK0 RAMStatus = 1 << 0 // RAM block 0 is powered up.
	RAMBLOCK1 RAMStatus = 1 << 1 // RAM block 1 is powered up.
	RAMBLOCK2 RAMStatus = 1 << 2 // RAM block 2 is powered up.
	RAMBLOCK3 RAMStatus = 1 << 3 // RAM block 3 is powered up.
)

// LoadRAMSTATUS returns bitfield that describes status of RAM blocks.
func (p *Periph) LoadRAMSTATUS() RAMStatus {
	return RAMStatus(p.ramstatus.Load())
}

// SetSYSTEMOFF sets system into OFF state.
func (p *Periph) SetSYSTEMOFF() {
	p.systemoff.Store(1)
}

// POFCon is power failure comparator configuration.
type POFCon byte

const (
	POF       POFCon = 1 << 0  // Set if power failure comparoator is enabled.
	THRESHOLD POFCon = 15 << 1 // Power failure comparator threshold mask.

	// Power failure comparoator thresholds.

	V2_1 POFCon = 0 << 1 // Threshold: 2.1 V (nrF51).
	V2_3 POFCon = 1 << 1 // Threshold: 2.3 V (nrF51).
	V2_5 POFCon = 2 << 1 // Threshold: 2.5 V (nrF51).
	V2_7 POFCon = 3 << 1 // Threshold: 2.5 V (nrF51).

	V17 POFCon = 4 << 1  // Threshold: 1.7 V (nRF52).
	V18 POFCon = 5 << 1  // Threshold: 1.8 V (nRF52).
	V19 POFCon = 6 << 1  // Threshold: 1.9 V (nRF52).
	V20 POFCon = 7 << 1  // Threshold: 2.0 V (nRF52).
	V21 POFCon = 8 << 1  // Threshold: 2.1 V (nRF52).
	V22 POFCon = 9 << 1  // Threshold: 2.2 V (nRF52).
	V23 POFCon = 10 << 1 // Threshold: 2.3 V (nRF52).
	V24 POFCon = 11 << 1 // Threshold: 2.4 V (nRF52).
	V25 POFCon = 12 << 1 // Threshold: 2.5 V (nRF52).
	V26 POFCon = 13 << 1 // Threshold: 2.6 V (nRF52).
	V27 POFCon = 14 << 1 // Threshold: 2.7 V (nRF52).
	V28 POFCon = 15 << 1 // Threshold: 2.8 V (nRF52).
)

// LoadPOFCON returns power failure comparoator configuration.
func (p *Periph) LoadPOFCON() POFCon {
	return POFCon(p.pofcon.Load())
}

// StorePOFCON sets power failure comparoator configuration.
func (p *Periph) StorePOFCON(pofcon POFCon) {
	p.pofcon.Store(uint32(pofcon))
}

// GPREGRET returns pointer to n-th general purpose retention register. nRF51
// supports one, nRF52 supports two. Only lowest 8 bits can be used.
func (p *Periph) GPREGRET(n int) *mmio.U32 {
	return &p.gpregret[n]
}

/*
// LoadRAMON
func (p *Periph) LoadRAMON() RAMBlocks {
	return RAMBlocks(p.ramon.Load())
}

func (p *Periph) StoreRAMON(ramon RAMBlocks) {
	p.ramon.Store(uint32(ramon))
}
*/
<commit_msg>nrf5/hal/power: All registers supported.<commit_after>// Package power provides interface to power managemnt peripheral.
package power

import (
	"bits"
	"mmio"
	"unsafe"

	"nrf5/hal/internal/mmap"
	"nrf5/hal/te"
)

type Periph struct {
	te.Regs

	resetreas mmio.U32     // 0x400
	_         [9]mmio.U32  //
	ramstatus mmio.U32     // 0x428
	_         [53]mmio.U32 //
	systemoff mmio.U32     // 0x500
	_         [3]mmio.U32  //
	pofcon    mmio.U32     // 0x510
	_         [2]mmio.U32  //
	gpregret  [2]mmio.U32  // 0x51C
	ramon     mmio.U32     // 0x524
	_         [7]mmio.U32  //
	reset     mmio.U32     // 0x544
	_         [3]mmio.U32  //
	ramonb    mmio.U32     // 0x554
	_         [8]mmio.U32  //
	dcdcen    mmio.U32     // 0x578
	_         [225]mmio.U32
	ram       [8]struct{ power, powerset, powerclr mmio.U32 }
}

//emgo:const
var POWER = (*Periph)(unsafe.Pointer(mmap.APB_BASE))

type Task byte

const (
	CONSTLAT Task = 0x78 // Enable constant latency mode.
	LOWPWR   Task = 0x7C // Enable low power mode (variable latency).
)

type Event byte

const (
	POFWARN    Event = 0x08 // Power failure warning.
	SLEEPENTER Event = 0x14 // CPU entered WFI/WFE sleep (nRF52).
	SLEEPEXIT  Event = 0x18 // CPU exited WFI/WFE sleep (nRF52).
)

func (p *Periph) Task(t Task) *te.Task    { return p.Regs.Task(int(t)) }
func (p *Periph) Event(e Event) *te.Event { return p.Regs.Event(int(e)) }

// ResetReas is a bitfield that describes reset reason.
type ResetReas uint32

const (
	RESETPIN ResetReas = 1 << 0  // Reset from pin-reset.
	DOG      ResetReas = 1 << 1  // Reset from watchdog.
	SREQ     ResetReas = 1 << 2  // Reset from AIRCR.SYSRESETREQ.
	LOCKUP   ResetReas = 1 << 3  // Reset from CPU lock-up.
	OFF      ResetReas = 1 << 16 // Wake up from OFF mode by GPIO DETECT.
	LPCOMP   ResetReas = 1 << 17 // Wake up from OFF mode by LPCOMP ANADETECT.
	DIF      ResetReas = 1 << 18 // Wake up from OFF mode by debug interface.
	NFC      ResetReas = 1 << 19 // Wake up from OFF mode by NFC.
)

// LoadRESETREAS returns reset reason bits.
func (p *Periph) LoadRESETREAS() ResetReas {
	return ResetReas(p.resetreas.Load())
}

// ClearRESETREAS clears reset reason bits specified by mask.
func (p *Periph) ClearRESETREAS(mask ResetReas) {
	p.resetreas.Store(uint32(mask))
}

// RAMBlocks is a bitfield that describes RAM blocks.
type RAMBlocks byte

const (
	RAMBLOCK0 RAMBlocks = 1 << 0 // RAM block 0.
	RAMBLOCK1 RAMBlocks = 1 << 1 // RAM block 1.
	RAMBLOCK2 RAMBlocks = 1 << 2 // RAM block 2.
	RAMBLOCK3 RAMBlocks = 1 << 3 // RAM block 3.
)

// LoadRAMSTATUS returns bitfield that lists RAM blocks that are powered up.
func (p *Periph) LoadRAMSTATUS() RAMBlocks {
	return RAMBlocks(p.ramstatus.Load())
}

// SetSYSTEMOFF sets system into OFF state.
func (p *Periph) SetSYSTEMOFF() {
	p.systemoff.Store(1)
}

// POFCon is power failure comparator configuration.
type POFCon byte

const (
	POF       POFCon = 1 << 0  // Set if power failure comparoator is enabled.
	THRESHOLD POFCon = 15 << 1 // Power failure comparator threshold mask.

	// Power failure comparator thresholds.

	V2_1 POFCon = 0 << 1 // Threshold: 2.1 V (nrF51).
	V2_3 POFCon = 1 << 1 // Threshold: 2.3 V (nrF51).
	V2_5 POFCon = 2 << 1 // Threshold: 2.5 V (nrF51).
	V2_7 POFCon = 3 << 1 // Threshold: 2.5 V (nrF51).

	V17 POFCon = 4 << 1  // Threshold: 1.7 V (nRF52).
	V18 POFCon = 5 << 1  // Threshold: 1.8 V (nRF52).
	V19 POFCon = 6 << 1  // Threshold: 1.9 V (nRF52).
	V20 POFCon = 7 << 1  // Threshold: 2.0 V (nRF52).
	V21 POFCon = 8 << 1  // Threshold: 2.1 V (nRF52).
	V22 POFCon = 9 << 1  // Threshold: 2.2 V (nRF52).
	V23 POFCon = 10 << 1 // Threshold: 2.3 V (nRF52).
	V24 POFCon = 11 << 1 // Threshold: 2.4 V (nRF52).
	V25 POFCon = 12 << 1 // Threshold: 2.5 V (nRF52).
	V26 POFCon = 13 << 1 // Threshold: 2.6 V (nRF52).
	V27 POFCon = 14 << 1 // Threshold: 2.7 V (nRF52).
	V28 POFCon = 15 << 1 // Threshold: 2.8 V (nRF52).
)

// LoadPOFCON returns power failure comparator configuration.
func (p *Periph) LoadPOFCON() POFCon {
	return POFCon(p.pofcon.Load())
}

// StorePOFCON sets power failure comparator configuration.
func (p *Periph) StorePOFCON(pofcon POFCon) {
	p.pofcon.Store(uint32(pofcon))
}

// GPREGRET returns pointer to n-th general purpose retention register. nRF51
// supports one, nRF52 supports two. Only lowest 8 bits can be used.
func (p *Periph) GPREGRET(n int) *mmio.U32 {
	return &p.gpregret[n]
}

// LoadRAMON returns configuration of four RAM blocks. On lists RAM blocks
// that are kept on in system ON mode, retain lists RAM blocks that should be
// retained when RAM block is off.
func (p *Periph) LoadRAMON() (on, retain RAMBlocks) {
	a := p.ramon.Load()
	b := p.ramonb.Load()
	return RAMBlocks(a | b<<2), RAMBlocks(a>>16 | b>>14)
}

// StoreRAMON sets configuration of four RAM blocks. On lists RAM blocks that
// should be kept on in system ON mode, retain lists RAM blocks that should be
// retained in system off mode.
func (p *Periph) StoreRAMON(on, retain RAMBlocks) {
	p.ramon.Store(uint32(on&3) | uint32(retain&3)<<16)
	p.ramonb.Store(uint32(on&12)>>2 | uint32(retain&12)<<14)
}

// LoadRESET reports wheter pin reset is enabled in debug mode (nRF51).
func (p *Periph) LoadRESET() bool {
	return p.reset.Load()&1 != 0
}

// StoreRESET enables/disables pin reset in debug mode (nRF51).
func (p *Periph) StoreRESET(pinreset bool) {
	p.reset.Store(uint32(bits.One(pinreset)))
}

// LoadDCDCEN reports wheter the DC/DC converter is enabled.
func (p *Periph) LoadDCDCEN() bool {
	return p.dcdcen.Load()&1 != 0
}

// StoreDCDCEN enables/disables DC/DC converter.
func (p *Periph) StoreDCDCEN(en bool) {
	p.dcdcen.Store(uint32(bits.One(en)))
}

// RAMPower describes power configuration for two sections of RAM block (nRF52).
type RAMPower uint32

const (
	S0POWER     RAMPower = 1 << 0  // Keep RAM section S0 on in system on mode.
	S1POWER     RAMPower = 1 << 1  // Keep RAM section S1 on in system on mode.
	S0RETENTION RAMPower = 1 << 16 // Keep retention of S0 when RAM is off.
	S1RETENTION RAMPower = 1 << 17 // Keep retention of S1 when RAM is off.
)

// LoadRAMPOWER returns power configuration of RAM block n (nRF52).
func (p *Periph) LoadRAMPOWER(n int) RAMPower {
	return RAMPower(p.ram[n].power.Load())
}

// LoadRAMPOWER power configuration of RAM block n (nRF52).
func (p *Periph) StoreRAMPOWER(n int, val RAMPower) {
	p.ram[n].power.Store(uint32(val))
}

// SetRAMPOWER sets on power configuration of RAM block n according to mask
// (nRF52).
func (p *Periph) SetRAMPOWER(n int, mask RAMPower) {
	p.ram[n].powerset.Store(uint32(mask))
}

// ClearRAMPOWER sets off power configuration of RAM block n according to mask
// (nRF52).
func (p *Periph) ClearRAMPOWER(n int, mask RAMPower) {
	p.ram[n].powerclr.Store(uint32(mask))
}
<|endoftext|>


<commit_before>package command

import (
	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/notification"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/spf13/viper"
)

func init() {
	cmdFilerExport.Run = runFilerExport // break init cycle
}

var cmdFilerExport = &Command{
	UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra",
	Short:     "export meta data in filer store",
	Long: `Iterate the file tree and export all metadata out

	Both source and target store:
        * should be a store name already specified in filer.toml
        * do not need to be enabled state

	If target store is empty, only the directory tree will be listed.

	If target store is "notification", the list of entries will be sent to notification.
	This is usually used to bootstrap filer replication to a remote system.

  `,
}

var (
	// filerExportOutputFile  = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
	filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml")
	filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml, or \"notification\" to export all files to message queue")
	dir                    = cmdFilerExport.Flag.String("dir", "/", "only process files under this directory")
	dirListLimit           = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size")
	dryRun                 = cmdFilerExport.Flag.Bool("dryRun", false, "not actually moving data")
)

type statistics struct {
	directoryCount int
	fileCount      int
}

func runFilerExport(cmd *Command, args []string) bool {

	weed_server.LoadConfiguration("filer", true)
	config := viper.GetViper()

	var sourceStore, targetStore filer2.FilerStore

	for _, store := range filer2.Stores {
		if store.GetName() == *filerExportSourceStore {
			viperSub := config.Sub(store.GetName())
			if err := store.Initialize(viperSub); err != nil {
				glog.Fatalf("Failed to initialize source store for %s: %+v",
					store.GetName(), err)
			} else {
				sourceStore = store
			}
			break
		}
	}

	for _, store := range filer2.Stores {
		if store.GetName() == *filerExportTargetStore {
			viperSub := config.Sub(store.GetName())
			if err := store.Initialize(viperSub); err != nil {
				glog.Fatalf("Failed to initialize target store for %s: %+v",
					store.GetName(), err)
			} else {
				targetStore = store
			}
			break
		}
	}

	if sourceStore == nil {
		glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
		println("existing data sources are:")
		for _, store := range filer2.Stores {
			println("    " + store.GetName())
		}
		return false
	}

	if targetStore == nil && *filerExportTargetStore != "" && *filerExportTargetStore != "notification" {
		glog.Errorf("Failed to find target store %s", *filerExportTargetStore)
		println("existing data sources are:")
		for _, store := range filer2.Stores {
			println("    " + store.GetName())
		}
		return false
	}

	stat := statistics{}

	var fn func(level int, entry *filer2.Entry) error

	if *filerExportTargetStore == "notification" {
		weed_server.LoadConfiguration("notification", false)
		v := viper.GetViper()
		notification.LoadConfiguration(v.Sub("notification"))

		fn = func(level int, entry *filer2.Entry) error {
			printout(level, entry)
			if *dryRun {
				return nil
			}
			return notification.Queue.SendMessage(
				string(entry.FullPath),
				&filer_pb.EventNotification{
					NewEntry: entry.ToProtoEntry(),
				},
			)
		}
	} else if targetStore == nil {
		fn = printout
	} else {
		fn = func(level int, entry *filer2.Entry) error {
			printout(level, entry)
			if *dryRun {
				return nil
			}
			return targetStore.InsertEntry(entry)
		}
	}

	doTraverse(&stat, sourceStore, filer2.FullPath(*dir), 0, fn)

	glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)

	return true
}

func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {

	limit := *dirListLimit
	lastEntryName := ""
	for {
		entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
		if err != nil {
			break
		}
		for _, entry := range entries {
			if fnErr := fn(level, entry); fnErr != nil {
				glog.Errorf("failed to process entry: %s", entry.FullPath)
			}
			if entry.IsDirectory() {
				stat.directoryCount++
				doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
			} else {
				stat.fileCount++
			}
		}
		if len(entries) < limit {
			break
		}
	}
}

func printout(level int, entry *filer2.Entry) error {
	for i := 0; i < level; i++ {
		if i == level-1 {
			print("+-")
		} else {
			print("| ")
		}
	}
	println(entry.FullPath.Name())
	return nil
}
<commit_msg>default "weed export to current enabled store"<commit_after>package command

import (
	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/notification"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/spf13/viper"
)

func init() {
	cmdFilerExport.Run = runFilerExport // break init cycle
}

var cmdFilerExport = &Command{
	UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra",
	Short:     "export meta data in filer store",
	Long: `Iterate the file tree and export all metadata out

	Both source and target store:
        * should be a store name already specified in filer.toml
        * do not need to be enabled state

	If target store is empty, only the directory tree will be listed.

	If target store is "notification", the list of entries will be sent to notification.
	This is usually used to bootstrap filer replication to a remote system.

  `,
}

var (
	// filerExportOutputFile  = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
	filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml, default to currently enabled store")
	filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml, or \"notification\" to export all files to message queue")
	dir                    = cmdFilerExport.Flag.String("dir", "/", "only process files under this directory")
	dirListLimit           = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size")
	dryRun                 = cmdFilerExport.Flag.Bool("dryRun", false, "not actually moving data")
)

type statistics struct {
	directoryCount int
	fileCount      int
}

func runFilerExport(cmd *Command, args []string) bool {

	weed_server.LoadConfiguration("filer", true)
	config := viper.GetViper()

	var sourceStore, targetStore filer2.FilerStore

	for _, store := range filer2.Stores {
		if store.GetName() == *filerExportSourceStore || *filerExportSourceStore == "" && config.GetBool(store.GetName()+".enabled") {
			viperSub := config.Sub(store.GetName())
			if err := store.Initialize(viperSub); err != nil {
				glog.Fatalf("Failed to initialize source store for %s: %+v",
					store.GetName(), err)
			} else {
				sourceStore = store
			}
			break
		}
	}

	for _, store := range filer2.Stores {
		if store.GetName() == *filerExportTargetStore {
			viperSub := config.Sub(store.GetName())
			if err := store.Initialize(viperSub); err != nil {
				glog.Fatalf("Failed to initialize target store for %s: %+v",
					store.GetName(), err)
			} else {
				targetStore = store
			}
			break
		}
	}

	if sourceStore == nil {
		glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
		println("existing data sources are:")
		for _, store := range filer2.Stores {
			println("    " + store.GetName())
		}
		return false
	}

	if targetStore == nil && *filerExportTargetStore != "" && *filerExportTargetStore != "notification" {
		glog.Errorf("Failed to find target store %s", *filerExportTargetStore)
		println("existing data sources are:")
		for _, store := range filer2.Stores {
			println("    " + store.GetName())
		}
		return false
	}

	stat := statistics{}

	var fn func(level int, entry *filer2.Entry) error

	if *filerExportTargetStore == "notification" {
		weed_server.LoadConfiguration("notification", false)
		v := viper.GetViper()
		notification.LoadConfiguration(v.Sub("notification"))

		fn = func(level int, entry *filer2.Entry) error {
			printout(level, entry)
			if *dryRun {
				return nil
			}
			return notification.Queue.SendMessage(
				string(entry.FullPath),
				&filer_pb.EventNotification{
					NewEntry: entry.ToProtoEntry(),
				},
			)
		}
	} else if targetStore == nil {
		fn = printout
	} else {
		fn = func(level int, entry *filer2.Entry) error {
			printout(level, entry)
			if *dryRun {
				return nil
			}
			return targetStore.InsertEntry(entry)
		}
	}

	doTraverse(&stat, sourceStore, filer2.FullPath(*dir), 0, fn)

	glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)

	return true
}

func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {

	limit := *dirListLimit
	lastEntryName := ""
	for {
		entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
		if err != nil {
			break
		}
		for _, entry := range entries {
			if fnErr := fn(level, entry); fnErr != nil {
				glog.Errorf("failed to process entry: %s", entry.FullPath)
			}
			if entry.IsDirectory() {
				stat.directoryCount++
				doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
			} else {
				stat.fileCount++
			}
		}
		if len(entries) < limit {
			break
		}
	}
}

func printout(level int, entry *filer2.Entry) error {
	for i := 0; i < level; i++ {
		if i == level-1 {
			print("+-")
		} else {
			print("| ")
		}
	}
	println(entry.FullPath.Name())
	return nil
}
<|endoftext|>


<commit_before>package stasher

import (
  "context"
  "time"
  "google.golang.org/api/blogger/v3"
  "google.golang.org/api/googleapi"
  "gopkg.in/mgo.v2"
  "gopkg.in/mgo.v2/bson"
)

const DefaultBlogPageCollection = "blog_page"
const DefaultBlogPostCollection = "blog_post"
const DefaultBlogCollection = "blog"

type BlogPageStasher interface {
  GetPageListEtag(ctx context.Context, blogId string) (etag *string)
  HasPage(ctx context.Context, pageId string, etag string) bool
  StashPage(context.Context, *blogger.Page)
  StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, updated *time.Time)
}

type MgoBlogStasher struct {
  BlogPageCollection *mgo.Collection
  BlogPostCollection *mgo.Collection
  BlogCollection *mgo.Collection
}

type MgoBlogPage struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPage *blogger.Page
  Updated *time.Time
}

type MgoBlog struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  Blog *blogger.Blog `bson:"omitempty"`
  PostListEtag *string `bson:"omitempty"`
  PostListUpdated *time.Time `bson:"omitempty"`
  PageListEtag *string `bson:"omitempty"`
  PageListUpdated *time.Time `bson:"omitempty"`
}

func DefaultMgoStasher(db *mgo.Database) (m *MgoBlogStasher) {
  m = new(MgoBlogStasher)
  m.BlogPageCollection = db.C(DefaultBlogPageCollection)
  m.BlogPostCollection = db.C(DefaultBlogPostCollection)
  m.BlogCollection = db.C(DefaultBlogCollection)
  return
}

type blogPageGetter interface {
  blogPageEtags(blogId string, oldListEtag *string) (newListEtag string, newPageEtags map[string]string)
  blogPage(blogId string, pageId string) *blogger.Page
}

type blogService blogger.Service;

func(b *blogService) blogPageEtags(blogId string, oldListEtag *string) (newListEtag string, newPageEtags map[string]string) {
  s := (*blogger.Service)(b)
  pageListCall := s.Pages.List(blogId).FetchBodies(false)
  if oldListEtag != nil {
    pageListCall = pageListCall.IfNoneMatch(*oldListEtag)
  }
  newPageEtags = make(map[string]string)
  err := pageListCall.Pages(nil, func(pList *blogger.PageList) error {
    newListEtag = pList.Etag
    for _, p := range pList.Items {
      newPageEtags[p.Id] = p.Etag
    }
    return nil
  })
  if err != nil && ! googleapi.IsNotModified(err) {
    panic(err)
  }
  return
}

func(b *blogService) blogPage(blogId string, pageId string) *blogger.Page {
  s := (*blogger.Service)(b)
  page, err := s.Pages.Get(blogId, pageId).Do()
  if err != nil { panic(err) }
  return page
}

func syncBlogPages(ctx context.Context, blogId string, stasher BlogPageStasher, getter blogPageGetter) {
  oldListEtag := stasher.GetPageListEtag(ctx, blogId)
  newListEtag, newPageEtags := getter.blogPageEtags(blogId, oldListEtag)
  newUpdated := time.Now()
  if newListEtag == "" { return }
  for id, etag := range newPageEtags {
    if stasher.HasPage(ctx, id, etag) {continue}
    page := getter.blogPage(blogId, id)
    stasher.StashPage(ctx, page)
  }
  stasher.StashPageEtags(ctx, blogId, newListEtag, newPageEtags, &newUpdated)
}

func(m *MgoBlogStasher) GetPageListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(MgoBlog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"PageListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PageListEtag
}
func(m *MgoBlogStasher) HasPage(ctx context.Context, pageId string, etag string) bool {
  n, err := m.BlogPageCollection.Find(bson.M{"blogPage.id": pageId, "blogPage.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}
func(m *MgoBlogStasher) StashPage(ctx context.Context, page *blogger.Page) {
  dbPage := MgoBlogPage{
    BlogPage: page,
  }
  var err error
  *dbPage.Updated, err = time.Parse(page.Updated, time.RFC3339)
  if err != nil { panic(err) }
  _, err = m.BlogPageCollection.Upsert(bson.M{"blogPage.id": page.Id, "updated": bson.M{"$lt": dbPage.Updated}}, &dbPage);
  if err != nil { panic(err) }
}
func(m *MgoBlogStasher) StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPageCollection.Find(nil).Select(bson.M{"dbPage.id": 1, "dbPage.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPage := new(MgoBlogPage)
  for iter.Next(dbPage) {
    if pageEtags[dbPage.BlogPage.Id] != dbPage.BlogPage.Etag {
      if err := m.BlogPageCollection.RemoveId(dbPage.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "pageListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &MgoBlog{PageListEtag: &listEtag, PageListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}
<commit_msg>Stash posts<commit_after>package stasher

import (
  "context"
  "time"
  "google.golang.org/api/blogger/v3"
  "google.golang.org/api/googleapi"
  "gopkg.in/mgo.v2"
  "gopkg.in/mgo.v2/bson"
)

const (
  DefaultBlogPageCollection = "blog_page"
  DefaultBlogPostCollection = "blog_post"
  DefaultBlogCollection = "blog"
)

type BlogPostStasher interface {
  GetPostListEtag(ctx context.Context, blogId string) (etag *string)
  HasPost(ctx context.Context, pageId string, etag string) bool
  StashPost(context.Context, *blogger.Post)
  StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, updated *time.Time)
}

type BlogPageStasher interface {
  GetPageListEtag(ctx context.Context, blogId string) (etag *string)
  HasPage(ctx context.Context, pageId string, etag string) bool
  StashPage(context.Context, *blogger.Page)
  StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, updated *time.Time)
}

type MgoBlogStasher struct {
  BlogPageCollection *mgo.Collection
  BlogPostCollection *mgo.Collection
  BlogCollection *mgo.Collection
}

type MgoBlogPost struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPost *blogger.Post
  Updated *time.Time
}

type MgoBlogPage struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPage *blogger.Page
  Updated *time.Time
}

type MgoBlog struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  Blog *blogger.Blog `bson:"omitempty"`
  PostListEtag *string `bson:"omitempty"`
  PostListUpdated *time.Time `bson:"omitempty"`
  PageListEtag *string `bson:"omitempty"`
  PageListUpdated *time.Time `bson:"omitempty"`
}

func DefaultMgoStasher(db *mgo.Database) (m *MgoBlogStasher) {
  m = new(MgoBlogStasher)
  m.BlogPageCollection = db.C(DefaultBlogPageCollection)
  m.BlogPostCollection = db.C(DefaultBlogPostCollection)
  m.BlogCollection = db.C(DefaultBlogCollection)
  return
}

type blogPostGetter interface {
  blogPostEtags(blogId string, oldListEtag *string) (newListEtag string, newPostEtags map[string]string)
  blogPost(blogId string, postId string) *blogger.Post
}

type blogPageGetter interface {
  blogPageEtags(blogId string, oldListEtag *string) (newListEtag string, newPageEtags map[string]string)
  blogPage(blogId string, pageId string) *blogger.Page
}

type blogService blogger.Service;

func(b *blogService) blogPostEtags(blogId string, oldListEtag *string) (newListEtag string, newPostEtags map[string]string) {
  s := (*blogger.Service)(b)
  postListCall := s.Posts.List(blogId).FetchBodies(false)
  if oldListEtag != nil {
    postListCall = postListCall.IfNoneMatch(*oldListEtag)
  }
  newPostEtags = make(map[string]string)
  err := postListCall.Pages(nil, func(pList *blogger.PostList) error {
    newListEtag = pList.Etag
    for _, p := range pList.Items {
      newPostEtags[p.Id] = p.Etag
    }
    return nil
  })
  if err != nil && ! googleapi.IsNotModified(err) {
    panic(err)
  }
  return
}

func(b *blogService) blogPageEtags(blogId string, oldListEtag *string) (newListEtag string, newPageEtags map[string]string) {
  s := (*blogger.Service)(b)
  pageListCall := s.Pages.List(blogId).FetchBodies(false)
  if oldListEtag != nil {
    pageListCall = pageListCall.IfNoneMatch(*oldListEtag)
  }
  newPageEtags = make(map[string]string)
  err := pageListCall.Pages(nil, func(pList *blogger.PageList) error {
    newListEtag = pList.Etag
    for _, p := range pList.Items {
      newPageEtags[p.Id] = p.Etag
    }
    return nil
  })
  if err != nil && ! googleapi.IsNotModified(err) {
    panic(err)
  }
  return
}

func(b *blogService) blogPost(blogId string, postId string) *blogger.Post {
  s := (*blogger.Service)(b)
  post, err := s.Posts.Get(blogId, postId).FetchImages(true).Do()
  if err != nil { panic(err) }
  return post
}

func(b *blogService) blogPage(blogId string, pageId string) *blogger.Page {
  s := (*blogger.Service)(b)
  page, err := s.Pages.Get(blogId, pageId).Do()
  if err != nil { panic(err) }
  return page
}

func syncBlogPosts(ctx context.Context, blogId string, stasher BlogPostStasher, getter blogPostGetter) {
  oldListEtag := stasher.GetPostListEtag(ctx, blogId)
  newListEtag, newPostEtags := getter.blogPostEtags(blogId, oldListEtag)
  newUpdated := time.Now()
  if newListEtag == "" { return }
  for id, etag := range newPostEtags {
    if stasher.HasPost(ctx, id, etag) {continue}
    post := getter.blogPost(blogId, id)
    stasher.StashPost(ctx, post)
  }
  stasher.StashPostEtags(ctx, blogId, newListEtag, newPostEtags, &newUpdated)
}

func syncBlogPages(ctx context.Context, blogId string, stasher BlogPageStasher, getter blogPageGetter) {
  oldListEtag := stasher.GetPageListEtag(ctx, blogId)
  newListEtag, newPageEtags := getter.blogPageEtags(blogId, oldListEtag)
  newUpdated := time.Now()
  if newListEtag == "" { return }
  for id, etag := range newPageEtags {
    if stasher.HasPage(ctx, id, etag) {continue}
    page := getter.blogPage(blogId, id)
    stasher.StashPage(ctx, page)
  }
  stasher.StashPageEtags(ctx, blogId, newListEtag, newPageEtags, &newUpdated)
}

func(m *MgoBlogStasher) GetPageListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(MgoBlog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"pageListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PageListEtag
}
func(m *MgoBlogStasher) HasPage(ctx context.Context, pageId string, etag string) bool {
  n, err := m.BlogPageCollection.Find(bson.M{"blogPage.id": pageId, "blogPage.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}
func(m *MgoBlogStasher) StashPage(ctx context.Context, page *blogger.Page) {
  dbPage := MgoBlogPage{
    BlogPage: page,
  }
  var err error
  *dbPage.Updated, err = time.Parse(page.Updated, time.RFC3339)
  if err != nil { panic(err) }
  _, err = m.BlogPageCollection.Upsert(bson.M{"blogPage.id": page.Id, "updated": bson.M{"$lt": dbPage.Updated}}, &dbPage);
  if err != nil { panic(err) }
}
func(m *MgoBlogStasher) StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPageCollection.Find(nil).Select(bson.M{"dbPage.id": 1, "dbPage.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPage := new(MgoBlogPage)
  for iter.Next(dbPage) {
    if pageEtags[dbPage.BlogPage.Id] != dbPage.BlogPage.Etag {
      if err := m.BlogPageCollection.RemoveId(dbPage.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "pageListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &MgoBlog{PageListEtag: &listEtag, PageListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *MgoBlogStasher) GetPostListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(MgoBlog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"postListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PostListEtag
}
func(m *MgoBlogStasher) HasPost(ctx context.Context, postId string, etag string) bool {
  n, err := m.BlogPostCollection.Find(bson.M{"blogPost.id": postId, "blogPost.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}
func(m *MgoBlogStasher) StashPost(ctx context.Context, post *blogger.Post) {
  dbPost := MgoBlogPost{
    BlogPost: post,
  }
  var err error
  *dbPost.Updated, err = time.Parse(post.Updated, time.RFC3339)
  if err != nil { panic(err) }
  _, err = m.BlogPostCollection.Upsert(bson.M{"blogPost.id": post.Id, "updated": bson.M{"$lt": dbPost.Updated}}, &dbPost);
  if err != nil { panic(err) }
}
func(m *MgoBlogStasher) StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPostCollection.Find(nil).Select(bson.M{"dbPost.id": 1, "dbPost.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPost := new(MgoBlogPost)
  for iter.Next(dbPost) {
    if postEtags[dbPost.BlogPost.Id] != dbPost.BlogPost.Etag {
      if err := m.BlogPostCollection.RemoveId(dbPost.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "postListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &MgoBlog{PostListEtag: &listEtag, PostListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}
<|endoftext|>


<commit_before>package util

import (
	"compress/bzip2"
	"compress/gzip"
	"io"
	"os"
	"strings"
)

// Implements io.ReadCloser
type ReadCloser struct {
	r      io.Reader
	fp     *os.File
	gz     *gzip.Reader
	isOpen bool
	isGzip bool
	isBz2  bool
}

// Opens for reading a plain file, gzip'ed file (extension .gz), or bzip2'ed file (extension .bz2)
func Open(filename string) (rc *ReadCloser, err error) {
	rc = &ReadCloser{}

	rc.fp, err = os.Open(filename)
	if err != nil {
		return
	}

	if strings.HasSuffix(filename, ".gz") {
		rc.gz, err = gzip.NewReader(rc.fp)
		if err != nil {
			rc.fp.Close()
			return
		}
		rc.r = rc.gz
		rc.isGzip = true
	} else if strings.HasSuffix(filename, ".bz2") {
		rc.r = bzip2.NewReader(rc.fp)
		rc.isBz2 = true
	} else {
		rc.r = rc.fp
	}

	rc.isOpen = true
	return
}

func (rc *ReadCloser) Read(p []byte) (n int, err error) {
	if !rc.isOpen {
		panic("ReadCloser is closed")
	}
	return rc.r.Read(p)
}

func (rc *ReadCloser) Close() (err error) {
	if rc.isOpen {
		rc.isOpen = false
		if rc.isGzip {
			err = rc.gz.Close()
		}
		err2 := rc.fp.Close()
		if err == nil {
			err = err2
		}
	}
	return
}
<commit_msg>readcloser.go: cleaner, more like true Go<commit_after>package util

import (
	"compress/bzip2"
	"compress/gzip"
	"io"
	"os"
	"strings"
)

type ReadCloser struct {
	r       io.Reader
	fp      *os.File
	gz      *gzip.Reader
	isOpen  bool
	isGzip  bool
	isBzip2 bool
}

// Opens for reading a plain file, gzip'ed file (extension .gz), or bzip2'ed file (extension .bz2)
func Open(filename string) (rc io.ReadCloser, err error) {
	fp, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	if strings.HasSuffix(filename, ".gz") {
		r := ReadCloser{
			fp:     fp,
			isGzip: true,
			isOpen: true,
		}
		r.gz, err = gzip.NewReader(fp)
		if err != nil {
			fp.Close()
			return nil, err
		}
		r.r = r.gz
		return r, nil
	}

	if strings.HasSuffix(filename, ".bz2") {
		r := ReadCloser{
			fp:      fp,
			r:       bzip2.NewReader(fp),
			isBzip2: true,
			isOpen:  true,
		}
		return r, nil
	}

	return fp, nil
}

func (rc ReadCloser) Read(p []byte) (n int, err error) {
	if !rc.isOpen {
		panic("ReadCloser is closed")
	}
	return rc.r.Read(p)
}

func (rc ReadCloser) Close() (err error) {
	if rc.isOpen {
		rc.isOpen = false

		if rc.isGzip {
			err = rc.gz.Close()
		}

		e := rc.fp.Close()
		if err == nil {
			err = e
		}
	}
	return
}
<|endoftext|>


<commit_before>package disruptor

func (this *Reader) Commit(upper int64) {
	this.read.Store(upper)
}
<commit_msg>Updated internal variable names.<commit_after>package disruptor

func (this *Reader) Commit(sequence int64) {
	this.read.Store(sequence)
}
<|endoftext|>