<commit_before>// Copyright (c) 2012 VMware, Inc.

package sigar

/*
#include <stdlib.h>
#include <sys/sysctl.h>
#include <sys/mount.h>
#include <mach/mach_init.h>
#include <mach/mach_host.h>
#include <mach/host_info.h>
#include <libproc.h>
#include <mach/processor_info.h>
#include <mach/vm_map.h>
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"syscall"
	"time"
	"unsafe"
)

func (self *LoadAverage) Get() error {
	avg := []C.double{0, 0, 0}

	C.getloadavg(&avg[0], C.int(len(avg)))

	self.One = float64(avg[0])
	self.Five = float64(avg[1])
	self.Fifteen = float64(avg[2])

	return nil
}

func (self *Uptime) Get() error {
	tv := syscall.Timeval{}

	if err := sysctlbyname("kern.boottime", &tv); err != nil {
		return err
	}

	self.Length = time.Since(time.Unix(tv.Unix())).Seconds()

	return nil
}

func (self *Mem) Get() error {
	var vmstat C.vm_statistics_data_t

	if err := sysctlbyname("hw.memsize", &self.Total); err != nil {
		return err
	}

	if err := vm_info(&vmstat); err != nil {
		return err
	}

	kern := uint64(vmstat.inactive_count) << 12
	self.Free = uint64(vmstat.free_count) << 12

	self.Used = self.Total - self.Free
	self.ActualFree = self.Free + kern
	self.ActualUsed = self.Used - kern

	return nil
}

type xsw_usage struct {
	Total, Avail, Used uint64
}

func (self *Swap) Get() error {
	sw_usage := xsw_usage{}

	if err := sysctlbyname("vm.swapusage", &sw_usage); err != nil {
		return err
	}

	self.Total = sw_usage.Total
	self.Used = sw_usage.Used
	self.Free = sw_usage.Avail

	return nil
}

func (self *Cpu) Get() error {
	var count C.mach_msg_type_number_t = C.HOST_CPU_LOAD_INFO_COUNT
	var cpuload C.host_cpu_load_info_data_t

	status := C.host_statistics(C.host_t(C.mach_host_self()),
		C.HOST_CPU_LOAD_INFO,
		C.host_info_t(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_statistics error=%d", status)
	}

	self.User = uint64(cpuload.cpu_ticks[C.CPU_STATE_USER])
	self.Sys = uint64(cpuload.cpu_ticks[C.CPU_STATE_SYSTEM])
	self.Idle = uint64(cpuload.cpu_ticks[C.CPU_STATE_IDLE])
	self.Nice = uint64(cpuload.cpu_ticks[C.CPU_STATE_NICE])

	return nil
}

func (self *CpuList) Get() error {
	var count C.mach_msg_type_number_t
	var cpuload *C.processor_cpu_load_info_data_t
	var ncpu C.natural_t

	status := C.host_processor_info(C.host_t(C.mach_host_self()),
		C.PROCESSOR_CPU_LOAD_INFO,
		&ncpu,
		(*C.processor_info_array_t)(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_processor_info error=%d", status)
	}

	// jump through some cgo casting hoops and ensure we properly free
	// the memory that cpuload points to
	target := C.vm_map_t(C.mach_task_self_)
	address := C.vm_address_t(uintptr(unsafe.Pointer(cpuload)))
	defer C.vm_deallocate(target, address, C.vm_size_t(ncpu))

	// the body of struct processor_cpu_load_info
	// aka processor_cpu_load_info_data_t
	var cpu_ticks [C.CPU_STATE_MAX]uint32

	// copy the cpuload array to a []byte buffer
	// where we can binary.Read the data
	size := int(ncpu) * binary.Size(cpu_ticks)
	buf := C.GoBytes(unsafe.Pointer(cpuload), C.int(size))

	bbuf := bytes.NewBuffer(buf)

	self.List = make([]Cpu, 0, ncpu)

	for i := 0; i < int(ncpu); i++ {
		cpu := Cpu{}

		err := binary.Read(bbuf, binary.LittleEndian, &cpu_ticks)
		if err != nil {
			return err
		}

		cpu.User = uint64(cpu_ticks[C.CPU_STATE_USER])
		cpu.Sys = uint64(cpu_ticks[C.CPU_STATE_SYSTEM])
		cpu.Idle = uint64(cpu_ticks[C.CPU_STATE_IDLE])
		cpu.Nice = uint64(cpu_ticks[C.CPU_STATE_NICE])

		self.List = append(self.List, cpu)
	}

	return nil
}

func (self *FileSystemList) Get() error {
	num, err := getfsstat(nil, C.MNT_NOWAIT)
	if num < 0 {
		return err
	}

	buf := make([]syscall.Statfs_t, num)

	num, err = getfsstat(buf, C.MNT_NOWAIT)
	if err != nil {
		return err
	}

	fslist := make([]FileSystem, 0, num)

	for i := 0; i < num; i++ {
		fs := FileSystem{}

		fs.DirName = bytePtrToString(&buf[i].Mntonname[0])
		fs.DevName = bytePtrToString(&buf[i].Mntfromname[0])
		fs.SysTypeName = bytePtrToString(&buf[i].Fstypename[0])

		fslist = append(fslist, fs)
	}

	self.List = fslist

	return err
}

func (self *ProcList) Get() error {
	n := C.proc_listpids(C.PROC_ALL_PIDS, 0, nil, 0)
	if n <= 0 {
		return syscall.EINVAL
	}
	buf := make([]byte, n)
	n = C.proc_listpids(C.PROC_ALL_PIDS, 0, unsafe.Pointer(&buf[0]), n)
	if n <= 0 {
		return syscall.ENOMEM
	}

	var pid int32
	num := int(n) / binary.Size(pid)
	list := make([]int, 0, num)
	bbuf := bytes.NewBuffer(buf)

	for i := 0; i < num; i++ {
		if err := binary.Read(bbuf, binary.LittleEndian, &pid); err != nil {
			return err
		}
		if pid == 0 {
			continue
		}

		list = append(list, int(pid))
	}

	self.List = list

	return nil
}

func (self *ProcState) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.Name = C.GoString(&info.pbsd.pbi_comm[0])

	switch info.pbsd.pbi_status {
	case C.SIDL:
		self.State = RunStateIdle
	case C.SRUN:
		self.State = RunStateRun
	case C.SSLEEP:
		self.State = RunStateSleep
	case C.SSTOP:
		self.State = RunStateStop
	case C.SZOMB:
		self.State = RunStateZombie
	default:
		self.State = RunStateUnknown
	}

	self.Ppid = int(info.pbsd.pbi_ppid)

	self.Tty = int(info.pbsd.e_tdev)

	self.Priority = int(info.ptinfo.pti_priority)

	self.Nice = int(info.pbsd.pbi_nice)

	return nil
}

func (self *ProcMem) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.Size = uint64(info.ptinfo.pti_virtual_size)
	self.Resident = uint64(info.ptinfo.pti_resident_size)
	self.PageFaults = uint64(info.ptinfo.pti_faults)

	return nil
}

func (self *ProcTime) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.User =
		uint64(info.ptinfo.pti_total_user) / uint64(time.Millisecond)

	self.Sys =
		uint64(info.ptinfo.pti_total_system) / uint64(time.Millisecond)

	self.Total = self.User + self.Sys

	self.StartTime = (uint64(info.pbsd.pbi_start_tvsec) * 1000) +
		(uint64(info.pbsd.pbi_start_tvusec) / 1000)

	return nil
}

func (self *ProcArgs) Get(pid int) error {
	var args []string

	argv := func(arg string) {
		args = append(args, arg)
	}

	err := kern_procargs(pid, nil, argv, nil)

	self.List = args

	return err
}

func (self *ProcExe) Get(pid int) error {
	exe := func(arg string) {
		self.Name = arg
	}

	return kern_procargs(pid, exe, nil, nil)
}

// wrapper around sysctl KERN_PROCARGS2
// callbacks params are optional,
// up to the caller as to which pieces of data they want
func kern_procargs(pid int,
	exe func(string),
	argv func(string),
	env func(string, string)) error {

	mib := []C.int{C.CTL_KERN, C.KERN_PROCARGS2, C.int(pid)}
	argmax := uintptr(C.ARG_MAX)
	buf := make([]byte, argmax)
	err := sysctl(mib, &buf[0], &argmax, nil, 0)
	if err != nil {
		return nil
	}

	bbuf := bytes.NewBuffer(buf)
	bbuf.Truncate(int(argmax))

	var argc int32
	binary.Read(bbuf, binary.LittleEndian, &argc)

	path, err := bbuf.ReadBytes(0)
	if exe != nil {
		exe(string(chop(path)))
	}

	// skip trailing \0's
	for {
		c, _ := bbuf.ReadByte()
		if c != 0 {
			bbuf.UnreadByte()
			break // start of argv[0]
		}
	}

	for i := 0; i < int(argc); i++ {
		arg, err := bbuf.ReadBytes(0)
		if err == io.EOF {
			break
		}
		if argv != nil {
			argv(string(chop(arg)))
		}
	}

	if env == nil {
		return nil
	}

	delim := []byte{61} // "="

	for {
		line, err := bbuf.ReadBytes(0)
		if err == io.EOF || line[0] == 0 {
			break
		}
		pair := bytes.SplitN(chop(line), delim, 2)
		env(string(pair[0]), string(pair[1]))
	}

	return nil
}

// XXX copied from zsyscall_darwin_amd64.go
func sysctl(mib []C.int, old *byte, oldlen *uintptr,
	new *byte, newlen uintptr) (err error) {
	var p0 unsafe.Pointer
	p0 = unsafe.Pointer(&mib[0])
	_, _, e1 := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(p0),
		uintptr(len(mib)),
		uintptr(unsafe.Pointer(old)), uintptr(unsafe.Pointer(oldlen)),
		uintptr(unsafe.Pointer(new)), uintptr(newlen))
	if e1 != 0 {
		err = e1
	}
	return
}

func vm_info(vmstat *C.vm_statistics_data_t) error {
	var count C.mach_msg_type_number_t = C.HOST_VM_INFO_COUNT

	status := C.host_statistics(
		C.host_t(C.mach_host_self()),
		C.HOST_VM_INFO,
		C.host_info_t(unsafe.Pointer(vmstat)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_statistics=%d", status)
	}

	return nil
}

// generic Sysctl buffer unmarshalling
func sysctlbyname(name string, data interface{}) (err error) {
	val, err := syscall.Sysctl(name)
	if err != nil {
		return err
	}

	buf := []byte(val)

	switch v := data.(type) {
	case *uint64:
		*v = *(*uint64)(unsafe.Pointer(&buf[0]))
		return
	}

	bbuf := bytes.NewBuffer([]byte(val))
	return binary.Read(bbuf, binary.LittleEndian, data)
}

// syscall.Getfsstat() wrapper is broken, roll our own to workaround.
func getfsstat(buf []syscall.Statfs_t, flags int) (n int, err error) {
	var ptr uintptr
	var size uintptr

	if len(buf) > 0 {
		ptr = uintptr(unsafe.Pointer(&buf[0]))
		size = unsafe.Sizeof(buf[0]) * uintptr(len(buf))
	} else {
		ptr = uintptr(0)
		size = uintptr(0)
	}

	trap := uintptr(syscall.SYS_GETFSSTAT64)
	ret, _, errno := syscall.Syscall(trap, ptr, size, uintptr(flags))

	n = int(ret)
	if errno != 0 {
		err = errno
	}

	return
}

func task_info(pid int, info *C.struct_proc_taskallinfo) error {
	size := C.int(unsafe.Sizeof(*info))
	ptr := unsafe.Pointer(info)

	n := C.proc_pidinfo(C.int(pid), C.PROC_PIDTASKALLINFO, 0, ptr, size)
	if n != size {
		return syscall.ENOMEM
	}

	return nil
}
<commit_msg>Fix for darwin uptime bug<commit_after>// Copyright (c) 2012 VMware, Inc.

package sigar

/*
#include <stdlib.h>
#include <sys/sysctl.h>
#include <sys/mount.h>
#include <mach/mach_init.h>
#include <mach/mach_host.h>
#include <mach/host_info.h>
#include <libproc.h>
#include <mach/processor_info.h>
#include <mach/vm_map.h>
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"syscall"
	"time"
	"unsafe"
)

func (self *LoadAverage) Get() error {
	avg := []C.double{0, 0, 0}

	C.getloadavg(&avg[0], C.int(len(avg)))

	self.One = float64(avg[0])
	self.Five = float64(avg[1])
	self.Fifteen = float64(avg[2])

	return nil
}

func (self *Uptime) Get() error {
	tv := syscall.Timeval32{}

	if err := sysctlbyname("kern.boottime", &tv); err != nil {
		return err
	}

	self.Length = time.Since(time.Unix(int64(tv.Sec), int64(tv.Usec)*1000)).Seconds()

	return nil
}

func (self *Mem) Get() error {
	var vmstat C.vm_statistics_data_t

	if err := sysctlbyname("hw.memsize", &self.Total); err != nil {
		return err
	}

	if err := vm_info(&vmstat); err != nil {
		return err
	}

	kern := uint64(vmstat.inactive_count) << 12
	self.Free = uint64(vmstat.free_count) << 12

	self.Used = self.Total - self.Free
	self.ActualFree = self.Free + kern
	self.ActualUsed = self.Used - kern

	return nil
}

type xsw_usage struct {
	Total, Avail, Used uint64
}

func (self *Swap) Get() error {
	sw_usage := xsw_usage{}

	if err := sysctlbyname("vm.swapusage", &sw_usage); err != nil {
		return err
	}

	self.Total = sw_usage.Total
	self.Used = sw_usage.Used
	self.Free = sw_usage.Avail

	return nil
}

func (self *Cpu) Get() error {
	var count C.mach_msg_type_number_t = C.HOST_CPU_LOAD_INFO_COUNT
	var cpuload C.host_cpu_load_info_data_t

	status := C.host_statistics(C.host_t(C.mach_host_self()),
		C.HOST_CPU_LOAD_INFO,
		C.host_info_t(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_statistics error=%d", status)
	}

	self.User = uint64(cpuload.cpu_ticks[C.CPU_STATE_USER])
	self.Sys = uint64(cpuload.cpu_ticks[C.CPU_STATE_SYSTEM])
	self.Idle = uint64(cpuload.cpu_ticks[C.CPU_STATE_IDLE])
	self.Nice = uint64(cpuload.cpu_ticks[C.CPU_STATE_NICE])

	return nil
}

func (self *CpuList) Get() error {
	var count C.mach_msg_type_number_t
	var cpuload *C.processor_cpu_load_info_data_t
	var ncpu C.natural_t

	status := C.host_processor_info(C.host_t(C.mach_host_self()),
		C.PROCESSOR_CPU_LOAD_INFO,
		&ncpu,
		(*C.processor_info_array_t)(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_processor_info error=%d", status)
	}

	// jump through some cgo casting hoops and ensure we properly free
	// the memory that cpuload points to
	target := C.vm_map_t(C.mach_task_self_)
	address := C.vm_address_t(uintptr(unsafe.Pointer(cpuload)))
	defer C.vm_deallocate(target, address, C.vm_size_t(ncpu))

	// the body of struct processor_cpu_load_info
	// aka processor_cpu_load_info_data_t
	var cpu_ticks [C.CPU_STATE_MAX]uint32

	// copy the cpuload array to a []byte buffer
	// where we can binary.Read the data
	size := int(ncpu) * binary.Size(cpu_ticks)
	buf := C.GoBytes(unsafe.Pointer(cpuload), C.int(size))

	bbuf := bytes.NewBuffer(buf)

	self.List = make([]Cpu, 0, ncpu)

	for i := 0; i < int(ncpu); i++ {
		cpu := Cpu{}

		err := binary.Read(bbuf, binary.LittleEndian, &cpu_ticks)
		if err != nil {
			return err
		}

		cpu.User = uint64(cpu_ticks[C.CPU_STATE_USER])
		cpu.Sys = uint64(cpu_ticks[C.CPU_STATE_SYSTEM])
		cpu.Idle = uint64(cpu_ticks[C.CPU_STATE_IDLE])
		cpu.Nice = uint64(cpu_ticks[C.CPU_STATE_NICE])

		self.List = append(self.List, cpu)
	}

	return nil
}

func (self *FileSystemList) Get() error {
	num, err := getfsstat(nil, C.MNT_NOWAIT)
	if num < 0 {
		return err
	}

	buf := make([]syscall.Statfs_t, num)

	num, err = getfsstat(buf, C.MNT_NOWAIT)
	if err != nil {
		return err
	}

	fslist := make([]FileSystem, 0, num)

	for i := 0; i < num; i++ {
		fs := FileSystem{}

		fs.DirName = bytePtrToString(&buf[i].Mntonname[0])
		fs.DevName = bytePtrToString(&buf[i].Mntfromname[0])
		fs.SysTypeName = bytePtrToString(&buf[i].Fstypename[0])

		fslist = append(fslist, fs)
	}

	self.List = fslist

	return err
}

func (self *ProcList) Get() error {
	n := C.proc_listpids(C.PROC_ALL_PIDS, 0, nil, 0)
	if n <= 0 {
		return syscall.EINVAL
	}
	buf := make([]byte, n)
	n = C.proc_listpids(C.PROC_ALL_PIDS, 0, unsafe.Pointer(&buf[0]), n)
	if n <= 0 {
		return syscall.ENOMEM
	}

	var pid int32
	num := int(n) / binary.Size(pid)
	list := make([]int, 0, num)
	bbuf := bytes.NewBuffer(buf)

	for i := 0; i < num; i++ {
		if err := binary.Read(bbuf, binary.LittleEndian, &pid); err != nil {
			return err
		}
		if pid == 0 {
			continue
		}

		list = append(list, int(pid))
	}

	self.List = list

	return nil
}

func (self *ProcState) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.Name = C.GoString(&info.pbsd.pbi_comm[0])

	switch info.pbsd.pbi_status {
	case C.SIDL:
		self.State = RunStateIdle
	case C.SRUN:
		self.State = RunStateRun
	case C.SSLEEP:
		self.State = RunStateSleep
	case C.SSTOP:
		self.State = RunStateStop
	case C.SZOMB:
		self.State = RunStateZombie
	default:
		self.State = RunStateUnknown
	}

	self.Ppid = int(info.pbsd.pbi_ppid)

	self.Tty = int(info.pbsd.e_tdev)

	self.Priority = int(info.ptinfo.pti_priority)

	self.Nice = int(info.pbsd.pbi_nice)

	return nil
}

func (self *ProcMem) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.Size = uint64(info.ptinfo.pti_virtual_size)
	self.Resident = uint64(info.ptinfo.pti_resident_size)
	self.PageFaults = uint64(info.ptinfo.pti_faults)

	return nil
}

func (self *ProcTime) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.User =
		uint64(info.ptinfo.pti_total_user) / uint64(time.Millisecond)

	self.Sys =
		uint64(info.ptinfo.pti_total_system) / uint64(time.Millisecond)

	self.Total = self.User + self.Sys

	self.StartTime = (uint64(info.pbsd.pbi_start_tvsec) * 1000) +
		(uint64(info.pbsd.pbi_start_tvusec) / 1000)

	return nil
}

func (self *ProcArgs) Get(pid int) error {
	var args []string

	argv := func(arg string) {
		args = append(args, arg)
	}

	err := kern_procargs(pid, nil, argv, nil)

	self.List = args

	return err
}

func (self *ProcExe) Get(pid int) error {
	exe := func(arg string) {
		self.Name = arg
	}

	return kern_procargs(pid, exe, nil, nil)
}

// wrapper around sysctl KERN_PROCARGS2
// callbacks params are optional,
// up to the caller as to which pieces of data they want
func kern_procargs(pid int,
	exe func(string),
	argv func(string),
	env func(string, string)) error {

	mib := []C.int{C.CTL_KERN, C.KERN_PROCARGS2, C.int(pid)}
	argmax := uintptr(C.ARG_MAX)
	buf := make([]byte, argmax)
	err := sysctl(mib, &buf[0], &argmax, nil, 0)
	if err != nil {
		return nil
	}

	bbuf := bytes.NewBuffer(buf)
	bbuf.Truncate(int(argmax))

	var argc int32
	binary.Read(bbuf, binary.LittleEndian, &argc)

	path, err := bbuf.ReadBytes(0)
	if exe != nil {
		exe(string(chop(path)))
	}

	// skip trailing \0's
	for {
		c, _ := bbuf.ReadByte()
		if c != 0 {
			bbuf.UnreadByte()
			break // start of argv[0]
		}
	}

	for i := 0; i < int(argc); i++ {
		arg, err := bbuf.ReadBytes(0)
		if err == io.EOF {
			break
		}
		if argv != nil {
			argv(string(chop(arg)))
		}
	}

	if env == nil {
		return nil
	}

	delim := []byte{61} // "="

	for {
		line, err := bbuf.ReadBytes(0)
		if err == io.EOF || line[0] == 0 {
			break
		}
		pair := bytes.SplitN(chop(line), delim, 2)
		env(string(pair[0]), string(pair[1]))
	}

	return nil
}

// XXX copied from zsyscall_darwin_amd64.go
func sysctl(mib []C.int, old *byte, oldlen *uintptr,
	new *byte, newlen uintptr) (err error) {
	var p0 unsafe.Pointer
	p0 = unsafe.Pointer(&mib[0])
	_, _, e1 := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(p0),
		uintptr(len(mib)),
		uintptr(unsafe.Pointer(old)), uintptr(unsafe.Pointer(oldlen)),
		uintptr(unsafe.Pointer(new)), uintptr(newlen))
	if e1 != 0 {
		err = e1
	}
	return
}

func vm_info(vmstat *C.vm_statistics_data_t) error {
	var count C.mach_msg_type_number_t = C.HOST_VM_INFO_COUNT

	status := C.host_statistics(
		C.host_t(C.mach_host_self()),
		C.HOST_VM_INFO,
		C.host_info_t(unsafe.Pointer(vmstat)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_statistics=%d", status)
	}

	return nil
}

// generic Sysctl buffer unmarshalling
func sysctlbyname(name string, data interface{}) (err error) {
	val, err := syscall.Sysctl(name)
	if err != nil {
		return err
	}

	buf := []byte(val)

	switch v := data.(type) {
	case *uint64:
		*v = *(*uint64)(unsafe.Pointer(&buf[0]))
		return
	}

	bbuf := bytes.NewBuffer([]byte(val))
	return binary.Read(bbuf, binary.LittleEndian, data)
}

// syscall.Getfsstat() wrapper is broken, roll our own to workaround.
func getfsstat(buf []syscall.Statfs_t, flags int) (n int, err error) {
	var ptr uintptr
	var size uintptr

	if len(buf) > 0 {
		ptr = uintptr(unsafe.Pointer(&buf[0]))
		size = unsafe.Sizeof(buf[0]) * uintptr(len(buf))
	} else {
		ptr = uintptr(0)
		size = uintptr(0)
	}

	trap := uintptr(syscall.SYS_GETFSSTAT64)
	ret, _, errno := syscall.Syscall(trap, ptr, size, uintptr(flags))

	n = int(ret)
	if errno != 0 {
		err = errno
	}

	return
}

func task_info(pid int, info *C.struct_proc_taskallinfo) error {
	size := C.int(unsafe.Sizeof(*info))
	ptr := unsafe.Pointer(info)

	n := C.proc_pidinfo(C.int(pid), C.PROC_PIDTASKALLINFO, 0, ptr, size)
	if n != size {
		return syscall.ENOMEM
	}

	return nil
}
<|endoftext|>


<commit_before>package vmess

import (
	"v2ray.com/core/common/dice"
	"v2ray.com/core/common/log"
	"v2ray.com/core/common/protocol"
	"v2ray.com/core/common/uuid"
)

type InternalAccount struct {
	ID       *protocol.ID
	AlterIDs []*protocol.ID
	Security protocol.Security
}

func (v *InternalAccount) AnyValidID() *protocol.ID {
	if len(v.AlterIDs) == 0 {
		return v.ID
	}
	return v.AlterIDs[dice.Roll(len(v.AlterIDs))]
}

func (v *InternalAccount) Equals(account protocol.Account) bool {
	vmessAccount, ok := account.(*InternalAccount)
	if !ok {
		return false
	}
	// TODO: handle AlterIds difference
	return v.ID.Equals(vmessAccount.ID)
}

func (v *Account) AsAccount() (protocol.Account, error) {
	id, err := uuid.ParseString(v.Id)
	if err != nil {
		log.Error("VMess: Failed to parse ID: ", err)
		return nil, err
	}
	protoId := protocol.NewID(id)
	return &InternalAccount{
		ID:       protoId,
		AlterIDs: protocol.NewAlterIDs(protoId, uint16(v.AlterId)),
		Security: v.SecuritySettings.AsSecurity(),
	}, nil
}
<commit_msg>fix lint warnings<commit_after>package vmess

import (
	"v2ray.com/core/common/dice"
	"v2ray.com/core/common/log"
	"v2ray.com/core/common/protocol"
	"v2ray.com/core/common/uuid"
)

type InternalAccount struct {
	ID       *protocol.ID
	AlterIDs []*protocol.ID
	Security protocol.Security
}

func (v *InternalAccount) AnyValidID() *protocol.ID {
	if len(v.AlterIDs) == 0 {
		return v.ID
	}
	return v.AlterIDs[dice.Roll(len(v.AlterIDs))]
}

func (v *InternalAccount) Equals(account protocol.Account) bool {
	vmessAccount, ok := account.(*InternalAccount)
	if !ok {
		return false
	}
	// TODO: handle AlterIds difference
	return v.ID.Equals(vmessAccount.ID)
}

func (v *Account) AsAccount() (protocol.Account, error) {
	id, err := uuid.ParseString(v.Id)
	if err != nil {
		log.Error("VMess: Failed to parse ID: ", err)
		return nil, err
	}
	protoID := protocol.NewID(id)
	return &InternalAccount{
		ID:       protoID,
		AlterIDs: protocol.NewAlterIDs(protoID, uint16(v.AlterId)),
		Security: v.SecuritySettings.AsSecurity(),
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"os"
	"path/filepath"
)

/* TODO
- Clean up output formatting
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Matches         int
	Misses          int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	dropboxManifest, err := getDropboxManifest(dbxClient, opts.RemoteRoot)
	if err != nil {
		panic(err)
	}

	localManifest, err := getLocalManifest(localRoot, opts.CheckContentHash)
	if err != nil {
		panic(err)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest)

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")
	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathDisplay)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore
	}

	return
}

func getLocalManifest(localRoot string, contentHash bool) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)

	err = filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() {
			relPath, err := normalizePath(localRoot, entryPath)
			if err != nil {
				return err
			}

			hash := ""
			if contentHash {
				hash, err = dropbox.FileContentHash(entryPath)
				if err != nil {
					return err
				}
			}

			heap.Push(manifest, &File{
				Path:        relPath,
				ContentHash: hash,
			})
		}

		return nil
	})

	return
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func compareManifests(remoteManifest, localManifest *FileHeap) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil || local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil || local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if remote.Path == local.Path {
			if !compareFileContents(remote, local) {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			} else {
				comparison.Matches++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		} else {
			// This should be unreachable...
			panic(fmt.Sprintf("Unexpected condition! Local: %v, remote: %v", local, remote))
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<commit_msg>Add todo for parallelization<commit_after>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"os"
	"path/filepath"
)

/* TODO
- Clean up output formatting
- Parallelize local/remote file listing
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Matches         int
	Misses          int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	dropboxManifest, err := getDropboxManifest(dbxClient, opts.RemoteRoot)
	if err != nil {
		panic(err)
	}

	localManifest, err := getLocalManifest(localRoot, opts.CheckContentHash)
	if err != nil {
		panic(err)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest)

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")
	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathDisplay)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore
	}

	return
}

func getLocalManifest(localRoot string, contentHash bool) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)

	err = filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() {
			relPath, err := normalizePath(localRoot, entryPath)
			if err != nil {
				return err
			}

			hash := ""
			if contentHash {
				hash, err = dropbox.FileContentHash(entryPath)
				if err != nil {
					return err
				}
			}

			heap.Push(manifest, &File{
				Path:        relPath,
				ContentHash: hash,
			})
		}

		return nil
	})

	return
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func compareManifests(remoteManifest, localManifest *FileHeap) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil || local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil || local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if remote.Path == local.Path {
			if !compareFileContents(remote, local) {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			} else {
				comparison.Matches++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		} else {
			// This should be unreachable...
			panic(fmt.Sprintf("Unexpected condition! Local: %v, remote: %v", local, remote))
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<|endoftext|>


<commit_before>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 Weaviate. All rights reserved.
 * LICENSE: https://github.com/weaviate/weaviate/blob/master/LICENSE
 * AUTHOR: Bob van Luijt (bob@weaviate.com)
 * See www.weaviate.com for details
 * Contact: @weaviate_iot / yourfriends@weaviate.com
 */

package datastore

import (
	"context"
	"fmt"
	"log"
	"time"

	"google.golang.org/api/iterator"

	uuid "github.com/satori/go.uuid"

	"cloud.google.com/go/datastore"
)

type Datastore struct{}

type Object struct {
	Uuid         string // uuid, also used in Object's id
	Owner        string // uuid of the owner
	RefType      string // type, as defined
	CreateTimeMs int64  // creation time in ms
	Object       string // the JSON object, id will be collected from current uuid
	Deleted      bool   // if true, it does not exsist anymore
}

// Connect to datastore
func (f Datastore) Connect() bool {
	return true
}

// Add item to DB
func (f Datastore) Add(owner string, refType string, object string) string {

	// Setx your Google Cloud Platform project ID.
	ctx := context.Background()

	projectID := "weaviate-dev-001"

	// Creates a client.
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}

	// Sets the kind for the new entity.
	kind := "weaviate"

	// Sets the name/ID for the new entity.
	uuid := fmt.Sprintf("%v", uuid.NewV4())

	// Creates a Key instance.
	taskKey := datastore.NameKey(kind, uuid, nil)

	// Creates a Task instance.
	task := Object{
		Uuid:         uuid,
		Owner:        owner,
		RefType:      refType,
		CreateTimeMs: time.Now().UnixNano() / int64(time.Millisecond),
		Object:       object,
		Deleted:      false,
	}

	// Saves the new entity.
	if _, err := client.Put(ctx, taskKey, &task); err != nil {
		log.Fatalf("Failed to save task: %v", err)
	}

	// return the ID that is used to create.
	return uuid

}

func (f Datastore) Get(Uuid string) string {

	// Setx your Google Cloud Platform project ID.
	ctx := context.Background()
	projectID := "weaviate-dev-001"

	// Creates a client.
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}

	query := datastore.NewQuery("weaviate").Filter("Uuid =", Uuid).Order("-CreateTimeMs").Limit(1)
	it := client.Run(ctx, query)

	count := 0

	for {
		var object Object
		_, err := it.Next(&object)
		if err == iterator.Done {
			break
		}
		if err != nil {
			log.Fatalf("Error fetching next object: %v", err)
		}

		count++
		fmt.Printf("Uid %q, Time %d, Object %d\n", object.Uuid, object.CreateTimeMs, object.Object)
	}

	println(count)

	return Uuid
}
<commit_msg>gh-52: Add single connector<commit_after>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 Weaviate. All rights reserved.
 * LICENSE: https://github.com/weaviate/weaviate/blob/master/LICENSE
 * AUTHOR: Bob van Luijt (bob@weaviate.com)
 * See www.weaviate.com for details
 * Contact: @weaviate_iot / yourfriends@weaviate.com
 */

package datastore

import (
	"context"
	"fmt"
	"log"
	"time"

	uuid "github.com/satori/go.uuid"

	"cloud.google.com/go/datastore"
)

type Datastore struct{}

type Object struct {
	Uuid         string // uuid, also used in Object's id
	Owner        string // uuid of the owner
	RefType      string // type, as defined
	CreateTimeMs int64  // creation time in ms
	Object       string // the JSON object, id will be collected from current uuid
	Deleted      bool   // if true, it does not exsist anymore
}

// Connect to datastore
func (f Datastore) Connect() bool {
	return true
}

// Add item to DB
func (f Datastore) Add(owner string, refType string, object string) string {

	// Setx your Google Cloud Platform project ID.
	ctx := context.Background()

	projectID := "weaviate-dev-001"

	// Creates a client.
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}

	// Sets the kind for the new entity.
	kind := "weaviate"

	// Sets the name/ID for the new entity.
	uuid := fmt.Sprintf("%v", uuid.NewV4())

	// Creates a Key instance.
	taskKey := datastore.NameKey(kind, uuid, nil)

	// Creates a Task instance.
	task := Object{
		Uuid:         uuid,
		Owner:        owner,
		RefType:      refType,
		CreateTimeMs: time.Now().UnixNano() / int64(time.Millisecond),
		Object:       object,
		Deleted:      false,
	}

	// Saves the new entity.
	if _, err := client.Put(ctx, taskKey, &task); err != nil {
		log.Fatalf("Failed to save task: %v", err)
	}

	// return the ID that is used to create.
	return uuid

}

func (f Datastore) Get(Uuid string) Object {

	// Setx your Google Cloud Platform project ID.
	ctx := context.Background()
	projectID := "weaviate-dev-001"

	// Creates a client.
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}

	query := datastore.NewQuery("weaviate").Filter("Uuid =", Uuid).Order("-CreateTimeMs").Limit(1)

	var object []Object

	client.GetAll(ctx, query, &object)

	return object[0]
}
<|endoftext|>


<commit_before>package quobyte

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"reflect"
	"strconv"
)

const (
	emptyResponse string = "Empty result and no error occured"
)

type request struct {
	ID      string      `json:"id"`
	Version string      `json:"jsonrpc"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params"`
}

type response struct {
	ID      string           `json:"id"`
	Version string           `json:"jsonrpc"`
	Result  *json.RawMessage `json:"result"`
	Error   *json.RawMessage `json:"error"`
}

type rpcError struct {
	Code    int64  `json:"code"`
	Message string `json:"message"`
}

func (err *rpcError) decodeErrorCode() string {
	switch err.Code {
	case -32600:
		return "ERROR_CODE_INVALID_REQUEST"
	case -32603:
		return "ERROR_CODE_JSON_ENCODING_FAILED"
	case -32601:
		return "ERROR_CODE_METHOD_NOT_FOUND"
	case -32700:
		return "ERROR_CODE_PARSE_ERROR"
	}

	return ""
}

func encodeRequest(method string, params interface{}) ([]byte, error) {
	return json.Marshal(&request{
		// Generate random ID and convert it to a string
		ID:      strconv.FormatInt(rand.Int63(), 10),
		Version: "2.0",
		Method:  method,
		Params:  params,
	})
}

func decodeResponse(ioReader io.Reader, reply interface{}) error {
	var resp response
	if err := json.NewDecoder(ioReader).Decode(&resp); err != nil {
		return err
	}

	if resp.Error != nil {
		var rpcErr rpcError
		if err := json.Unmarshal(*resp.Error, &rpcErr); err != nil {
			return err
		}

		if rpcErr.Message != "" {
			return errors.New(rpcErr.Message)
		}

		respError := rpcErr.decodeErrorCode()
		if respError != "" {
			return errors.New(respError)
		}
	}

	if resp.Result != nil && reply != nil {
		return json.Unmarshal(*resp.Result, reply)
	}

	return errors.New(emptyResponse)
}

func (client *QuobyteClient) SendRequest(method string, request interface{}, response interface{}) error {
	etype := reflect.ValueOf(request).Elem()
	field := etype.FieldByName("RetryPolicy")
	if field.IsValid() {
		field.SetString(client.GetAPIRetryPolicy())
	}
	message, err := encodeRequest(method, request)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("POST", client.url, bytes.NewBuffer(message))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	req.SetBasicAuth(client.username, client.password)
	resp, err := client.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		if resp.StatusCode == 401 {
			return errors.New("Unable to authenticate with Quobyte API service")
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return (err)
		}
		return fmt.Errorf("JsonRPC failed with error (error code: %d) %s",
			resp.StatusCode, string(body))
	}
	return decodeResponse(resp.Body, &response)
}
<commit_msg>Revert "Export rpc send to be used outside api package"<commit_after>package quobyte

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"reflect"
	"strconv"
)

const (
	emptyResponse string = "Empty result and no error occured"
)

type request struct {
	ID      string      `json:"id"`
	Version string      `json:"jsonrpc"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params"`
}

type response struct {
	ID      string           `json:"id"`
	Version string           `json:"jsonrpc"`
	Result  *json.RawMessage `json:"result"`
	Error   *json.RawMessage `json:"error"`
}

type rpcError struct {
	Code    int64  `json:"code"`
	Message string `json:"message"`
}

func (err *rpcError) decodeErrorCode() string {
	switch err.Code {
	case -32600:
		return "ERROR_CODE_INVALID_REQUEST"
	case -32603:
		return "ERROR_CODE_JSON_ENCODING_FAILED"
	case -32601:
		return "ERROR_CODE_METHOD_NOT_FOUND"
	case -32700:
		return "ERROR_CODE_PARSE_ERROR"
	}

	return ""
}

func encodeRequest(method string, params interface{}) ([]byte, error) {
	return json.Marshal(&request{
		// Generate random ID and convert it to a string
		ID:      strconv.FormatInt(rand.Int63(), 10),
		Version: "2.0",
		Method:  method,
		Params:  params,
	})
}

func decodeResponse(ioReader io.Reader, reply interface{}) error {
	var resp response
	if err := json.NewDecoder(ioReader).Decode(&resp); err != nil {
		return err
	}

	if resp.Error != nil {
		var rpcErr rpcError
		if err := json.Unmarshal(*resp.Error, &rpcErr); err != nil {
			return err
		}

		if rpcErr.Message != "" {
			return errors.New(rpcErr.Message)
		}

		respError := rpcErr.decodeErrorCode()
		if respError != "" {
			return errors.New(respError)
		}
	}

	if resp.Result != nil && reply != nil {
		return json.Unmarshal(*resp.Result, reply)
	}

	return errors.New(emptyResponse)
}

func (client QuobyteClient) sendRequest(method string, request interface{}, response interface{}) error {
	etype := reflect.ValueOf(request).Elem()
	field := etype.FieldByName("RetryPolicy")
	if field.IsValid() {
		field.SetString(client.GetAPIRetryPolicy())
	}
	message, err := encodeRequest(method, request)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("POST", client.url, bytes.NewBuffer(message))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	req.SetBasicAuth(client.username, client.password)
	resp, err := client.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		if resp.StatusCode == 401 {
			return errors.New("Unable to authenticate with Quobyte API service")
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return (err)
		}
		return fmt.Errorf("JsonRPC failed with error (error code: %d) %s",
			resp.StatusCode, string(body))
	}
	return decodeResponse(resp.Body, &response)
}
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"os"
	"path/filepath"
	"strings"
	"time"
)

/* TODO
- Clean up output formatting
- Parallelize local/remote file listing
  - Maybe find additional ways to speed up? Generating local hashes is probably
    largest bottleneck, would this benefit at all from parallelization?
- Add progress printing - maybe collect progress from remote/local listing through channels
- Ignore more file names in skipLocalFile - see https://www.dropbox.com/help/syncing-uploads/files-not-syncing
- Do a real retry + backoff for Dropbox API errors (do we have access to the Retry-After header?)
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileError records a local file that could not be read due to an error
type FileError struct {
	Path  string
	Error error
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Errored         []FileError
	Matches         int
	Misses          int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	dropboxManifest, err := getDropboxManifest(dbxClient, opts.RemoteRoot)
	if err != nil {
		panic(err)
	}

	// so we can see how many results from dropbox we're counting up to
	fmt.Println("")

	localManifest, errored, err := getLocalManifest(localRoot, opts.CheckContentHash)
	if err != nil {
		panic(err)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest, errored)

	fmt.Println("")

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")

	fmt.Printf("Errored: %d\n\n", len(manifestComparison.Errored))
	if len(manifestComparison.Errored) > 0 {
		for _, rec := range manifestComparison.Errored {
			fmt.Printf("%s: %s\n", rec.Path, rec.Error)
		}
		if len(manifestComparison.Errored) > 0 {
			fmt.Print("\n\n")
		}
	}

	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			if strings.HasPrefix(err.Error(), "too_many_requests") {
				fmt.Fprint(os.Stderr, "Dropbox returned too many requests error, sleeping 60 seconds...\n")
				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				fmt.Fprintf(os.Stderr, "Response: %v\n", resp)
				time.Sleep(60 * time.Second)
				continue
			}
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathLower)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		// DEBUG info
		fmt.Fprintf(os.Stderr, "%d dropbox files listed          \r", manifest.Len())
	}

	return
}

func getLocalManifest(localRoot string, contentHash bool) (manifest *FileHeap, errored []FileError, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	localRootLowercase := strings.ToLower(localRoot)

	err = filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
		if err != nil {
			errored = append(errored, FileError{Path: entryPath, Error: err})
			return nil
		}

		if info.Mode().IsRegular() && !skipLocalFile(entryPath) {
			relPath, err := normalizePath(localRootLowercase, strings.ToLower(entryPath))
			if err != nil {
				errored = append(errored, FileError{Path: entryPath, Error: err})
				return nil
			}

			hash := ""
			if contentHash {
				hash, err = dropbox.FileContentHash(entryPath)
				if err != nil {
					errored = append(errored, FileError{Path: relPath, Error: err})
					return nil
				}
			}

			heap.Push(manifest, &File{
				Path:        relPath,
				ContentHash: hash,
			})

			// DEBUG info
			fmt.Fprintf(os.Stderr, "%d local files listed (%d errors)    \r", manifest.Len(), len(errored))
		}

		return nil
	})

	return
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func skipLocalFile(path string) bool {
	if filepath.Base(path) == ".DS_Store" {
		return true
	}
	return false
}

func compareManifests(remoteManifest, localManifest *FileHeap, errored []FileError) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{Errored: errored}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else {
			// this must mean that remote.Path == local.Path
			if compareFileContents(remote, local) {
				comparison.Matches++
			} else {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<commit_msg>Parallelize local file processing<commit_after>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

/* TODO

- Test if buffered channels improve performance in the parallel local file processing
- Test other performance tweaks - CPU usage is still low, are the goroutines mostly idle? Could we use more than number of CPUs?
- Clean up output formatting
- Ignore more file names in skipLocalFile - see https://www.dropbox.com/help/syncing-uploads/files-not-syncing
- Do a real retry + backoff for Dropbox API errors (do we have access to the Retry-After header?)
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileError records a local file that could not be read due to an error
type FileError struct {
	Path  string
	Error error
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Errored         []*FileError
	Matches         int
	Misses          int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
		WorkerCount      int    `short:"w" long:"workers" description:"Number of worker threads to use (defaults to number of CPUs)"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	dropboxManifest, err := getDropboxManifest(dbxClient, opts.RemoteRoot)
	if err != nil {
		panic(err)
	}

	// so we can see how many results from dropbox we're counting up to
	fmt.Println("")

	localManifest, errored, err := getLocalManifest(localRoot, opts.CheckContentHash, opts.WorkerCount)
	if err != nil {
		panic(err)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest, errored)

	fmt.Println("")

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")

	fmt.Printf("Errored: %d\n\n", len(manifestComparison.Errored))
	if len(manifestComparison.Errored) > 0 {
		for _, rec := range manifestComparison.Errored {
			fmt.Printf("%s: %s\n", rec.Path, rec.Error)
		}
		if len(manifestComparison.Errored) > 0 {
			fmt.Print("\n\n")
		}
	}

	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			if strings.HasPrefix(err.Error(), "too_many_requests") {
				fmt.Fprint(os.Stderr, "Dropbox returned too many requests error, sleeping 60 seconds...\n")
				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				fmt.Fprintf(os.Stderr, "Response: %v\n", resp)
				time.Sleep(60 * time.Second)
				continue
			}
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathLower)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		// DEBUG info
		fmt.Fprintf(os.Stderr, "%d dropbox files listed          \r", manifest.Len())
	}

	return
}

func getLocalManifest(localRoot string, contentHash bool, workerCount int) (manifest *FileHeap, errored []*FileError, err error) {
	localRootLowercase := strings.ToLower(localRoot)
	manifest = &FileHeap{}
	heap.Init(manifest)
	if workerCount <= 0 {
		workerCount = int(math.Max(1, float64(runtime.NumCPU())))
	}
	processChan := make(chan string)
	resultChan := make(chan *File)
	errorChan := make(chan *FileError)
	var wg sync.WaitGroup

	for i := 0; i < workerCount; i++ {
		// spin up workers
		wg.Add(1)
		go handleLocalFile(localRootLowercase, contentHash, processChan, resultChan, errorChan, &wg)
	}

	// walk in separate goroutine so that sends to errorChan don't block
	go func() {
		filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
			if err != nil {
				errorChan <- &FileError{Path: entryPath, Error: err}
				return nil
			}

			if info.Mode().IsRegular() && !skipLocalFile(entryPath) {
				processChan <- entryPath
			}

			return nil
		})

		close(processChan)
	}()

	// Once processing goroutines are done, close result and error channels to indicate no more results streaming in
	go func() {
		wg.Wait()
		close(resultChan)
		close(errorChan)
	}()

	for {
		select {
		case result, ok := <-resultChan:
			if ok {
				heap.Push(manifest, result)
			} else {
				resultChan = nil
			}

		case e, ok := <-errorChan:
			if ok {
				errored = append(errored, e)
			} else {
				errorChan = nil
			}
		}

		// DEBUG info
		fmt.Fprintf(os.Stderr, "%d local files listed (%d errors) (%d workers)    \r", manifest.Len(), len(errored), workerCount)

		if resultChan == nil && errorChan == nil {
			break
		}
	}

	return
}

// fill in args etc
func handleLocalFile(localRootLowercase string, contentHash bool, processChan <-chan string, resultChan chan<- *File, errorChan chan<- *FileError, wg *sync.WaitGroup) {
	for entryPath := range processChan {

		relPath, err := normalizePath(localRootLowercase, strings.ToLower(entryPath))
		if err != nil {
			errorChan <- &FileError{Path: entryPath, Error: err}
			continue
		}

		hash := ""
		if contentHash {
			hash, err = dropbox.FileContentHash(entryPath)
			if err != nil {
				errorChan <- &FileError{Path: relPath, Error: err}
				continue
			}
		}

		resultChan <- &File{
			Path:        relPath,
			ContentHash: hash,
		}
	}
	wg.Done()
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func skipLocalFile(path string) bool {
	if filepath.Base(path) == ".DS_Store" {
		return true
	}
	return false
}

func compareManifests(remoteManifest, localManifest *FileHeap, errored []*FileError) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{Errored: errored}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else {
			// this must mean that remote.Path == local.Path
			if compareFileContents(remote, local) {
				comparison.Matches++
			} else {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"fmt"
	"math"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/jessevdk/go-flags"
	"github.com/pkg/profile"
	"github.com/tj/go-dropbox"

	"golang.org/x/text/unicode/norm"
)

/*
TODO
- Performance improvements:
	- Profile to find bottlenecks
	- Test if buffered channels improve performance in the parallel local file processing
	- Could printing progress for each local file result slow things down? (When processing lots of small files)
	- Print I/O usage? i.e. how many MB/s are we processing
- Clean up output formatting
	- Consolidate Dropbox error retry printing somehow? Maybe print retries on stderr, print info about error and how many retries it took to stdout
- Ignore more file names in skipLocalFile - see https://www.dropbox.com/help/syncing-uploads/files-not-syncing
- Do a real retry + backoff for Dropbox API errors (do we have access to the Retry-After header?)
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileError records a local file that could not be read due to an error
type FileError struct {
	Path  string
	Error error
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Errored         []*FileError
	Matches         int
	Misses          int
}

type progressType int

const (
	remoteProgress progressType = iota
	localProgress
	errorProgress
)

type scanProgressUpdate struct {
	Type  progressType
	Count int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose            bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot         string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:""`
		LocalRoot          string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		SkipContentHash    bool   `long:"skip-hash" description:"Skip checking content hash of local files"`
		WorkerCount        int    `short:"w" long:"workers" description:"Number of worker threads to use (defaults to 8) - set to 0 to use all CPU cores" default:"8"`
		FreeMemoryInterval int    `long:"free-memory-interval" description:"Interval (in seconds) to manually release unused memory back to the OS on low-memory systems" default:"0"`
		ProfileMemory      bool   `long:"profile-memory" description:"Generate a pprof memory profile"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	if opts.ProfileMemory {
		defer profile.Start(profile.MemProfile).Stop()
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	remoteRoot := opts.RemoteRoot
	if remoteRoot == "" {
		remoteRoot = defaultRemoteRoot(localRoot)
	}
	if remoteRoot[0] != '/' {
		remoteRoot = "/" + remoteRoot
	}

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", remoteRoot, localRoot)
	if !opts.SkipContentHash {
		fmt.Println("Checking content hashes.")
	}
	workerCount := opts.WorkerCount
	if workerCount <= 0 {
		workerCount = int(math.Max(1, float64(runtime.NumCPU())))
	}
	fmt.Printf("Using %d local worker threads.\n", workerCount)
	fmt.Println("")

	updateChan := make(chan *scanProgressUpdate)
	var wg sync.WaitGroup
	wg.Add(2)

	var dropboxManifest *FileHeap
	var dropboxErr error
	go func() {
		dropboxManifest, dropboxErr = getDropboxManifest(updateChan, dbxClient, remoteRoot)
		wg.Done()
	}()

	var localManifest *FileHeap
	var errored []*FileError
	var localErr error
	go func() {
		localDir := NewLocalDirectory(localRoot, opts.SkipContentHash, workerCount)
		localManifest, errored, localErr = localDir.Manifest(updateChan)
		wg.Done()
	}()

	go func() {
		remoteCount := 0
		localCount := 0
		errorCount := 0
		for update := range updateChan {
			switch update.Type {
			case remoteProgress:
				remoteCount = update.Count
			case localProgress:
				localCount = update.Count
			case errorProgress:
				errorCount = update.Count
			}

			if opts.Verbose {
				fmt.Fprintf(os.Stderr, "Scanning: %d (remote) %d (local) %d (errored)\r", remoteCount, localCount, errorCount)
			}
		}
		fmt.Fprintf(os.Stderr, "\n")
	}()

	// set up manual garbage collection routine
	if opts.FreeMemoryInterval > 0 {
		go func() {
			for range time.Tick(time.Duration(opts.FreeMemoryInterval) * time.Second) {
				debug.FreeOSMemory()
				if opts.Verbose {
					var m runtime.MemStats
					runtime.ReadMemStats(&m)
					fmt.Fprintf(
						os.Stderr,
						"\n[%s] HeapAlloc: %s / HeapInuse: %s / HeapReleased: %s / Sys: %s / GCCPUFraction: %.2f%%\n",
						time.Now().Format("15:04:05"),
						humanize.Bytes(m.Alloc),
						humanize.Bytes(m.HeapInuse),
						humanize.Bytes(m.HeapReleased),
						humanize.Bytes(m.Sys),
						m.GCCPUFraction*100,
					)
				}
			}
		}()
	}

	// wait until remote and local scans are complete, then close progress reporting channel
	wg.Wait()
	close(updateChan)
	fmt.Printf("\nGenerated manifests for %d remote files, %d local files, with %d local errors\n\n", dropboxManifest.Len(), localManifest.Len(), len(errored))

	// check for fatal errors
	if dropboxErr != nil {
		panic(dropboxErr)
	}
	if localErr != nil {
		panic(localErr)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest, errored)

	fmt.Println("")

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")

	fmt.Printf("Errored: %d\n\n", len(manifestComparison.Errored))
	if len(manifestComparison.Errored) > 0 {
		for _, rec := range manifestComparison.Errored {
			fmt.Printf("%s: %s\n", rec.Path, rec.Error)
		}
		if len(manifestComparison.Errored) > 0 {
			fmt.Print("\n\n")
		}
	}

	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func defaultRemoteRoot(localRoot string) string {
	relPath := ""
	for {
		dir, base := path.Split(localRoot)
		if base == "Dropbox" {
			return "/" + relPath
		} else if dir == "" {
			return "/"
		} else {
			relPath = path.Join(base, relPath)
			localRoot = strings.TrimRight(dir, "/")
		}
	}
}

func getDropboxManifest(updateChan chan<- *scanProgressUpdate, dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true
	retryCount := 0

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			// TODO: submit feature request for dropbox client to expose retry_after param
			if strings.HasPrefix(err.Error(), "too_many_requests") {
				fmt.Fprintf(os.Stderr, "\n[%s] [%d retries] Dropbox returned too many requests error, sleeping 60 seconds\n", time.Now().Format("15:04:05"), retryCount)
				// fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				// fmt.Fprintf(os.Stderr, "Response: %v\n", resp)
				retryCount++
				time.Sleep(60 * time.Second)
				continue
			} else if retryCount < 10 { // TODO extract this magic number
				fmt.Fprintf(os.Stderr, "\n[%s] [%d retries] Error: %s - sleeping 1 second and retrying\n", time.Now().Format("15:04:05"), retryCount, err)
				fmt.Fprintf(os.Stderr, "Full Error: %#v\n", err)
				retryCount++
				time.Sleep(1 * time.Second)
				continue
			} else {
				fmt.Fprintf(os.Stderr, "\n[%s] Hit maximum of %d retries; aborting.\n", time.Now().Format("15:04:05"), retryCount)
				return
			}
		}
		// call was successful, reset retryCount
		retryCount = 0
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathLower)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		updateChan <- &scanProgressUpdate{Type: remoteProgress, Count: manifest.Len()}
	}

	return
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func compareManifests(remoteManifest, localManifest *FileHeap, errored []*FileError) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{Errored: errored}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else {
			// this must mean that remote.Path == local.Path
			if compareFileContents(remote, local) {
				comparison.Matches++
			} else {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<commit_msg>Add notes on potential memory usage improvements<commit_after>package main

import (
	"container/heap"
	"fmt"
	"math"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/jessevdk/go-flags"
	"github.com/pkg/profile"
	"github.com/tj/go-dropbox"

	"golang.org/x/text/unicode/norm"
)

/*
TODO
- Performance improvements:
	- Profile to find bottlenecks
	- Test if buffered channels improve performance in the parallel local file processing
	- Could printing progress for each local file result slow things down? (When processing lots of small files)
	- Print I/O usage? i.e. how many MB/s are we processing
- Memory usage improvements (if memory continues to be an issue):
	- Could use smaller representation of content hash (hex string -> fixed size byte array) - would probably cut out 30-35% of manifest size (60-70MB on my file set)
	- Evaluate heaps earlier to avoid hanging on to the entries that already match? This would increase complexity a lot and maybe require refactoring to use a map
- Clean up output formatting
	- Consolidate Dropbox error retry printing somehow? Maybe print retries on stderr, print info about error and how many retries it took to stdout
- Ignore more file names in skipLocalFile - see https://www.dropbox.com/help/syncing-uploads/files-not-syncing
- Do a real retry + backoff for Dropbox API errors (do we have access to the Retry-After header?)
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileError records a local file that could not be read due to an error
type FileError struct {
	Path  string
	Error error
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Errored         []*FileError
	Matches         int
	Misses          int
}

type progressType int

const (
	remoteProgress progressType = iota
	localProgress
	errorProgress
)

type scanProgressUpdate struct {
	Type  progressType
	Count int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose            bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot         string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:""`
		LocalRoot          string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		SkipContentHash    bool   `long:"skip-hash" description:"Skip checking content hash of local files"`
		WorkerCount        int    `short:"w" long:"workers" description:"Number of worker threads to use (defaults to 8) - set to 0 to use all CPU cores" default:"8"`
		FreeMemoryInterval int    `long:"free-memory-interval" description:"Interval (in seconds) to manually release unused memory back to the OS on low-memory systems" default:"0"`
		ProfileMemory      bool   `long:"profile-memory" description:"Generate a pprof memory profile"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	if opts.ProfileMemory {
		defer profile.Start(profile.MemProfile).Stop()
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	remoteRoot := opts.RemoteRoot
	if remoteRoot == "" {
		remoteRoot = defaultRemoteRoot(localRoot)
	}
	if remoteRoot[0] != '/' {
		remoteRoot = "/" + remoteRoot
	}

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", remoteRoot, localRoot)
	if !opts.SkipContentHash {
		fmt.Println("Checking content hashes.")
	}
	workerCount := opts.WorkerCount
	if workerCount <= 0 {
		workerCount = int(math.Max(1, float64(runtime.NumCPU())))
	}
	fmt.Printf("Using %d local worker threads.\n", workerCount)
	fmt.Println("")

	updateChan := make(chan *scanProgressUpdate)
	var wg sync.WaitGroup
	wg.Add(2)

	var dropboxManifest *FileHeap
	var dropboxErr error
	go func() {
		dropboxManifest, dropboxErr = getDropboxManifest(updateChan, dbxClient, remoteRoot)
		wg.Done()
	}()

	var localManifest *FileHeap
	var errored []*FileError
	var localErr error
	go func() {
		localDir := NewLocalDirectory(localRoot, opts.SkipContentHash, workerCount)
		localManifest, errored, localErr = localDir.Manifest(updateChan)
		wg.Done()
	}()

	go func() {
		remoteCount := 0
		localCount := 0
		errorCount := 0
		for update := range updateChan {
			switch update.Type {
			case remoteProgress:
				remoteCount = update.Count
			case localProgress:
				localCount = update.Count
			case errorProgress:
				errorCount = update.Count
			}

			if opts.Verbose {
				fmt.Fprintf(os.Stderr, "Scanning: %d (remote) %d (local) %d (errored)\r", remoteCount, localCount, errorCount)
			}
		}
		fmt.Fprintf(os.Stderr, "\n")
	}()

	// set up manual garbage collection routine
	if opts.FreeMemoryInterval > 0 {
		go func() {
			for range time.Tick(time.Duration(opts.FreeMemoryInterval) * time.Second) {
				debug.FreeOSMemory()
				if opts.Verbose {
					var m runtime.MemStats
					runtime.ReadMemStats(&m)
					fmt.Fprintf(
						os.Stderr,
						"\n[%s] HeapAlloc: %s / HeapInuse: %s / HeapReleased: %s / Sys: %s / GCCPUFraction: %.2f%%\n",
						time.Now().Format("15:04:05"),
						humanize.Bytes(m.Alloc),
						humanize.Bytes(m.HeapInuse),
						humanize.Bytes(m.HeapReleased),
						humanize.Bytes(m.Sys),
						m.GCCPUFraction*100,
					)
				}
			}
		}()
	}

	// wait until remote and local scans are complete, then close progress reporting channel
	wg.Wait()
	close(updateChan)
	fmt.Printf("\nGenerated manifests for %d remote files, %d local files, with %d local errors\n\n", dropboxManifest.Len(), localManifest.Len(), len(errored))

	// check for fatal errors
	if dropboxErr != nil {
		panic(dropboxErr)
	}
	if localErr != nil {
		panic(localErr)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest, errored)

	fmt.Println("")

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")

	fmt.Printf("Errored: %d\n\n", len(manifestComparison.Errored))
	if len(manifestComparison.Errored) > 0 {
		for _, rec := range manifestComparison.Errored {
			fmt.Printf("%s: %s\n", rec.Path, rec.Error)
		}
		if len(manifestComparison.Errored) > 0 {
			fmt.Print("\n\n")
		}
	}

	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func defaultRemoteRoot(localRoot string) string {
	relPath := ""
	for {
		dir, base := path.Split(localRoot)
		if base == "Dropbox" {
			return "/" + relPath
		} else if dir == "" {
			return "/"
		} else {
			relPath = path.Join(base, relPath)
			localRoot = strings.TrimRight(dir, "/")
		}
	}
}

func getDropboxManifest(updateChan chan<- *scanProgressUpdate, dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true
	retryCount := 0

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			// TODO: submit feature request for dropbox client to expose retry_after param
			if strings.HasPrefix(err.Error(), "too_many_requests") {
				fmt.Fprintf(os.Stderr, "\n[%s] [%d retries] Dropbox returned too many requests error, sleeping 60 seconds\n", time.Now().Format("15:04:05"), retryCount)
				// fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				// fmt.Fprintf(os.Stderr, "Response: %v\n", resp)
				retryCount++
				time.Sleep(60 * time.Second)
				continue
			} else if retryCount < 10 { // TODO extract this magic number
				fmt.Fprintf(os.Stderr, "\n[%s] [%d retries] Error: %s - sleeping 1 second and retrying\n", time.Now().Format("15:04:05"), retryCount, err)
				fmt.Fprintf(os.Stderr, "Full Error: %#v\n", err)
				retryCount++
				time.Sleep(1 * time.Second)
				continue
			} else {
				fmt.Fprintf(os.Stderr, "\n[%s] Hit maximum of %d retries; aborting.\n", time.Now().Format("15:04:05"), retryCount)
				return
			}
		}
		// call was successful, reset retryCount
		retryCount = 0
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathLower)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		updateChan <- &scanProgressUpdate{Type: remoteProgress, Count: manifest.Len()}
	}

	return
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func compareManifests(remoteManifest, localManifest *FileHeap, errored []*FileError) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{Errored: errored}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else {
			// this must mean that remote.Path == local.Path
			if compareFileContents(remote, local) {
				comparison.Matches++
			} else {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<|endoftext|>