<commit_before>package lily

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"strconv"
)

type sendReplySt struct {
	ID        uint64 `json:"id"`
	CNT       int    `json:"cnt"`
	ErrorCode int    `json:"error_code"`
	Error     string `json:"error"`
}

type getBalanceReplySt struct {
	Balance   string `json:"balance"`
	ErrorCode int    `json:"error_code"`
	Error     string `json:"error"`
}

var (
	SmscDebug = false
)

func SmscSend(username, password string, phones string, msg string) bool {
	if SmscDebug {
		log.Printf("Sent sms: %s - %q\n", phones, msg)
		return true
	}
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, err := http.NewRequest("GET", "https://smsc.kz/sys/send.php", nil)
	ErrPanic(err)
	params := req.URL.Query()
	params.Add("login", username)
	params.Add("psw", password)
	params.Add("phones", phones)
	params.Add("mes", msg)
	params.Add("charset", "utf-8")
	params.Add("fmt", "3")
	req.URL.RawQuery = params.Encode()
	resp, err := client.Do(req)
	if err != nil {
		log.Println("(sms-send) fail to send sms:", err)
		return false
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("(sms-send) fail to read sms-send reply body:", err)
		return false
	}
	if resp.StatusCode != 200 {
		log.Printf("(sms-send) bad status code %d in sms-send reply, data: %s\n", resp.StatusCode, string(data))
		return false
	}
	reply := sendReplySt{}
	err = json.Unmarshal(data, &reply)
	if err != nil {
		log.Printf("(sms-send) fail to parse sms-send reply: %s, %s\n", err.Error(), string(data))
		return false
	}
	if (reply.ErrorCode != 0) || (reply.Error != "") {
		if reply.ErrorCode != 7 && reply.ErrorCode != 8 { // 7 - invalid number, 8 - can't to deliver
			log.Printf("(sms-send) sms provider error for (%s, %q):\n%s\n", phones, msg, string(data))
		}
		return false
	}
	return true
}

func SmscSendBcast(username, password string, phones string, msg string) (bool, uint64) {
	if SmscDebug {
		log.Printf("Sent sms-bcast: %s - %q\n", phones, msg)
		return true, 777
	}
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, err := http.NewRequest("GET", "https://smsc.kz/sys/jobs.php", nil)
	ErrPanic(err)
	params := req.URL.Query()
	params.Add("add", "1")
	params.Add("login", username)
	params.Add("psw", password)
	params.Add("name", "bcast")
	params.Add("phones", phones)
	params.Add("mes", msg)
	params.Add("charset", "utf-8")
	params.Add("fmt", "3")
	req.URL.RawQuery = params.Encode()
	resp, err := client.Do(req)
	if err != nil {
		log.Println("(sms-bcast) fail to send sms:", err)
		return false, 0
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("(sms-bcast) fail to read sms-send reply body:", err)
		return false, 0
	}
	if resp.StatusCode != 200 {
		log.Printf("(sms-bcast) bad status code %d in sms-send reply, data: %s\n", resp.StatusCode, string(data))
		return false, 0
	}
	reply := sendReplySt{}
	err = json.Unmarshal(data, &reply)
	if err != nil {
		log.Printf("(sms-bcast) fail to parse sms-send reply: %s, %s\n", err.Error(), string(data))
		return false, 0
	}
	if (reply.ErrorCode != 0) || (reply.Error != "") {
		log.Printf("(sms-bcast) sms provider error for (%s, %q):\n%s\n", phones, msg, string(data))
		return false, 0
	}
	return true, reply.ID
}

func SmscGetBalance(username, password string) (result float64) {
	result = 0
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, err := http.NewRequest("GET", "https://smsc.kz/sys/balance.php", nil)
	ErrPanic(err)
	params := req.URL.Query()
	params.Add("login", username)
	params.Add("psw", password)
	params.Add("fmt", "3")
	req.URL.RawQuery = params.Encode()
	resp, err := client.Do(req)
	if err != nil {
		log.Println("fail to get sms-balance:", err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		log.Printf("bad status code %d in sms-balance reply\n", resp.StatusCode)
		return
	}
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("fail to read sms-balance reply body:", err)
		return
	}
	reply := getBalanceReplySt{}
	err = json.Unmarshal(data, &reply)
	if err != nil {
		log.Println("fail to parse sms-balance reply:", err)
		return
	}
	if (reply.ErrorCode != 0) || (reply.Error != "") {
		log.Printf("sms provider error for getting balance:\n%s\n", string(data))
		return
	}
	result, _ = strconv.ParseFloat(reply.Balance, 64)
	return
}
<commit_msg>some fixes<commit_after>package lily

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"strconv"
)

type sendReplySt struct {
	ID        uint64 `json:"id"`
	CNT       int    `json:"cnt"`
	ErrorCode int    `json:"error_code"`
	Error     string `json:"error"`
}

type getBalanceReplySt struct {
	Balance   string `json:"balance"`
	ErrorCode int    `json:"error_code"`
	Error     string `json:"error"`
}

var (
	SmscDebug = false
)

func SmscSend(username, password string, phones string, msg string) bool {
	if SmscDebug {
		log.Printf("Sent sms: %s - %q\n", phones, msg)
		return true
	}
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, err := http.NewRequest("GET", "https://smsc.kz/sys/send.php", nil)
	ErrPanic(err)
	params := req.URL.Query()
	params.Add("login", username)
	params.Add("psw", password)
	params.Add("phones", phones)
	params.Add("mes", msg)
	params.Add("charset", "utf-8")
	params.Add("fmt", "3")
	req.URL.RawQuery = params.Encode()
	resp, err := client.Do(req)
	if err != nil {
		log.Println("(sms-send) fail to send sms:", err)
		return false
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("(sms-send) fail to read sms-send reply body:", err)
		return false
	}
	if resp.StatusCode != 200 {
		log.Printf("(sms-send) bad status code %d in sms-send reply, data: %s\n", resp.StatusCode, string(data))
		return false
	}
	reply := sendReplySt{}
	err = json.Unmarshal(data, &reply)
	if err != nil {
		log.Printf("(sms-send) fail to parse sms-send reply: %s, %s\n", err.Error(), string(data))
		return false
	}
	if (reply.ErrorCode != 0) || (reply.Error != "") {
		if reply.ErrorCode != 7 && reply.ErrorCode != 8 { // 7 - invalid number, 8 - can't to deliver
			log.Printf("(sms-send) sms provider error for (%s, %q):\n%s\n", phones, msg, string(data))
		}
		return false
	}
	return true
}

func SmscSendBcast(username, password string, phones string, msg string) (bool, uint64) {
	if SmscDebug {
		log.Printf("Sent sms-bcast: %s - %q\n", phones, msg)
		return true, 777
	}
	client := &http.Client{
		Timeout: 20 * time.Second,
	}
	req, err := http.NewRequest("GET", "https://smsc.kz/sys/jobs.php", nil)
	ErrPanic(err)
	params := req.URL.Query()
	params.Add("add", "1")
	params.Add("login", username)
	params.Add("psw", password)
	params.Add("name", "bcast")
	params.Add("phones", phones)
	params.Add("mes", msg)
	params.Add("charset", "utf-8")
	params.Add("fmt", "3")
	req.URL.RawQuery = params.Encode()
	resp, err := client.Do(req)
	if err != nil {
		log.Println("(sms-bcast) fail to send sms:", err)
		return false, 0
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("(sms-bcast) fail to read sms-send reply body:", err)
		return false, 0
	}
	if resp.StatusCode != 200 {
		log.Printf("(sms-bcast) bad status code %d in sms-send reply, data: %s\n", resp.StatusCode, string(data))
		return false, 0
	}
	reply := sendReplySt{}
	err = json.Unmarshal(data, &reply)
	if err != nil {
		log.Printf("(sms-bcast) fail to parse sms-send reply: %s, %s\n", err.Error(), string(data))
		return false, 0
	}
	if (reply.ErrorCode != 0) || (reply.Error != "") {
		log.Printf("(sms-bcast) sms provider error for (%s, %q):\n%s\n", phones, msg, string(data))
		return false, 0
	}
	return true, reply.ID
}

func SmscGetBalance(username, password string) (bool, float64) {
	var result float64

	client := &http.Client{
		Timeout: 20 * time.Second,
	}

	req, err := http.NewRequest("GET", "https://smsc.kz/sys/balance.php", nil)
	if err != nil {
		return false, 0
	}

	params := req.URL.Query()
	params.Add("login", username)
	params.Add("psw", password)
	params.Add("fmt", "3")

	req.URL.RawQuery = params.Encode()

	resp, err := client.Do(req)
	if err != nil {
		log.Println("fail to get sms-balance:", err)
		return false, 0
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		log.Printf("bad status code %d in sms-balance reply\n", resp.StatusCode)
		return false, 0
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("fail to read sms-balance reply body:", err)
		return false, 0
	}

	reply := getBalanceReplySt{}
	err = json.Unmarshal(data, &reply)
	if err != nil {
		log.Println("fail to parse sms-balance reply:", err)
		return false, 0
	}

	if (reply.ErrorCode != 0) || (reply.Error != "") {
		log.Printf("sms provider error for getting balance:\n%s\n", string(data))
		return false, 0
	}

	result, _ = strconv.ParseFloat(reply.Balance, 64)

	return true, result
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/user"
	"github.com/timob/list"
	"fmt"
	"log"
	"strings"
	"syscall"
	"unsafe"
	"path"
	"github.com/bradfitz/slice"
	"time"
)

type DisplayEntry struct {
	path string
	os.FileInfo
}

type DisplayEntryList struct {
	Data []DisplayEntry
	list.Slice
}

var now = time.Now()

var showDirEntries bool
var showAll bool
var showAlmostAll bool
var longList bool
const (
	name int = iota
	modTime int = iota
	size int = iota
)
var sortType int = name
var reverseSort bool
var humanReadable bool

var width int

func getTermSize() (int, int, error) {
	var dimensions [4]uint16

	fd := os.Stdout.Fd()
	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TIOCGWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
		return -1, -1, err
	}
	return int(dimensions[1]), int(dimensions[0]), nil
}

func human(n int64) string {
	var i int64
	var w = n
	var uSize int64 = 1
	for i = 0; i < 12; i++ {
		if w / 1024 == 0 {
			break
		}
		w = w / 1024
		uSize *= 1024
	}

	var f int64
	var unit string
	f = (n - w * uSize)
	if f != 0 && i > 0{
		if w < 10 {
			if f != 0 {
				lowerSize := uSize / 1024
				// magic plus one here (seems to be what GNU ls does)
				tenth := int64(1024 / 10) + 1
				f = f / lowerSize / tenth
				// round up
				f++
				if f == 10 {
					w++
					f = 0
				}
			}
		} else {
			// round up
			w++
		}
	}

	switch i {
	case 1:
		unit = "K"
	case 2:
		unit = "M"
	case 3:
		unit = "G"
	}

	if w > 0 && w < 10 {
		return fmt.Sprintf("%d.%d%s", w, f, unit)
	} else {
		return fmt.Sprintf("%d%s", w, unit)
	}
}

func decimalLen(n int64) (i int) {
	for i = 1; i < 12; i++ {
		if n / 10 == 0 {
			break
		}
		n = n / 10
	}
	return
}

var userLookupCache = make(map[string]string)
func userLookUp(id string) (string, error) {
	if v, ok := userLookupCache[id]; ok {
		return v, nil
	} else {
		u, err := user.LookupId(id)
		if err == nil {
			userLookupCache[id] = u.Name
			return u.Name, nil
		}
		return "", err
	}
}

type longInfo struct {
	userName, groupName string
	hardLinks int
}

func getLongInfo(info os.FileInfo) *longInfo {
	stat := info.Sys().(*syscall.Stat_t)
	userName := fmt.Sprintf("%d", stat.Uid)
	if u, err := userLookUp(userName); err == nil {
		userName = u
	}
	group := fmt.Sprintf("%d", stat.Gid)
	if g, err := userLookUp(group); err == nil {
		group = g
	}
	return &longInfo{userName, group, int(stat.Nlink)}
}

func strcmpi(a, b string) int {
	for i, av := range a {
		if i >= len(b) {
			return 1
		}
		if av > 96 && av < 123 {
			av -= 32
		}
		bv := rune(b[i])
		if bv > 96 && bv < 123 {
			bv -= 32
		}

		if av != bv {
			if av > bv {
				return 1
			} else {
				return -1
			}
		}
	}

	if len(b) > len(a) {
		return -1
	} else {
		return 0
	}
}

func modeString(mode os.FileMode) string {
	output := []byte(strings.Repeat("-", 10))
	if mode & os.ModeDir != 0{
		output[0] = 'd'
	} else if mode & os.ModeSymlink != 0{
		output[0] = 'l'
	} else if mode & os.ModeNamedPipe != 0{
		output[0] = 'p'
	} else if mode & os.ModeSocket != 0{
		output[0] = 's'
	} else if mode & os.ModeCharDevice != 0 && mode & os.ModeDevice != 0 {
		output[0] = 'c'
	}

	const rwx = "rwxrwxrwx"
	for i, c := range rwx {
		bitSet := mode&(1<<uint(9-1-i)) != 0
		if bitSet {
			if (i == 2 && mode & os.ModeSetuid != 0) || (i == 5 && mode & os.ModeSetgid != 0) {
				output[i+1] = 's'
			} else if i == 8 && mode & os.ModeSticky != 0 {
				output[i+1] = 't'
			} else {
				output[i+1] = byte(c)
			}
		} else if (i == 2 && mode & os.ModeSetuid != 0) || (i == 5 && mode & os.ModeSetgid != 0) {
			output[i+1] = 'S'
		} else if i == 8 && mode & os.ModeSticky != 0 {
			output[i+1] = 'T'
		}
	}

	return string(output)
}

func display(selected []DisplayEntry, root string) {
	slice.Sort(selected, func(i, j int) (v bool) {
		var same bool
		if sortType == modTime {
			v = selected[i].ModTime().Before(selected[j].ModTime())
			if !v {
				same = selected[i].ModTime().Equal(selected[	j].ModTime())
			}
			v = !v
		} else if sortType == size {
			d := selected[j].Size() - selected[i].Size()
			if d > 0 {
				v = true
			} else if d == 0 {
				same = true
			}
			v = !v
		} else {
			// strcoll?
			v = strcmpi(selected[i].path, selected[j].path) == -1
		}
		if same {
			v = strcmpi(selected[i].path, selected[j].path) == -1
		}

		if reverseSort {
			v = !v
		}
		return
	})

	padding := 2
	smallestWord := 1
	var cols int
	var colWidths []int

	if longList {
		cols = 4
		colWidths = make([]int, cols)
		for _, v := range selected {
			li := getLongInfo(v)
			if decimalLen(int64(li.hardLinks)) > colWidths[0] {
				colWidths[0] = decimalLen(int64(li.hardLinks))
			}
			if len(li.userName) > colWidths[1] {
				colWidths[1] = len(li.userName)
			}
			if len(li.groupName) > colWidths[2] {
				colWidths[2] = len(li.groupName)
			}
			if humanReadable {
				if len(human(v.Size())) > colWidths[3] {
					colWidths[3] = len(human(v.Size()))
				}
			} else {
				if decimalLen(v.Size()) > colWidths[3] {
					colWidths[3] = decimalLen(v.Size())
				}
			}
		}
	} else {
		cols = width / (padding + smallestWord)
		colWidths = make([]int, cols)
		A:
		for cols > 1 {
			colWidths = colWidths[:cols]
			for i := range colWidths {
				colWidths[i] = 0
			}
			pos := (cols - 1) * padding
			for i, v := range selected {
				p := i % cols
				if len(v.path) > colWidths[p] {
					pos += len(v.path) - colWidths[p]
					if pos >= width {
						cols--
						continue A
					}
					colWidths[p] = len(v.path)
				}
			}
			break
		}
	}

	for i, v := range selected {
		if longList {
			li := getLongInfo(v)
			var timeStr string
			if now.Year() == v.ModTime().Year() {
				timeStr = v.ModTime().Format("Jan _2 15:04")
			} else {
				timeStr = v.ModTime().Format("Jan _2  2006")
			}
			linkPad := strings.Repeat(" ", colWidths[0] - decimalLen(int64(li.hardLinks)))
			userPad := strings.Repeat(" ", colWidths[1] - len(li.userName))
			groupPad := strings.Repeat(" ", colWidths[2] - len(li.groupName))
			name := v.path
			if v.Mode() & os.ModeSymlink != 0 {
				if l, err  := os.Readlink(root + v.path); err == nil {
					name = name + " -> " + l
				} else {
					log.Print(err)
				}
			}
			var sizeStr string
			if humanReadable {
				sizeStr = human(v.Size())
			} else {
				sizeStr = fmt.Sprintf("%d", v.Size())
			}
			sizePad := strings.Repeat(" ", colWidths[3] - len(sizeStr))

			fmt.Printf("%s %s%d %s%s %s%s %s%s %s %s\n", modeString(v.Mode()) , linkPad,
				li.hardLinks, li.userName, userPad, li.groupName, groupPad, sizePad, sizeStr, timeStr, name)
		} else {
			w := colWidths[i % cols]
			if i % cols == 0 {
				if i != 0 {
					fmt.Println()
				}
			}
			fmt.Printf("%s", v.path)
			fmt.Print(strings.Repeat(" ", (w - len(v.path)) + padding))
		}
	}
	if !longList {
		fmt.Println()
	}
}

func main() {
	files := list.NewSliceList(&list.StringSlice{Data:os.Args}).(*list.StringSlice)
	options := list.NewSliceList(&list.StringSlice{}).(*list.StringSlice)

	files.Remove(0)
	for iter := files.Iterator(0); iter.Next(); {
		if v := files.Data[iter.Pos()]; strings.HasPrefix(v, "-") {
			options.Data[options.Append()] = v
			iter.Remove()
			if v == "--" {
				break
			}
		}
	}

	if files.Len() == 0 {
		files.Data[files.Append()] = "."
	}

	for iter := options.Iterator(0); iter.Next(); {
		if option := options.Data[iter.Pos()]; !strings.HasPrefix(option, "--") && len(option) > 2 {
			letters := list.NewSliceList(&list.ByteSlice{Data:[]byte(option[1:])}).(*list.ByteSlice)
			var removed bool
			for iter2 := letters.Iterator(letters.Len() - 1); iter2.Prev(); {
				options.Data[iter.Insert()] = "-" + string(letters.Data[iter2.Pos()])
				if !removed {
					iter.Remove()
					removed = true
				}
				iter.Prev()
			}
		}

		switch options.Data[iter.Pos()] {
		case "-d":
			showDirEntries = true
		case "-a":
			showAll = true
		case "-A":
			showAlmostAll = true
			showAll = true
		case "-t":
			sortType = modTime
		case "-S":
			sortType = size
		case "-r":
			reverseSort = true
		case "-l":
			longList = true
		case "-h":
			humanReadable = true
		default:
			log.Fatalf("unkown option %s", options.Data[iter.Pos()])
		}
	}

	if w, _, err := getTermSize(); err == nil {
		width = w
	} else {
		width = 80
	}

	selected := list.NewSliceList(&DisplayEntryList{}).(*DisplayEntryList)

	for iter := files.Iterator(0); iter.Next(); {
		fileName := files.Data[iter.Pos()]
		if showDirEntries {
			if stat, err := os.Lstat(fileName); err == nil {
				selected.Data[selected.Append()] = DisplayEntry{fileName, stat}
			} else {
				log.Print(err)
			}
			iter.Remove()
		} else {
			if stat, err := os.Lstat(fileName); err == nil {
				if stat.IsDir() {
					continue
				} else {
					selected.Data[selected.Append()] = DisplayEntry{fileName, stat}
					iter.Remove()
				}
			} else {
				log.Print(err)
				iter.Remove()
			}
		}
	}

	if selected.Len() > 0 {
		display(selected.Data, "")
	}

	// directories
	for iter := files.Iterator(0); iter.Next(); {
		fileName := files.Data[iter.Pos()]

		if selected.Len() > 0 {
			selected.Clear()
			fmt.Println()
			fmt.Printf("%s:\n", fileName)
		} else if files.Len() > 1 {
			fmt.Printf("%s:\n", fileName)
		}

		var total int64 = 0
		if file, err := os.Open(fileName); err == nil {
			if fileInfos, err := file.Readdir(0); err == nil {
				if showAll && !showAlmostAll {
					if stat, err := os.Stat(fileName); err == nil {
						selected.Data[selected.Append()] = DisplayEntry{".", stat}
					} else {
						log.Print(err)
					}
					if parent, err := os.Stat(path.Clean(fileName + "/..")); err == nil {
						selected.Data[selected.Append()] = DisplayEntry{"..", parent}
					} else {
						log.Print(err)
					}
				}
				for _, v := range fileInfos {
					if showAll || !strings.HasPrefix(v.Name(), ".") {
						total += v.Size()
						selected.Data[selected.Append()] = DisplayEntry{v.Name(), v}
					}
				}
			} else {
				log.Print(err)
			}
		} else {
			log.Print(err)
		}

		if longList {
			if humanReadable {
				fmt.Printf("total %s\n", human(total))
			} else {
				fmt.Printf("total %d\n", total/1024)
			}
		}
		display(selected.Data, fileName + "/")
	}
}
<commit_msg>fix human<commit_after>package main

import (
	"os"
	"os/user"
	"github.com/timob/list"
	"fmt"
	"log"
	"strings"
	"syscall"
	"unsafe"
	"path"
	"github.com/bradfitz/slice"
	"time"
)

type DisplayEntry struct {
	path string
	os.FileInfo
}

type DisplayEntryList struct {
	Data []DisplayEntry
	list.Slice
}

var now = time.Now()

var showDirEntries bool
var showAll bool
var showAlmostAll bool
var longList bool
const (
	name int = iota
	modTime int = iota
	size int = iota
)
var sortType int = name
var reverseSort bool
var humanReadable bool

var width int

func getTermSize() (int, int, error) {
	var dimensions [4]uint16

	fd := os.Stdout.Fd()
	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TIOCGWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
		return -1, -1, err
	}
	return int(dimensions[1]), int(dimensions[0]), nil
}

func human(n int64) string {
	var i int64
	var w = n
	var uSize int64 = 1
	for i = 0; i < 12; i++ {
		if w / 1024 == 0 {
			break
		}
		w = w / 1024
		uSize *= 1024
	}

	var f int64
	var unit string
	f = (n - w * uSize)
	if f != 0 && i > 0{
		if w < 10 {
			if f != 0 {
				lowerSize := uSize / 1024
				// magic plus one here (seems to be what GNU ls does)
				tenth := int64(1024 / 10) + 1
				f = f / lowerSize / tenth
				// round up
				f++
				if f == 10 {
					w++
					f = 0
				}
			}
		} else {
			// round up
			w++
		}
	}

	switch i {
	case 1:
		unit = "K"
	case 2:
		unit = "M"
	case 3:
		unit = "G"
	case 4:
		unit = "T"
	}

	if w > 0 && w < 10 {
		return fmt.Sprintf("%d.%d%s", w, f, unit)
	} else {
		return fmt.Sprintf("%d%s", w, unit)
	}
}

func decimalLen(n int64) (i int) {
	for i = 1; i < 12; i++ {
		if n / 10 == 0 {
			break
		}
		n = n / 10
	}
	return
}

var userLookupCache = make(map[string]string)
func userLookUp(id string) (string, error) {
	if v, ok := userLookupCache[id]; ok {
		return v, nil
	} else {
		u, err := user.LookupId(id)
		if err == nil {
			userLookupCache[id] = u.Name
			return u.Name, nil
		}
		return "", err
	}
}

type longInfo struct {
	userName, groupName string
	hardLinks int
}

func getLongInfo(info os.FileInfo) *longInfo {
	stat := info.Sys().(*syscall.Stat_t)
	userName := fmt.Sprintf("%d", stat.Uid)
	if u, err := userLookUp(userName); err == nil {
		userName = u
	}
	group := fmt.Sprintf("%d", stat.Gid)
	if g, err := userLookUp(group); err == nil {
		group = g
	}
	return &longInfo{userName, group, int(stat.Nlink)}
}

func strcmpi(a, b string) int {
	for i, av := range a {
		if i >= len(b) {
			return 1
		}
		if av > 96 && av < 123 {
			av -= 32
		}
		bv := rune(b[i])
		if bv > 96 && bv < 123 {
			bv -= 32
		}

		if av != bv {
			if av > bv {
				return 1
			} else {
				return -1
			}
		}
	}

	if len(b) > len(a) {
		return -1
	} else {
		return 0
	}
}

func modeString(mode os.FileMode) string {
	output := []byte(strings.Repeat("-", 10))
	if mode & os.ModeDir != 0{
		output[0] = 'd'
	} else if mode & os.ModeSymlink != 0{
		output[0] = 'l'
	} else if mode & os.ModeNamedPipe != 0{
		output[0] = 'p'
	} else if mode & os.ModeSocket != 0{
		output[0] = 's'
	} else if mode & os.ModeCharDevice != 0 && mode & os.ModeDevice != 0 {
		output[0] = 'c'
	}

	const rwx = "rwxrwxrwx"
	for i, c := range rwx {
		bitSet := mode&(1<<uint(9-1-i)) != 0
		if bitSet {
			if (i == 2 && mode & os.ModeSetuid != 0) || (i == 5 && mode & os.ModeSetgid != 0) {
				output[i+1] = 's'
			} else if i == 8 && mode & os.ModeSticky != 0 {
				output[i+1] = 't'
			} else {
				output[i+1] = byte(c)
			}
		} else if (i == 2 && mode & os.ModeSetuid != 0) || (i == 5 && mode & os.ModeSetgid != 0) {
			output[i+1] = 'S'
		} else if i == 8 && mode & os.ModeSticky != 0 {
			output[i+1] = 'T'
		}
	}

	return string(output)
}

func display(selected []DisplayEntry, root string) {
	slice.Sort(selected, func(i, j int) (v bool) {
		var same bool
		if sortType == modTime {
			v = selected[i].ModTime().Before(selected[j].ModTime())
			if !v {
				same = selected[i].ModTime().Equal(selected[	j].ModTime())
			}
			v = !v
		} else if sortType == size {
			d := selected[j].Size() - selected[i].Size()
			if d > 0 {
				v = true
			} else if d == 0 {
				same = true
			}
			v = !v
		} else {
			// strcoll?
			v = strcmpi(selected[i].path, selected[j].path) == -1
		}
		if same {
			v = strcmpi(selected[i].path, selected[j].path) == -1
		}

		if reverseSort {
			v = !v
		}
		return
	})

	padding := 2
	smallestWord := 1
	var cols int
	var colWidths []int

	if longList {
		cols = 4
		colWidths = make([]int, cols)
		for _, v := range selected {
			li := getLongInfo(v)
			if decimalLen(int64(li.hardLinks)) > colWidths[0] {
				colWidths[0] = decimalLen(int64(li.hardLinks))
			}
			if len(li.userName) > colWidths[1] {
				colWidths[1] = len(li.userName)
			}
			if len(li.groupName) > colWidths[2] {
				colWidths[2] = len(li.groupName)
			}
			if humanReadable {
				if len(human(v.Size())) > colWidths[3] {
					colWidths[3] = len(human(v.Size()))
				}
			} else {
				if decimalLen(v.Size()) > colWidths[3] {
					colWidths[3] = decimalLen(v.Size())
				}
			}
		}
	} else {
		cols = width / (padding + smallestWord)
		colWidths = make([]int, cols)
		A:
		for cols > 1 {
			colWidths = colWidths[:cols]
			for i := range colWidths {
				colWidths[i] = 0
			}
			pos := (cols - 1) * padding
			for i, v := range selected {
				p := i % cols
				if len(v.path) > colWidths[p] {
					pos += len(v.path) - colWidths[p]
					if pos >= width {
						cols--
						continue A
					}
					colWidths[p] = len(v.path)
				}
			}
			break
		}
	}

	for i, v := range selected {
		if longList {
			li := getLongInfo(v)
			var timeStr string
			if now.Year() == v.ModTime().Year() {
				timeStr = v.ModTime().Format("Jan _2 15:04")
			} else {
				timeStr = v.ModTime().Format("Jan _2  2006")
			}
			linkPad := strings.Repeat(" ", colWidths[0] - decimalLen(int64(li.hardLinks)))
			userPad := strings.Repeat(" ", colWidths[1] - len(li.userName))
			groupPad := strings.Repeat(" ", colWidths[2] - len(li.groupName))
			name := v.path
			if v.Mode() & os.ModeSymlink != 0 {
				if l, err  := os.Readlink(root + v.path); err == nil {
					name = name + " -> " + l
				} else {
					log.Print(err)
				}
			}
			var sizeStr string
			if humanReadable {
				sizeStr = human(v.Size())
			} else {
				sizeStr = fmt.Sprintf("%d", v.Size())
			}
			sizePad := strings.Repeat(" ", colWidths[3] - len(sizeStr))

			fmt.Printf("%s %s%d %s%s %s%s %s%s %s %s\n", modeString(v.Mode()) , linkPad,
				li.hardLinks, li.userName, userPad, li.groupName, groupPad, sizePad, sizeStr, timeStr, name)
		} else {
			w := colWidths[i % cols]
			if i % cols == 0 {
				if i != 0 {
					fmt.Println()
				}
			}
			fmt.Printf("%s", v.path)
			fmt.Print(strings.Repeat(" ", (w - len(v.path)) + padding))
		}
	}
	if !longList {
		fmt.Println()
	}
}

func main() {
	files := list.NewSliceList(&list.StringSlice{Data:os.Args}).(*list.StringSlice)
	options := list.NewSliceList(&list.StringSlice{}).(*list.StringSlice)

	files.Remove(0)
	for iter := files.Iterator(0); iter.Next(); {
		if v := files.Data[iter.Pos()]; strings.HasPrefix(v, "-") {
			options.Data[options.Append()] = v
			iter.Remove()
			if v == "--" {
				break
			}
		}
	}

	if files.Len() == 0 {
		files.Data[files.Append()] = "."
	}

	for iter := options.Iterator(0); iter.Next(); {
		if option := options.Data[iter.Pos()]; !strings.HasPrefix(option, "--") && len(option) > 2 {
			letters := list.NewSliceList(&list.ByteSlice{Data:[]byte(option[1:])}).(*list.ByteSlice)
			var removed bool
			for iter2 := letters.Iterator(letters.Len() - 1); iter2.Prev(); {
				options.Data[iter.Insert()] = "-" + string(letters.Data[iter2.Pos()])
				if !removed {
					iter.Remove()
					removed = true
				}
				iter.Prev()
			}
		}

		switch options.Data[iter.Pos()] {
		case "-d":
			showDirEntries = true
		case "-a":
			showAll = true
		case "-A":
			showAlmostAll = true
			showAll = true
		case "-t":
			sortType = modTime
		case "-S":
			sortType = size
		case "-r":
			reverseSort = true
		case "-l":
			longList = true
		case "-h":
			humanReadable = true
		default:
			log.Fatalf("unkown option %s", options.Data[iter.Pos()])
		}
	}

	if w, _, err := getTermSize(); err == nil {
		width = w
	} else {
		width = 80
	}

	selected := list.NewSliceList(&DisplayEntryList{}).(*DisplayEntryList)

	for iter := files.Iterator(0); iter.Next(); {
		fileName := files.Data[iter.Pos()]
		if showDirEntries {
			if stat, err := os.Lstat(fileName); err == nil {
				selected.Data[selected.Append()] = DisplayEntry{fileName, stat}
			} else {
				log.Print(err)
			}
			iter.Remove()
		} else {
			if stat, err := os.Lstat(fileName); err == nil {
				if stat.IsDir() {
					continue
				} else {
					selected.Data[selected.Append()] = DisplayEntry{fileName, stat}
					iter.Remove()
				}
			} else {
				log.Print(err)
				iter.Remove()
			}
		}
	}

	if selected.Len() > 0 {
		display(selected.Data, "")
	}

	// directories
	for iter := files.Iterator(0); iter.Next(); {
		fileName := files.Data[iter.Pos()]

		if selected.Len() > 0 {
			selected.Clear()
			fmt.Println()
			fmt.Printf("%s:\n", fileName)
		} else if files.Len() > 1 {
			fmt.Printf("%s:\n", fileName)
		}

		var total int64 = 0
		if file, err := os.Open(fileName); err == nil {
			if fileInfos, err := file.Readdir(0); err == nil {
				if showAll && !showAlmostAll {
					if stat, err := os.Stat(fileName); err == nil {
						selected.Data[selected.Append()] = DisplayEntry{".", stat}
					} else {
						log.Print(err)
					}
					if parent, err := os.Stat(path.Clean(fileName + "/..")); err == nil {
						selected.Data[selected.Append()] = DisplayEntry{"..", parent}
					} else {
						log.Print(err)
					}
				}
				for _, v := range fileInfos {
					if showAll || !strings.HasPrefix(v.Name(), ".") {
						total += v.Size()
						selected.Data[selected.Append()] = DisplayEntry{v.Name(), v}
					}
				}
			} else {
				log.Print(err)
			}
		} else {
			log.Print(err)
		}

		if longList {
			if humanReadable {
				fmt.Printf("total %s\n", human(total))
			} else {
				fmt.Printf("total %d\n", total/1024)
			}
		}
		display(selected.Data, fileName + "/")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013, Peter H. Froehlich. All rights reserved.
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file.

// Package ratelimit provides a rate-limiting wrapper for net.Conn
// network connections.
//
// Our approach to rate-limiting is somewhat crude: We simply sleep
// for an appropriate amount of time after each Read or Write. This
// assumes (for Read limiting anyway) that TCP's congestion control
// mechanisms eventually "catch on" and reduce the amount of traffic
// they send our way.
package ratelimit

import (
	"errors"
	"net"
	"time"
)

type RateLimitedConn struct {
	net.Conn               // underlying network connection
	rlim, wlim   int       // in bytes/second, 0 means no limit
	rtime, wtime time.Time // time of last actual read/write
}

// New returns a rate-limited connection based on the given connection.
// The limits are specified in bytes per second (bps).
//
// Note that rate-limiting doesn't take connection deadlines into account
// (see SetDeadline, SetReadDeadline, and SetWriteDeadline) so be careful
// when using both.
func New(conn net.Conn, readLimit, writeLimit int) (rlc net.Conn, err error) {
	if readLimit < 0 || writeLimit < 0 {
		err = errors.New("read/write limits cannot be negative")
		return
	}
	rlc = RateLimitedConn{Conn: conn, rlim: readLimit, wlim: writeLimit}
	return
}

// Read reads data from the connection.
// If necessary this function will sleep for an appropriate amount
// of time to achieve the requested rate-limit.
func (rlc RateLimitedConn) Read(b []byte) (n int, err error) {
	// fast path if there is no limit
	if rlc.rlim <= 0 {
		n, err = rlc.Conn.Read(b)
		return
	}

	// lazy initialization
	if rlc.rtime.IsZero() {
		rlc.rtime = time.Now()
	}

	// perform the read operation
	n, err = rlc.Conn.Read(b)

	// how long since the last read?
	t := time.Now()
	d := t.Sub(rlc.rtime).Nanoseconds()

	// allowed time
	timePerByteInNano := time.Second.Nanoseconds() / int64(rlc.rlim)
	timeForThisWriteInNano := timePerByteInNano * int64(n)

	// sleep if we have to
	if n > 0 && d < timeForThisWriteInNano {
		time.Sleep(time.Duration(timeForThisWriteInNano - d))
	}

	// remember when last read finished
	rlc.rtime = t
	return
}

// Write writes data to the connection.
// If necessary this function will sleep for an appropriate amount
// of time to achieve the requested rate-limit.
func (rlc RateLimitedConn) Write(b []byte) (n int, err error) {
	// fast path if there is no limit
	if rlc.wlim <= 0 {
		n, err = rlc.Conn.Write(b)
		return
	}

	// lazy initialization
	if rlc.wtime.IsZero() {
		rlc.wtime = time.Now()
	}

	// perform the write operation
	n, err = rlc.Conn.Write(b)

	// how long since the last write?
	t := time.Now()
	d := t.Sub(rlc.wtime).Nanoseconds()

	// allowed time
	timePerByteInNano := time.Second.Nanoseconds() / int64(rlc.wlim)
	timeForThisWriteInNano := timePerByteInNano * int64(n)

	// sleep if we have to
	if n > 0 && d < timeForThisWriteInNano {
		time.Sleep(time.Duration(timeForThisWriteInNano - d))
	}

	// remember when last write finished
	rlc.wtime = t
	return
}

// SetReadLimit establishes a new limit (in bytes per second) for
// reading from this connection.
func (rlc RateLimitedConn) SetReadLimit(lim int) (err error) {
	if lim < 0 {
		err = errors.New("read limit cannot be negative")
		return
	}
	rlc.rlim = lim
	return
}

// SetWriteLimit establishes a new limit (in bytes per second) for
// writing to this connection.
func (rlc RateLimitedConn) SetWriteLimit(lim int) (err error) {
	if lim < 0 {
		err = errors.New("write limit cannot be negative")
		return
	}
	rlc.wlim = lim
	return
}
<commit_msg>Mention 0 limit in documentation.<commit_after>// Copyright (c) 2013, Peter H. Froehlich. All rights reserved.
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file.

// Package ratelimit provides a rate-limiting wrapper for net.Conn
// network connections.
//
// Our approach to rate-limiting is somewhat crude: We simply sleep
// for an appropriate amount of time after each Read or Write. This
// assumes (for Read limiting anyway) that TCP's congestion control
// mechanisms eventually "catch on" and reduce the amount of traffic
// they send our way.
package ratelimit

import (
	"errors"
	"net"
	"time"
)

type RateLimitedConn struct {
	net.Conn               // underlying network connection
	rlim, wlim   int       // in bytes/second, 0 means no limit
	rtime, wtime time.Time // time of last actual read/write
}

// New returns a rate-limited connection based on the given connection.
// The limits are specified in bytes per second (bps) and 0 means no
// limit.
//
// Note that rate-limiting doesn't take connection deadlines into account
// (see SetDeadline, SetReadDeadline, and SetWriteDeadline) so be careful
// when using both.
func New(conn net.Conn, readLimit, writeLimit int) (rlc net.Conn, err error) {
	if readLimit < 0 || writeLimit < 0 {
		err = errors.New("read/write limits cannot be negative")
		return
	}
	rlc = RateLimitedConn{Conn: conn, rlim: readLimit, wlim: writeLimit}
	return
}

// Read reads data from the connection.
// If necessary this function will sleep for an appropriate amount
// of time to achieve the requested rate-limit.
func (rlc RateLimitedConn) Read(b []byte) (n int, err error) {
	// fast path if there is no limit
	if rlc.rlim <= 0 {
		n, err = rlc.Conn.Read(b)
		return
	}

	// lazy initialization
	if rlc.rtime.IsZero() {
		rlc.rtime = time.Now()
	}

	// perform the read operation
	n, err = rlc.Conn.Read(b)

	// how long since the last read?
	t := time.Now()
	d := t.Sub(rlc.rtime).Nanoseconds()

	// allowed time
	timePerByteInNano := time.Second.Nanoseconds() / int64(rlc.rlim)
	timeForThisWriteInNano := timePerByteInNano * int64(n)

	// sleep if we have to
	if n > 0 && d < timeForThisWriteInNano {
		time.Sleep(time.Duration(timeForThisWriteInNano - d))
	}

	// remember when last read finished
	rlc.rtime = t
	return
}

// Write writes data to the connection.
// If necessary this function will sleep for an appropriate amount
// of time to achieve the requested rate-limit.
func (rlc RateLimitedConn) Write(b []byte) (n int, err error) {
	// fast path if there is no limit
	if rlc.wlim <= 0 {
		n, err = rlc.Conn.Write(b)
		return
	}

	// lazy initialization
	if rlc.wtime.IsZero() {
		rlc.wtime = time.Now()
	}

	// perform the write operation
	n, err = rlc.Conn.Write(b)

	// how long since the last write?
	t := time.Now()
	d := t.Sub(rlc.wtime).Nanoseconds()

	// allowed time
	timePerByteInNano := time.Second.Nanoseconds() / int64(rlc.wlim)
	timeForThisWriteInNano := timePerByteInNano * int64(n)

	// sleep if we have to
	if n > 0 && d < timeForThisWriteInNano {
		time.Sleep(time.Duration(timeForThisWriteInNano - d))
	}

	// remember when last write finished
	rlc.wtime = t
	return
}

// SetReadLimit establishes a new limit (in bytes per second, 0 for
// no limit) for reading from this connection.
func (rlc RateLimitedConn) SetReadLimit(lim int) (err error) {
	if lim < 0 {
		err = errors.New("read limit cannot be negative")
		return
	}
	rlc.rlim = lim
	return
}

// SetWriteLimit establishes a new limit (in bytes per second, 0 for
// no limit) for writing to this connection.
func (rlc RateLimitedConn) SetWriteLimit(lim int) (err error) {
	if lim < 0 {
		err = errors.New("write limit cannot be negative")
		return
	}
	rlc.wlim = lim
	return
}
<|endoftext|>


<commit_before>package instructions

import "jvmgo/rtda"

// Branch if int comparison with zero succeeds
type ifeq struct {BranchInstruction}
func (self *ifeq) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    val := stack.PopInt()
    if val == 0 {
        // todo
    }
}

type ifne struct {BranchInstruction}
func (self *ifne) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    val := stack.PopInt()
    if val != 0 {
        // todo
    }
}

type iflt struct {BranchInstruction}
func (self *iflt) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    val := stack.PopInt()
    if val < 0 {
        // todo
    }
}

type ifle struct {BranchInstruction}
func (self *ifle) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    val := stack.PopInt()
    if val <= 0 {
        // todo
    }
}

type ifgt struct {BranchInstruction}
func (self *ifgt) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    val := stack.PopInt()
    if val > 0 {
        // todo
    }
}

type ifge struct {BranchInstruction}
func (self *ifge) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    val := stack.PopInt()
    if val >= 0 {
        // todo
    }
}
<commit_msg>code refactor<commit_after>package instructions

import "jvmgo/rtda"

// Branch if int comparison with zero succeeds
type ifeq struct {BranchInstruction}
func (self *ifeq) execute(thread *rtda.Thread) {
    val := thread.CurrentFrame().OperandStack().PopInt()
    if val == 0 {
        // todo
    }
}

type ifne struct {BranchInstruction}
func (self *ifne) execute(thread *rtda.Thread) {
    val := thread.CurrentFrame().OperandStack().PopInt()
    if val != 0 {
        // todo
    }
}

type iflt struct {BranchInstruction}
func (self *iflt) execute(thread *rtda.Thread) {
    val := thread.CurrentFrame().OperandStack().PopInt()
    if val < 0 {
        // todo
    }
}

type ifle struct {BranchInstruction}
func (self *ifle) execute(thread *rtda.Thread) {
    val := thread.CurrentFrame().OperandStack().PopInt()
    if val <= 0 {
        // todo
    }
}

type ifgt struct {BranchInstruction}
func (self *ifgt) execute(thread *rtda.Thread) {
    val := thread.CurrentFrame().OperandStack().PopInt()
    if val > 0 {
        // todo
    }
}

type ifge struct {BranchInstruction}
func (self *ifge) execute(thread *rtda.Thread) {
    val := thread.CurrentFrame().OperandStack().PopInt()
    if val >= 0 {
        // todo
    }
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/miekg/dns"
	"strings"
)

type SpamCheck struct {
	NS   []NSData
	Spam []SpamData
	Report
}

type SpamData struct {
	Name  string
	IP    string
	Dmarc []dns.RR
	Spf   []dns.RR
	Error string
}

func (c *SpamCheck) Scan(domain string) {
	c.ScanDmarc(domain)
	c.ScanSpf(domain)
}

func (c *SpamCheck) ScanDmarc(domain string) {
	log.Debugf("Spam: scan")
	defer log.Debugf("Spam: scan exit")
	for _, ns := range c.NS {
		for _, nsip := range ns.IP {
			data := SpamData{Name: ns.Name, IP: nsip.String()}
			dmarc, _, err := queryRRset("_dmarc."+domain, dns.TypeTXT, nsip.String(), true)
			if !scanerror(&c.Report, "DMARC scan", ns.Name, nsip.String(), domain, dmarc, err) {
				data.Dmarc = dmarc
				c.Spam = append(c.Spam, data)
			}
		}
	}
}

func (c *SpamCheck) ScanSpf(domain string) {
	log.Debugf("Spam: scanspf")
	defer log.Debugf("Spam: scanspf exit")
	for _, ns := range c.NS {
		for _, nsip := range ns.IP {
			data := SpamData{Name: ns.Name, IP: nsip.String()}
			txt, _, err := queryRRset(domain, dns.TypeTXT, nsip.String(), true)
			if !scanerror(&c.Report, "SPF scan", ns.Name, nsip.String(), domain, txt, err) {
				spf := []dns.RR{}
				for _, rr := range txt {
					if strings.Contains(rr.String(), "v=spf") {
						spf = append(spf, rr)
					}
				}
				data.Spf = spf
				c.Spam = append(c.Spam, data)
			}
		}
	}
}

func (c *SpamCheck) Values() []ReportResult {
	var results []ReportResult
	var rrset []dns.RR
	for _, ns := range c.Spam {
		if ns.Dmarc != nil {
			rrset = ns.Dmarc
			break
		}
	}
	if len(rrset) > 0 {
		results = append(results, ReportResult{Result: "OK  : DMARC records found.",
			Status: true, Name: "DMARC"})
		records := []string{}
		for _, rr := range rrset {
			records = append(records, rr.String())
			if strings.Contains(rr.String(), "p=none") {
				results = append(results, ReportResult{Result: "WARN: DMARC with monitoring policy found.",
					Status: false, Name: "DMARCPolicy"})
			}
			if strings.Contains(rr.String(), "p=quarantine") {
				results = append(results, ReportResult{Result: "WARN: DMARC with quarantine policy found.",
					Status: false, Name: "DMARCPolicy"})
			}
			if strings.Contains(rr.String(), "p=reject") {
				results = append(results, ReportResult{Result: "OK  : DMARC with reject policy.",
					Status: true, Name: "DMARCPolicy"})
			}
		}
		results = append(results, ReportResult{Status: true, Records: records})
	} else {
		results = append(results, ReportResult{Result: "WARN: No DMARC records found. Along with DKIM and SPF, DMARC helps prevent spam from your domain.",
			Status: false, Name: "DMARC"})
	}

	for _, ns := range c.Spam {
		if ns.Spf != nil {
			rrset = ns.Spf
			break
		}
	}

	if len(rrset) > 0 {
		records := []string{}
		for _, rr := range rrset {
			records = append(records, rr.String())
		}
		results = append(results, ReportResult{Result: "OK  : SPF records found.",
			Status: true, Records: records, Name: "SPF"})
	} else {
		results = append(results, ReportResult{Result: "WARN: No SPF records found. Along with DKIM and DMARC, SPF helps prevent spam from your domain.",
			Status: false, Name: "SPF"})
	}

	for _, rr := range rrset {
		if strings.Contains(rr.String(), "-all") {
			results = append(results, ReportResult{Result: "OK  : SPF records set up restrictively.",
				Status: true, Name: "SPF"})
		}
		if strings.Contains(rr.String(), "~all") {
            results = append(results, ReportResult{Result: "WARN: SPF record set to softfail.",
				Status: true, Name: "SPF"})
		}

		break
	}

	// TODO
	// dmarc: p=none recommendation?
	// spf: further recommendations ?

	return results
}

func (c *SpamCheck) CreateReport(domain string) Report {
	c.Scan(domain)
	c.Report.Type = "Spam"
	c.Report.Result = append(c.Report.Result, c.Values()...)
	return c.Report
}
<commit_msg>warn if ptr mechanism is used in SPF<commit_after>package main

import (
	"github.com/miekg/dns"
	"strings"
)

type SpamCheck struct {
	NS   []NSData
	Spam []SpamData
	Report
}

type SpamData struct {
	Name  string
	IP    string
	Dmarc []dns.RR
	Spf   []dns.RR
	Error string
}

func (c *SpamCheck) Scan(domain string) {
	c.ScanDmarc(domain)
	c.ScanSpf(domain)
}

func (c *SpamCheck) ScanDmarc(domain string) {
	log.Debugf("Spam: scan")
	defer log.Debugf("Spam: scan exit")
	for _, ns := range c.NS {
		for _, nsip := range ns.IP {
			data := SpamData{Name: ns.Name, IP: nsip.String()}
			dmarc, _, err := queryRRset("_dmarc."+domain, dns.TypeTXT, nsip.String(), true)
			if !scanerror(&c.Report, "DMARC scan", ns.Name, nsip.String(), domain, dmarc, err) {
				data.Dmarc = dmarc
				c.Spam = append(c.Spam, data)
			}
		}
	}
}

func (c *SpamCheck) ScanSpf(domain string) {
	log.Debugf("Spam: scanspf")
	defer log.Debugf("Spam: scanspf exit")
	for _, ns := range c.NS {
		for _, nsip := range ns.IP {
			data := SpamData{Name: ns.Name, IP: nsip.String()}
			txt, _, err := queryRRset(domain, dns.TypeTXT, nsip.String(), true)
			if !scanerror(&c.Report, "SPF scan", ns.Name, nsip.String(), domain, txt, err) {
				spf := []dns.RR{}
				for _, rr := range txt {
					if strings.Contains(rr.String(), "v=spf") {
						spf = append(spf, rr)
					}
				}
				data.Spf = spf
				c.Spam = append(c.Spam, data)
			}
		}
	}
}

func (c *SpamCheck) Values() []ReportResult {
	var results []ReportResult
	var rrset []dns.RR
	for _, ns := range c.Spam {
		if ns.Dmarc != nil {
			rrset = ns.Dmarc
			break
		}
	}
	if len(rrset) > 0 {
		results = append(results, ReportResult{Result: "OK  : DMARC records found.",
			Status: true, Name: "DMARC"})
		records := []string{}
		for _, rr := range rrset {
			records = append(records, rr.String())
			if strings.Contains(rr.String(), "p=none") {
				results = append(results, ReportResult{Result: "WARN: DMARC with monitoring policy found.",
					Status: false, Name: "DMARCPolicy"})
			}
			if strings.Contains(rr.String(), "p=quarantine") {
				results = append(results, ReportResult{Result: "WARN: DMARC with quarantine policy found.",
					Status: false, Name: "DMARCPolicy"})
			}
			if strings.Contains(rr.String(), "p=reject") {
				results = append(results, ReportResult{Result: "OK  : DMARC with reject policy.",
					Status: true, Name: "DMARCPolicy"})
			}
		}
		results = append(results, ReportResult{Status: true, Records: records})
	} else {
		results = append(results, ReportResult{Result: "WARN: No DMARC records found. Along with DKIM and SPF, DMARC helps prevent spam from your domain.",
			Status: false, Name: "DMARC"})
	}

	for _, ns := range c.Spam {
		if ns.Spf != nil {
			rrset = ns.Spf
			break
		}
	}

	if len(rrset) > 0 {
		records := []string{}
		for _, rr := range rrset {
			records = append(records, rr.String())
		}
		results = append(results, ReportResult{Result: "OK  : SPF records found.",
			Status: true, Records: records, Name: "SPF"})
	} else {
		results = append(results, ReportResult{Result: "WARN: No SPF records found. Along with DKIM and DMARC, SPF helps prevent spam from your domain.",
			Status: false, Name: "SPF"})
	}

	for _, rr := range rrset {
		if strings.Contains(rr.String(), "-all") {
			results = append(results, ReportResult{Result: "OK  : SPF records set up restrictively.",
				Status: true, Name: "SPF"})
		}
		if strings.Contains(rr.String(), "~all") {
            results = append(results, ReportResult{Result: "WARN: SPF record set to softfail.",
				Status: true, Name: "SPF"})
		}
        if strings.Contains(rr.String(), " ptr ") || strings.Contains(rr.String(), " ptr:") {
            results = append(results, ReportResult{Result: "WARN: SPF record uses ptr mechanism (see RFC7208 5.5).",
				Status: true, Name: "SPF"})
		}

		break
	}

	// TODO
	// dmarc: p=none recommendation?
	// spf: further recommendations ?

	return results
}

func (c *SpamCheck) CreateReport(domain string) Report {
	c.Scan(domain)
	c.Report.Type = "Spam"
	c.Report.Result = append(c.Report.Result, c.Values()...)
	return c.Report
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"
    "time"
)

var processes = make(map[string]*lyprocess)
var nRunning = 0;

func main() {
	args := os.Args[1:]

	if len(args) == 0 {
		// Start REPL
		fmt.Println("Welcome to Ly!")

		reader := bufio.NewReader(os.Stdin)

		for {
            fmt.Print("ly > ")
			line, _ := reader.ReadString('\n')
			words := strings.Fields(line)

            if len(words) == 0 {
                continue
            }

			switch words[0] {
    		    case "new":
    		        if len(words) < 3 {
    		            fmt.Println("Not enough arguments")
    		            break
    		        }
    		        startProcess(words[1], words[2], words[2:])
	            case "kill":
	                if len(words) < 2 {
                        fmt.Println("Not enough arguments")
                        break
                    }
                    killProcess(words[1])
                case "out":
	                if len(words) < 2 {
                        fmt.Println("Not enough arguments")
                        break
                    }
                    printOut(words[1])
                case "list":
                    list()
                case "exit":
                    exit()
		        default:
                    printUsage()
			}

            fmt.Println()
		}
	}
}

func startProcess(name string, cmd string, args []string) {
    if processRunning(name) {
        fmt.Println("Process", name, "is already running.")
        return
    }

    fmt.Println("Starting", name, ":", cmd)
    processes[name] = newLyprocess(cmd)

    go func() {
        processes[name].Running = true
        nRunning++
        processes[name].Cmd.Run()
        processes[name].Running = false
        nRunning--
        // Todo: add checking for errors
    }()
}

func killProcess(name string) {
    if !processExists(name) {
        fmt.Println("Process", name, "does not exist.")
    } else {
        processes[name].Cmd.Process.Kill()
    }
}

func printOut(name string) {
    if !processExists(name) {
        fmt.Println("Process", name, "does not exist.")
    } else {
        // for _, line := range(processes[name].Stdout) {
        //     fmt.Println(line)
        // }
        processes[name].Stdout.Do(func(line interface{}) {
            fmt.Println(line)
        })
    }
}

func list() {
    if len(processes) == 0 {
        fmt.Println("No processes")
    } else {
        fmt.Println(len(processes), "processes:")

        for k, v := range(processes) {
            fmt.Printf("  %v(%v)", k, v.Cmd.Process.Pid)
            if !processRunning(k) {
                fmt.Println(" -- Exited")
            } else {
                fmt.Println()
            }
        }
    }
}

func exit() {
    // Try to kill all processes for 3 seconds, then exit forcefully.
    go func() {
        for k, _ := range(processes) {
            killProcess(k)
        }
    }()

    // Try to exit for 3000 ms if no processes are running
    for t := 0; t < 3000; {
        if nRunning == 0 {
            fmt.Println("Goodbye!")
            os.Exit(0)
        }

        t += 300;
        time.Sleep(300 * time.Millisecond)
    }

    fmt.Println("Ly couldn't close all processes in time. These processes may still be running:")
    for k, v := range(processes) {
        if v.Running {
            fmt.Println("  " + k)
        }
    }
    fmt.Println("Goodbye!")
    os.Exit(1)
}

func printUsage() {
    helpStr := `Usage: <operation> [arguments]
Operations:
    new <name> <command>    Spawns a new process called <name> by running <command>.
                            All processes are started in their own shell.
    kill <name>             Kills a running process by <name>
    out <name>              Outputs the most recent standard output/error for a process.
    list                    Lists running processes with their PID and status.
    exit                    Nicely quits all processes and exists.
    help                    Prints this help message.`

    fmt.Println(helpStr)
}
<commit_msg>Some cleanup<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"
    "time"
)

var processes = make(map[string]*lyprocess)
var nRunning = 0;

func main() {
	args := os.Args[1:]

	if len(args) == 0 {
		// Start REPL
		fmt.Println("Welcome to Ly!")

		reader := bufio.NewReader(os.Stdin)

		for {
            fmt.Print("ly > ")
			line, _ := reader.ReadString('\n')
			words := strings.Fields(line)

            if len(words) == 0 {
                continue
            }

			switch words[0] {
    		    case "new":
    		        if len(words) < 3 {
    		            fmt.Println("Not enough arguments")
    		            break
    		        }
    		        startProcess(words[1], words[2], words[2:])
	            case "kill":
	                if len(words) < 2 {
                        fmt.Println("Not enough arguments")
                        break
                    }
                    killProcess(words[1])
                case "out":
	                if len(words) < 2 {
                        fmt.Println("Not enough arguments")
                        break
                    }
                    printOut(words[1])
                case "list":
                    list()
                case "exit":
                    exit()
		        default:
                    printUsage()
			}

            fmt.Println()
		}
	}
}

func startProcess(name string, cmd string, args []string) {
    if processRunning(name) {
        fmt.Println("Process", name, "is already running.")
        return
    }

    fmt.Println("Starting", name, ":", cmd)
    processes[name] = newLyprocess(cmd)

    go func() {
        processes[name].Running = true
        nRunning++
        processes[name].Cmd.Run()
        processes[name].Running = false
        nRunning--
        // Todo: add checking for errors
    }()
}

func killProcess(name string) {
    if !processExists(name) {
        fmt.Println("Process", name, "does not exist.")
    } else {
        processes[name].Cmd.Process.Kill()
    }
}

func printOut(name string) {
    if !processExists(name) {
        fmt.Println("Process", name, "does not exist.")
    } else {
        processes[name].Stdout.Do(func(line interface{}) {
            fmt.Println(line)
        })
    }
}

func list() {
    if len(processes) == 0 {
        fmt.Println("No processes")
    } else {
        fmt.Println(len(processes), "processes:")

        for k, v := range(processes) {
            fmt.Printf("  %v(%v)", k, v.Cmd.Process.Pid)
            if !processRunning(k) {
                fmt.Println(" -- Exited")
            } else {
                fmt.Println()
            }
        }
    }
}

func exit() {
    // Try to kill all processes for 3 seconds, then exit forcefully.
    go func() {
        for k, _ := range(processes) {
            killProcess(k)
        }
    }()

    // Try to exit for 3000 ms if no processes are running
    for t := 0; t < 3000; {
        if nRunning == 0 {
            fmt.Println("Goodbye!")
            os.Exit(0)
        }

        t += 300;
        time.Sleep(300 * time.Millisecond)
    }

    fmt.Println("Ly couldn't close all processes in time. These processes may still be running:")
    for k, v := range(processes) {
        if v.Running {
            fmt.Println("  " + k)
        }
    }
    fmt.Println("Goodbye!")
    os.Exit(1)
}

func printUsage() {
    helpStr := `Usage: <operation> [arguments]
Operations:
    new <name> <command>    Spawns a new process called <name> by running <command>.
                            All processes are started in their own shell.
    kill <name>             Kills a running process by <name>
    out <name>              Outputs the most recent standard output/error for a process.
    list                    Lists running processes with their PID and status.
    exit                    Nicely quits all processes and exists.
    help                    Prints this help message.`

    fmt.Println(helpStr)
}
<|endoftext|>


<commit_before>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	str "strings"
	"time"
)

// Task represents a single static shell command, or go function, to be
// executed, and are scheduled and managed by a corresponding Process
type Task struct {
	Name          string
	Command       string
	ExecMode      ExecMode // TODO: Probably implement via different struct types (local/slurm/k8s, etc etc)
	CustomExecute func(*Task)
	InIPs         map[string]*FileIP
	OutIPs        map[string]*FileIP
	Params        map[string]string
	Done          chan int
	Image         string // TODO: Later probably only include in k8s-task
	DataFolder    string // TODO: Later probably only include in k8s-task
	cores         int
	workflow      *Workflow
	process       *Process
}

// NewTask instantiates and initializes a new Task
func NewTask(workflow *Workflow, process *Process, name string, cmdPat string, inIPs map[string]*FileIP, outPathFuncs map[string]func(*Task) string, outPortsDoStream map[string]bool, params map[string]string, prepend string, execMode ExecMode, customExecute func(*Task), cores int) *Task {
	t := &Task{
		Name:          name,
		InIPs:         inIPs,
		OutIPs:        make(map[string]*FileIP),
		Params:        params,
		Command:       "",
		ExecMode:      execMode,
		CustomExecute: customExecute,
		Done:          make(chan int),
		cores:         cores,
		workflow:      workflow,
		process:       process,
	}

	// Create out IPs
	Debug.Printf("Task:%s: Creating outIPs now ... [%s]", name, cmdPat)
	outIPs := make(map[string]*FileIP)
	for oname, ofun := range outPathFuncs {
		opath := ofun(t)
		otgt := NewFileIP(opath)
		if outPortsDoStream[oname] {
			otgt.doStream = true
		}
		Debug.Printf("Task:%s: Creating outIP with path %s ...\n", name, otgt.Path())
		outIPs[oname] = otgt
	}
	t.OutIPs = outIPs
	t.Command = formatCommand(cmdPat, inIPs, outIPs, params, prepend)
	Debug.Printf("Task:%s: Created formatted command: %s [%s]", name, t.Command, cmdPat)
	return t
}

// InIP returns an IP for the in-port with name portName
func (t *Task) InIP(portName string) *FileIP {
	if t.InIPs[portName] == nil {
		Failf("No such in-portname (%s) in task (%s)\n", portName, t.Name)
	}
	return t.InIPs[portName]
}

// InPath returns the path name of an input file for the task
func (t *Task) InPath(portName string) string {
	return t.InIP(portName).Path()
}

// OutIP returns an IP for the in-port with name portName
func (t *Task) OutIP(portName string) *FileIP {
	if t.OutIPs[portName] == nil {
		Failf("No such out-portname (%s) in task (%s)\n", portName, t.Name)
	}
	return t.OutIPs[portName]
}

// OutPath returns the path name of an input file for the task
func (t *Task) OutPath(portName string) string {
	return t.OutIP(portName).Path()
}

// Param returns the value of a param, for the task
func (t *Task) Param(portName string) string {
	if param, ok := t.Params[portName]; ok {
		return param
	}
	Failf("No such param port '%s' for task '%s'\n", portName, t.Name)
	return "invalid"
}

// Execute executes the task (the shell command or go function configured for
// this task)
func (t *Task) Execute() {
	defer close(t.Done)

	if !t.anyOutputExists() {
		Debug.Printf("Task:%-12s Executing task. [%s]\n", t.Name, t.Command)

		// Create directories for out-IPs
		for _, oip := range t.OutIPs {
			oipDir := filepath.Dir(oip.Path())
			err := os.MkdirAll(oipDir, 0777)
			CheckWithMsg(err, "Could not create directory: "+oipDir)
		}

		t.workflow.IncConcurrentTasks(t.cores) // Will block if max concurrent tasks is reached
		startTime := time.Now()
		if t.CustomExecute != nil {
			Audit.Printf("Task:%-12s Executing custom execution function.\n", t.Name)
			t.CustomExecute(t)
		} else {
			switch t.ExecMode {
			case ExecModeLocal:
				t.executeCommand(t.Command)
			case ExecModeSLURM:
				Failf("Task:%-12s SLURM Execution mode not implemented!", t.Name)
			}
		}
		finishTime := time.Now()
		execTime := finishTime.Sub(startTime)
		t.workflow.DecConcurrentTasks(t.cores)

		// Append audit info for the task to all its output IPs

		auditInfo := NewAuditInfo()
		auditInfo.Command = t.Command
		auditInfo.ProcessName = t.process.Name()
		auditInfo.Params = t.Params
		auditInfo.StartTime = startTime
		auditInfo.FinishTime = finishTime
		auditInfo.ExecTimeMS = execTime / time.Millisecond
		// Set the audit infos from incoming IPs into the "Upstream" map
		for _, iip := range t.InIPs {
			iipPath := iip.Path()
			iipAuditInfo := iip.AuditInfo()
			auditInfo.Upstream[iipPath] = iipAuditInfo
		}
		// Add the current audit info to output ips and write them to file
		for _, oip := range t.OutIPs {
			oip.SetAuditInfo(auditInfo)
			for _, iip := range t.InIPs {
				oip.AddKeys(iip.Keys())
			}
			oip.WriteAuditLogToFile()
		}

		Debug.Printf("Task:%-12s Atomizing IPs. [%s]\n", t.Name, t.Command)
		t.atomizeIPs()

	}
	Debug.Printf("Task:%s: Starting to send Done in t.Execute() ...) [%s]\n", t.Name, t.Command)
	t.Done <- 1
	Debug.Printf("Task:%s: Done sending Done, in t.Execute() [%s]\n", t.Name, t.Command)
}

// Check if any output file IP, or temporary file IPs, exist
func (t *Task) anyOutputExists() (anyFileExists bool) {
	anyFileExists = false
	for _, tgt := range t.OutIPs {
		opath := tgt.Path()
		otmpPath := tgt.TempPath()
		if !tgt.doStream {
			if _, err := os.Stat(opath); err == nil {
				Info.Printf("Task:%-12s Output file already exists, so skipping: %s\n", t.Name, opath)
				anyFileExists = true
			}
			if _, err := os.Stat(otmpPath); err == nil {
				Failf("Task:%-12s Temp file already exists, so exiting: %s (Note: If resuming from a failed run, clean up .tmp files first. Also, make sure that two processes don't produce the same output files!).\n", t.Name, otmpPath)
				anyFileExists = true
			}
		}
	}
	return
}

func (t *Task) executeCommand(cmd string) {
	Audit.Printf("Task:%-12s Executing command: %s\n", t.Name, cmd)
	out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
	if err != nil {
		Failf("Command failed!\nCommand:\n%s\n\nOutput:\n%s\n\n", cmd, string(out))
	}
}

// Rename temporary output files to their proper file names
func (t *Task) atomizeIPs() {
	for _, tgt := range t.OutIPs {
		if !tgt.doStream {
			Debug.Printf("Atomizing file: %s -> %s", tgt.TempPath(), tgt.Path())
			tgt.Atomize()
			Debug.Printf("Done atomizing file: %s -> %s", tgt.TempPath(), tgt.Path())
		} else {
			Debug.Printf("IP is streaming, so not atomizing: %s", tgt.Path())
		}
	}
}

// Clean up any remaining FIFOs
// TODO: this is actually not really used anymore ...
func (t *Task) cleanUpFifos() {
	for _, tgt := range t.OutIPs {
		if tgt.doStream {
			Debug.Printf("Task:%s: Cleaning up FIFO for output IP: %s [%s]\n", t.Name, tgt.FifoPath(), t.Command)
			tgt.RemoveFifo()
		} else {
			Debug.Printf("Task:%s: output IP is not FIFO, so not removing any FIFO: %s [%s]\n", t.Name, tgt.Path(), t.Command)
		}
	}
}

var (
	trueVal  = true
	falseVal = false
)

func formatCommand(cmd string, inIPs map[string]*FileIP, outIPs map[string]*FileIP, params map[string]string, prepend string) string {
	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		reduceInputs := false

		placeHolderStr := m[0]
		typ := m[1]
		name := m[2]
		sep := " " // Default
		if len(m) > 3 {
			sep = m[5]
			reduceInputs = true
		}
		Debug.Printf("Found the following parts in the command: (type: '%s', name: '%s', sep: '%s', reduceInputs: %v). Command: %s\n", typ, name, sep, reduceInputs, cmd)
		var filePath string
		if typ == "o" || typ == "os" {
			// Out-ports
			if outIPs[name] == nil {
				msg := fmt.Sprint("Missing outpath for outport '", name, "' for command '", cmd, "'")
				CheckWithMsg(errors.New(msg), msg)
			} else {
				if typ == "o" {
					filePath = outIPs[name].TempPath() // Means important to Atomize afterwards!
				} else if typ == "os" {
					filePath = outIPs[name].FifoPath()
				}
			}
		} else if typ == "i" {
			// In-ports
			if inIPs[name] == nil {
				msg := fmt.Sprint("Missing in-IP for inport '", name, "' for command '", cmd, "'")
				CheckWithMsg(errors.New(msg), msg)
			} else if inIPs[name].Path() == "" && reduceInputs {
				ips := []*FileIP{}
				for ip := range inIPs[name].SubStream.Chan {
					Debug.Println("Got ip: ", ip)
					ips = append(ips, ip)
				}
				Debug.Println("Got ips: ", ips)
				paths := []string{}
				Debug.Println("Got paths: ", paths)
				for _, ip := range ips {
					paths = append(paths, ip.Path())
				}
				Debug.Println("Got paths: ", paths)
				filePath = str.Join(paths, sep)
				Debug.Println("Got filePath: ", filePath)
			} else if inIPs[name].Path() == "" {
				msg := fmt.Sprint("Missing inpath for inport '", name, "', and no substream, for command '", cmd, "'")
				CheckWithMsg(errors.New(msg), msg)
			} else {
				if inIPs[name].doStream {
					filePath = inIPs[name].FifoPath()
				} else {
					filePath = inIPs[name].Path()
				}
			}
			Debug.Printf("filePath determined to: %s, for command '%s'\n", filePath, cmd)
		} else if typ == "p" {
			if params[name] == "" {
				msg := fmt.Sprint("Missing param value param '", name, "' for command '", cmd, "'")
				CheckWithMsg(errors.New(msg), msg)
			} else {
				filePath = params[name]
			}
		}
		if filePath == "" {
			msg := fmt.Sprint("Replace failed for port ", name, " for command '", cmd, "'")
			CheckWithMsg(errors.New(msg), msg)
		}
		cmd = str.Replace(cmd, placeHolderStr, filePath, -1)
	}
	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}
	return cmd
}
<commit_msg>Greatly clean up task.go<commit_after>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

// Task represents a single static shell command, or go function, to be
// executed, and are scheduled and managed by a corresponding Process
type Task struct {
	Name          string
	Command       string
	ExecMode      ExecMode // TODO: Probably implement via different struct types (local/slurm/k8s, etc etc)
	CustomExecute func(*Task)
	InIPs         map[string]*FileIP
	OutIPs        map[string]*FileIP
	Params        map[string]string
	Done          chan int
	cores         int
	workflow      *Workflow
	process       *Process
}

// ------------------------------------------------------------------------
// Factory method(s)
// ------------------------------------------------------------------------

// NewTask instantiates and initializes a new Task
func NewTask(workflow *Workflow, process *Process, name string, cmdPat string, inIPs map[string]*FileIP, outPathFuncs map[string]func(*Task) string, outPortsDoStream map[string]bool, params map[string]string, prepend string, execMode ExecMode, customExecute func(*Task), cores int) *Task {
	t := &Task{
		Name:          name,
		InIPs:         inIPs,
		OutIPs:        make(map[string]*FileIP),
		Params:        params,
		Command:       "",
		ExecMode:      execMode,
		CustomExecute: customExecute,
		Done:          make(chan int),
		cores:         cores,
		workflow:      workflow,
		process:       process,
	}

	// Create Out-IPs
	for oname, outPathFunc := range outPathFuncs {
		oip := NewFileIP(outPathFunc(t))
		if outPortsDoStream[oname] {
			oip.doStream = true
		}
		t.OutIPs[oname] = oip
	}
	t.Command = formatCommand(cmdPat, inIPs, t.OutIPs, params, prepend)
	return t
}

// formatCommand is a helper function for NewTask, that formats a shell command
// based on concrete file paths and parameter values
func formatCommand(cmd string, inIPs map[string]*FileIP, outIPs map[string]*FileIP, params map[string]string, prepend string) string {
	r := getShellCommandPlaceHolderRegex()
	placeHolderMatches := r.FindAllStringSubmatch(cmd, -1)
	for _, placeHolderMatch := range placeHolderMatches {
		var filePath string

		placeHolderStr := placeHolderMatch[0]
		portType := placeHolderMatch[1]
		portName := placeHolderMatch[2]
		sep := " " // Default

		switch portType {
		case "o":
			if outIPs[portName] == nil {
				Fail("Missing outpath for outport '", portName, "' for command '", cmd, "'")
			}
			filePath = outIPs[portName].TempPath()
		case "os":
			if outIPs[portName] == nil {
				Fail("Missing outpath for outport '", portName, "' for command '", cmd, "'")
			}
			filePath = outIPs[portName].FifoPath()
		case "i":
			if inIPs[portName] == nil {
				Fail("Missing in-IP for inport '", portName, "' for command '", cmd, "'")
			}
			// Identify if the place holder represents a reduce-type in-port
			reduceInputs := false
			if len(placeHolderMatch) > 3 {
				sep = placeHolderMatch[5]
				reduceInputs = true
			}
			if reduceInputs && inIPs[portName].Path() == "" {
				// Merge multiple input paths from a substream on the IP, into one string
				ips := []*FileIP{}
				for ip := range inIPs[portName].SubStream.Chan {
					ips = append(ips, ip)
				}
				paths := []string{}
				for _, ip := range ips {
					paths = append(paths, ip.Path())
				}
				filePath = strings.Join(paths, sep)
			} else {
				if inIPs[portName].Path() == "" {
					Fail("Missing inpath for inport '", portName, "', and no substream, for command '", cmd, "'")
				}
				if inIPs[portName].doStream {
					filePath = inIPs[portName].FifoPath()
				} else {
					filePath = inIPs[portName].Path()
				}
			}
		case "p":
			if params[portName] == "" {
				msg := fmt.Sprint("Missing param value param '", portName, "' for command '", cmd, "'")
				CheckWithMsg(errors.New(msg), msg)
			} else {
				filePath = params[portName]
			}
		default:
			Fail("Replace failed for port ", portName, " for command '", cmd, "'")
		}
		cmd = strings.Replace(cmd, placeHolderStr, filePath, -1)
	}

	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}

	return cmd
}

// ------------------------------------------------------------------------
// Main API methods: Accessor methods
// ------------------------------------------------------------------------

// InIP returns an IP for the in-port with name portName
func (t *Task) InIP(portName string) *FileIP {
	if t.InIPs[portName] == nil {
		Failf("No such in-portname (%s) in task (%s)\n", portName, t.Name)
	}
	return t.InIPs[portName]
}

// InPath returns the path name of an input file for the task
func (t *Task) InPath(portName string) string {
	return t.InIP(portName).Path()
}

// OutIP returns an IP for the in-port with name portName
func (t *Task) OutIP(portName string) *FileIP {
	if t.OutIPs[portName] == nil {
		Failf("No such out-portname (%s) in task (%s)\n", portName, t.Name)
	}
	return t.OutIPs[portName]
}

// OutPath returns the path name of an input file for the task
func (t *Task) OutPath(portName string) string {
	return t.OutIP(portName).Path()
}

// Param returns the value of a param, for the task
func (t *Task) Param(portName string) string {
	if param, ok := t.Params[portName]; ok {
		return param
	}
	Failf("No such param port '%s' for task '%s'\n", portName, t.Name)
	return "invalid"
}

// ------------------------------------------------------------------------
// Execute the task
// ------------------------------------------------------------------------

// Execute executes the task (the shell command or go function in CustomExecute)
func (t *Task) Execute() {
	defer close(t.Done)

	// Do some sanity checks
	if t.anyTempfileExists() {
		Failf("Task:%s-12s Existing temp files found so existing. Clean up .tmp files before restarting the workflow!", t.Name)
	}

	if t.anyOutputsExist() {
		Audit.Printf("Task:%s-12s Outputs exist, so not executing", t.Name)
		t.Done <- 1
		return
	}

	// Execute task
	t.workflow.IncConcurrentTasks(t.cores) // Will block if max concurrent tasks is reached
	t.createDirs()
	startTime := time.Now()
	if t.CustomExecute != nil {
		t.CustomExecute(t)
	} else {
		switch t.ExecMode {
		case ExecModeLocal:
			t.executeCommand(t.Command)
		case ExecModeSLURM:
			Failf("Task:%-12s SLURM Execution mode not implemented!", t.Name)
		}
	}
	finishTime := time.Now()
	t.writeAuditLogs(startTime, finishTime)
	t.atomizeIPs()
	t.workflow.DecConcurrentTasks(t.cores)

	t.Done <- 1
}

// ------------------------------------------------------------------------
// Helper methods for the Execute method
// ------------------------------------------------------------------------

// anyTempFileExists checks if any temporary workflow files exist and if so, returns true
func (t *Task) anyTempfileExists() (anyTempfileExists bool) {
	anyTempfileExists = false
	for _, oip := range t.OutIPs {
		if !oip.doStream {
			otmpPath := oip.TempPath()
			if _, err := os.Stat(otmpPath); err == nil {
				Warning.Printf("Task:%-12s Temp file already exists: %s (Note: If resuming from a failed run, clean up .tmp files first. Also, make sure that two processes don't produce the same output files!).\n", t.Name, otmpPath)
				anyTempfileExists = true
			}
		}
	}
	return
}

// anyOutputsExist if any output file IP, or temporary file IPs, exist
func (t *Task) anyOutputsExist() (anyFileExists bool) {
	anyFileExists = false
	for _, oip := range t.OutIPs {
		if !oip.doStream {
			opath := oip.Path()
			if _, err := os.Stat(opath); err == nil {
				Info.Printf("Task:%-12s Output file already exists, so skipping: %s\n", t.Name, opath)
				anyFileExists = true
			}
		}
	}
	return
}

// createDirs creates directories for out-IPs of the task
func (t *Task) createDirs() {
	for _, oip := range t.OutIPs {
		oipDir := filepath.Dir(oip.Path())
		err := os.MkdirAll(oipDir, 0777)
		CheckWithMsg(err, "Could not create directory: "+oipDir)
	}

}

// executeCommand executes the shell command cmd via bash
func (t *Task) executeCommand(cmd string) {
	Audit.Printf("Task:%-12s Executing command: %s\n", t.Name, cmd)
	out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
	if err != nil {
		Failf("Command failed!\nCommand:\n%s\n\nOutput:\n%s\n\n", cmd, string(out))
	}
}

func (t *Task) writeAuditLogs(startTime time.Time, finishTime time.Time) {
	// Append audit info for the task to all its output IPs
	auditInfo := NewAuditInfo()
	auditInfo.Command = t.Command
	auditInfo.ProcessName = t.process.Name()
	auditInfo.Params = t.Params
	auditInfo.StartTime = startTime
	auditInfo.FinishTime = finishTime
	auditInfo.ExecTimeMS = finishTime.Sub(startTime) / time.Millisecond
	// Set the audit infos from incoming IPs into the "Upstream" map
	for _, iip := range t.InIPs {
		iipPath := iip.Path()
		iipAuditInfo := iip.AuditInfo()
		auditInfo.Upstream[iipPath] = iipAuditInfo
	}
	// Add the current audit info to output ips and write them to file
	for _, oip := range t.OutIPs {
		oip.SetAuditInfo(auditInfo)
		for _, iip := range t.InIPs {
			oip.AddKeys(iip.Keys())
		}
		oip.WriteAuditLogToFile()
	}
}

// Rename temporary output files to their proper file names
func (t *Task) atomizeIPs() {
	for _, oip := range t.OutIPs {
		if !oip.doStream {
			oip.Atomize()
		}
	}
}
<|endoftext|>


<commit_before>// Advertise network services via multicast DNS
package mdns

import (
	"errors"
	"log"
	"net"

	"dns"
)

var (
	ipv4mcastaddr = &net.UDPAddr{
		IP:   net.ParseIP("224.0.0.251"),
		Port: 5353,
	}

	ipv6mcastaddr = &net.UDPAddr{
		IP:   net.ParseIP("ff02::fb"),
		Port: 5353,
	}
	local *zone // the local mdns zone
)

func init() {
	local = &zone{
		entries: make(map[string]entries),
		add:     make(chan *entry, 16),
		queries: make(chan *query, 16),
	}
	go local.mainloop()
	if err := local.listen(ipv4mcastaddr); err != nil {
		log.Fatalf("Failed to listen %s: %s", ipv4mcastaddr, err)
	}
	if err := local.listen(ipv6mcastaddr); err != nil {
		log.Printf("Failed to listen %s: %s", ipv6mcastaddr, err)
	}
}

// Add an A record. fqdn should be fully qualified, including local.
func PublishA(fqdn string, ttl int, ip net.IP) {
	local.add <- &entry{
		RR: &dns.RR_A{
			Hdr: dns.RR_Header{
				Name:   fqdn,
				Rrtype: dns.TypeA,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl),
			},
			A: ip,
		},
	}
}

// Publish a PTR record, fqdn should be fully qualified, including local.
func PublishPTR(fqdn string, ttl int, target string) {
	local.add <- &entry{
		RR: &dns.RR_PTR{
			Hdr: dns.RR_Header{
				Name:   fqdn,
				Rrtype: dns.TypePTR,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl),
			},
			Ptr: target,
		},
	}
}

// Publish a SRV record, fqdn and target should be fully qualified, including local.
func PublishSRV(fqdn string, ttl int, target string, port int) {
	local.add <- &entry{
		RR: &dns.RR_SRV{
			Hdr: dns.RR_Header{
				Name:   fqdn,
				Rrtype: dns.TypeSRV,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl),
			},
			Target: target,
			Port:   uint16(port),
		},
	}
}

// Publish a TXT record, fqdn should be fully qualified, including local.
func PublishTXT(fqdn string, ttl int, txt string) {
	local.add <- &entry{
		RR: &dns.RR_TXT{
			Hdr: dns.RR_Header{
				Name:   fqdn,
				Rrtype: dns.TypeTXT,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl),
			},
			Txt: txt,
		},
	}
}

type entry struct {
	dns.RR
}

func (e *entry) fqdn() string {
	return e.Header().Name
}

type query struct {
	dns.Question
	result chan *entry
}

type entries []*entry

func (e entries) contains(entry *entry) bool {
	for _, ee := range e {
		if equals(entry, ee) {
			return true
		}
	}
	return false
}

type zone struct {
	entries map[string]entries
	add     chan *entry // add entries to zone
	queries chan *query // query exsting entries in zone
}

func (z *zone) mainloop() {
	for {
		select {
		case entry := <-z.add:
			if !z.entries[entry.fqdn()].contains(entry) {
				z.entries[entry.fqdn()] = append(z.entries[entry.fqdn()], entry)
			}
		case q := <-z.queries:
			for _, entry := range z.entries[q.Question.Name] {
				if q.matches(entry) {
					q.result <- entry
				}
			}
			close(q.result)
		}
	}
}

func (z *zone) query(q dns.Question) (entries []*entry) {
	res := make(chan *entry, 16)
	z.queries <- &query{q, res}
	for e := range res {
		entries = append(entries, e)
	}
	return
}

func (q *query) matches(entry *entry) bool {
	return q.Question.Qtype == dns.TypeANY || q.Question.Qtype == entry.RR.Header().Rrtype
}

func equals(this, that *entry) bool {
	if _, ok := this.RR.(*dns.RR_ANY); ok {
		return true // *RR_ANY matches anything
	}
	if _, ok := that.RR.(*dns.RR_ANY); ok {
		return true // *RR_ANY matches all
	}
	return false
}

type connector struct {
	*net.UDPAddr
	*net.UDPConn
	*zone
}

func (z *zone) listen(addr *net.UDPAddr) error {
	conn, err := openSocket(addr)
	if err != nil {
		return err
	}
	c := &connector{
		UDPAddr: addr,
		UDPConn: conn,
		zone:    z,
	}
	go c.mainloop()
	return nil
}

func openSocket(addr *net.UDPAddr) (*net.UDPConn, error) {
	switch addr.IP.To4() {
	case nil:
		return net.ListenMulticastUDP("udp6", nil, &net.UDPAddr{
			IP:   net.IPv6zero,
			Port: addr.Port,
		})
	default:
		return net.ListenMulticastUDP("udp4", nil, &net.UDPAddr{
			IP:   net.IPv4zero,
			Port: addr.Port,
		})
	}
	panic("unreachable")
}

func (c *connector) mainloop() {
	in := make(chan struct {
		*dns.Msg
		*net.UDPAddr
	}, 32)
	go func() {
		for {
			msg, addr, err := c.readMessage()
			if err != nil {
				// log dud packets
				log.Printf("Cound not read from %s: %s", c.UDPConn, err)
			}
			if msg.IsQuestion() {
				in <- struct {
					*dns.Msg
					*net.UDPAddr
				}{msg, addr}
			}
		}
	}()
	for {
		msg := <-in
		msg.MsgHdr.Response = true // convert question to response
		for _, result := range c.query(msg.Question) {
			msg.Answer = append(msg.Answer, result.RR)
		}
		msg.Extra = append(msg.Extra, c.findExtra(msg.Answer...)...)
		if len(msg.Answer) > 0 {
			if err := c.writeMessage(msg.Msg); err != nil {
				log.Fatalf("Cannot send: %s", err)
			}
		}
	}
}

func (c *connector) query(qs []dns.Question) (results []*entry) {
	for _, q := range qs {
		results = append(results, c.zone.query(q)...)
	}
	return
}

// recursively probe for related records
func (c *connector) findExtra(r ...dns.RR) (extra []dns.RR) {
	for _, rr := range r {
		var q dns.Question
		switch rr := rr.(type) {
		case *dns.RR_PTR:
			q = dns.Question{
				Name:   rr.Ptr,
				Qtype:  dns.TypeANY,
				Qclass: dns.ClassINET,
			}
		case *dns.RR_SRV:
			q = dns.Question{
				Name:   rr.Target,
				Qtype:  dns.TypeA,
				Qclass: dns.ClassINET,
			}
		default:
			continue
		}
		res := c.zone.query(q)
		if len(res) > 0 {
			for _, entry := range res {
				extra = append(append(extra, entry.RR), c.findExtra(entry.RR)...)
			}
		}
	}
	return
}

// encode an mdns msg and broadcast it on the wire
func (c *connector) writeMessage(msg *dns.Msg) (err error) {
	if buf, ok := msg.Pack(); ok {
		_, err = c.WriteToUDP(buf, c.UDPAddr)
	}
	return
}

// consume an mdns packet from the wire and decode it
func (c *connector) readMessage() (*dns.Msg, *net.UDPAddr, error) {
	buf := make([]byte, 1500)
	read, addr, err := c.ReadFromUDP(buf)
	if err != nil {
		return nil, nil, err
	}
	if msg := new(dns.Msg); msg.Unpack(buf[:read]) {
		return msg, addr, nil
	}
	return nil, addr, errors.New("Unable to unpack buffer")
}
<commit_msg>Use the correct mcast address<commit_after>// Advertise network services via multicast DNS
package mdns

import (
	"errors"
	"log"
	"net"

	"dns"
)

var (
	ipv4mcastaddr = &net.UDPAddr{
		IP:   net.ParseIP("224.0.0.251"),
		Port: 5353,
	}

	ipv6mcastaddr = &net.UDPAddr{
		IP:   net.ParseIP("ff02::fb"),
		Port: 5353,
	}
	local *zone // the local mdns zone
)

func init() {
	local = &zone{
		entries: make(map[string]entries),
		add:     make(chan *entry, 16),
		queries: make(chan *query, 16),
	}
	go local.mainloop()
	if err := local.listen(ipv4mcastaddr); err != nil {
		log.Fatalf("Failed to listen %s: %s", ipv4mcastaddr, err)
	}
	if err := local.listen(ipv6mcastaddr); err != nil {
		log.Printf("Failed to listen %s: %s", ipv6mcastaddr, err)
	}
}

// Add an A record. fqdn should be fully qualified, including local.
func PublishA(fqdn string, ttl int, ip net.IP) {
	local.add <- &entry{
		RR: &dns.RR_A{
			Hdr: dns.RR_Header{
				Name:   fqdn,
				Rrtype: dns.TypeA,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl),
			},
			A: ip,
		},
	}
}

// Publish a PTR record, fqdn should be fully qualified, including local.
func PublishPTR(fqdn string, ttl int, target string) {
	local.add <- &entry{
		RR: &dns.RR_PTR{
			Hdr: dns.RR_Header{
				Name:   fqdn,
				Rrtype: dns.TypePTR,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl),
			},
			Ptr: target,
		},
	}
}

// Publish a SRV record, fqdn and target should be fully qualified, including local.
func PublishSRV(fqdn string, ttl int, target string, port int) {
	local.add <- &entry{
		RR: &dns.RR_SRV{
			Hdr: dns.RR_Header{
				Name:   fqdn,
				Rrtype: dns.TypeSRV,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl),
			},
			Target: target,
			Port:   uint16(port),
		},
	}
}

// Publish a TXT record, fqdn should be fully qualified, including local.
func PublishTXT(fqdn string, ttl int, txt string) {
	local.add <- &entry{
		RR: &dns.RR_TXT{
			Hdr: dns.RR_Header{
				Name:   fqdn,
				Rrtype: dns.TypeTXT,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl),
			},
			Txt: txt,
		},
	}
}

type entry struct {
	dns.RR
}

func (e *entry) fqdn() string {
	return e.Header().Name
}

type query struct {
	dns.Question
	result chan *entry
}

type entries []*entry

func (e entries) contains(entry *entry) bool {
	for _, ee := range e {
		if equals(entry, ee) {
			return true
		}
	}
	return false
}

type zone struct {
	entries map[string]entries
	add     chan *entry // add entries to zone
	queries chan *query // query exsting entries in zone
}

func (z *zone) mainloop() {
	for {
		select {
		case entry := <-z.add:
			if !z.entries[entry.fqdn()].contains(entry) {
				z.entries[entry.fqdn()] = append(z.entries[entry.fqdn()], entry)
			}
		case q := <-z.queries:
			for _, entry := range z.entries[q.Question.Name] {
				if q.matches(entry) {
					q.result <- entry
				}
			}
			close(q.result)
		}
	}
}

func (z *zone) query(q dns.Question) (entries []*entry) {
	res := make(chan *entry, 16)
	z.queries <- &query{q, res}
	for e := range res {
		entries = append(entries, e)
	}
	return
}

func (q *query) matches(entry *entry) bool {
	return q.Question.Qtype == dns.TypeANY || q.Question.Qtype == entry.RR.Header().Rrtype
}

func equals(this, that *entry) bool {
	if _, ok := this.RR.(*dns.RR_ANY); ok {
		return true // *RR_ANY matches anything
	}
	if _, ok := that.RR.(*dns.RR_ANY); ok {
		return true // *RR_ANY matches all
	}
	return false
}

type connector struct {
	*net.UDPAddr
	*net.UDPConn
	*zone
}

func (z *zone) listen(addr *net.UDPAddr) error {
	conn, err := openSocket(addr)
	if err != nil {
		return err
	}
	c := &connector{
		UDPAddr: addr,
		UDPConn: conn,
		zone:    z,
	}
	go c.mainloop()
	return nil
}

func openSocket(addr *net.UDPAddr) (*net.UDPConn, error) {
	switch addr.IP.To4() {
	case nil:
		return net.ListenMulticastUDP("udp6", nil, ipv6mcastaddr)
	default:
		return net.ListenMulticastUDP("udp4", nil, ipv4mcastaddr)
	}
	panic("unreachable")
}

func (c *connector) mainloop() {
	in := make(chan struct {
		*dns.Msg
		*net.UDPAddr
	}, 32)
	go func() {
		for {
			msg, addr, err := c.readMessage()
			if err != nil {
				// log dud packets
				log.Printf("Cound not read from %s: %s", c.UDPConn, err)
			}
			if msg.IsQuestion() {
				in <- struct {
					*dns.Msg
					*net.UDPAddr
				}{msg, addr}
			}
		}
	}()
	for {
		msg := <-in
		msg.MsgHdr.Response = true // convert question to response
		for _, result := range c.query(msg.Question) {
			msg.Answer = append(msg.Answer, result.RR)
		}
		msg.Extra = append(msg.Extra, c.findExtra(msg.Answer...)...)
		if len(msg.Answer) > 0 {
			if err := c.writeMessage(msg.Msg); err != nil {
				log.Fatalf("Cannot send: %s", err)
			}
		}
	}
}

func (c *connector) query(qs []dns.Question) (results []*entry) {
	for _, q := range qs {
		results = append(results, c.zone.query(q)...)
	}
	return
}

// recursively probe for related records
func (c *connector) findExtra(r ...dns.RR) (extra []dns.RR) {
	for _, rr := range r {
		var q dns.Question
		switch rr := rr.(type) {
		case *dns.RR_PTR:
			q = dns.Question{
				Name:   rr.Ptr,
				Qtype:  dns.TypeANY,
				Qclass: dns.ClassINET,
			}
		case *dns.RR_SRV:
			q = dns.Question{
				Name:   rr.Target,
				Qtype:  dns.TypeA,
				Qclass: dns.ClassINET,
			}
		default:
			continue
		}
		res := c.zone.query(q)
		if len(res) > 0 {
			for _, entry := range res {
				extra = append(append(extra, entry.RR), c.findExtra(entry.RR)...)
			}
		}
	}
	return
}

// encode an mdns msg and broadcast it on the wire
func (c *connector) writeMessage(msg *dns.Msg) (err error) {
	if buf, ok := msg.Pack(); ok {
		_, err = c.WriteToUDP(buf, c.UDPAddr)
	}
	return
}

// consume an mdns packet from the wire and decode it
func (c *connector) readMessage() (*dns.Msg, *net.UDPAddr, error) {
	buf := make([]byte, 1500)
	read, addr, err := c.ReadFromUDP(buf)
	if err != nil {
		return nil, nil, err
	}
	if msg := new(dns.Msg); msg.Unpack(buf[:read]) {
		return msg, addr, nil
	}
	return nil, addr, errors.New("Unable to unpack buffer")
}
<|endoftext|>


<commit_before>package tent

import (
	"encoding/json"
	"net/url"
	"strings"
)

const RelMetaPost = "https://tent.io/rels/meta-post"

type MetaPostServer struct {
	Version    string `json:"version"`
	Preference int    `json:"preference"`

	URLs MetaPostServerURLs `json:"urls"`
}

type MetaPostServerURLs struct {
	OAuthAuth      string `json:"oauth_auth"`
	OAuthToken     string `json:"oauth_token"`
	PostsFeed      string `json:"posts_feed"`
	Post           string `json:"post"`
	NewPost        string `json:"new_post"`
	PostAttachment string `json:"post_attachment"`
	Attachment     string `json:"attachment"`
	Batch          string `json:"batch"`
	ServerInfo     string `json:"server_info"`
}

func (urls *MetaPostServerURLs) OAuthURL(appID string, state string) string {
	return appendQuery(urls.OAuthAuth, url.Values{"client_id": {appID}, "state": {state}}.Encode())
}

func (urls *MetaPostServerURLs) PostURL(entity, post, version string) string {
	u := strings.Replace(urls.Post, "{entity}", url.QueryEscape(entity), 1)
	u = strings.Replace(u, "{post}", post, 1)
	if version != "" {
		u = appendQuery(u, "version="+version)
	}
	return u
}

func (urls *MetaPostServerURLs) PostAttachmentURL(entity, post, version, name string) string {
	u := strings.Replace(urls.PostAttachment, "{entity}", url.QueryEscape(entity), 1)
	u = strings.Replace(u, "{post}", post, 1)
	u = strings.Replace(u, "{name}", url.QueryEscape(name), 1)
	if version != "" {
		u = appendQuery(u, "version="+version)
	}
	return u
}

func (urls *MetaPostServerURLs) AttachmentURL(entity, digest string) string {
	u := strings.Replace(urls.Attachment, "{entity}", url.QueryEscape(entity), 1)
	return strings.Replace(u, "{digest}", digest, 1)
}

func appendQuery(url, query string) string {
	if strings.Contains(url, "?") {
		return url + "&" + query
	}
	return url + "?" + query
}

type MetaPost struct {
	Entity  string           `json:"entity"`
	Profile MetaProfile      `json:"profile"`
	Servers []MetaPostServer `json:"servers"`
	Post    *Post            `json:"-"`
}

type MetaProfile struct {
	Name     string `json:"name,omitempty"`
	Bio      string `json:"bio,omitempty"`
	Website  string `json:"website,omitempty"`
	Location string `json:"location,omitempty"`

	AvatarDigest string `json:"avatar_digest,omitempty"`
}

func GetMetaPost(url string) (*MetaPost, error) {
	post, err := GetPost(url)
	if err != nil {
		return nil, err
	}
	metaPost, err := ParseMeta(post.Post.Content, post.Post.Attachments)
	metaPost.Post = post.Post
	return metaPost, err
}

func ParseMeta(content []byte, attachments []*PostAttachment) (*MetaPost, error) {
	meta := &MetaPost{}
	err := json.Unmarshal(content, meta)
	if len(attachments) > 0 {
		// TODO: make this more strict
		meta.Profile.AvatarDigest = attachments[0].Digest
	}
	return meta, err
}

func getMetaPost(links []string, reqURL *url.URL) (*MetaPost, error) {
	for i, l := range links {
		// replace percent symbols with a private unicode character so that the url doesn't get decoded
		u, err := url.Parse(strings.Replace(l, "%", "\uFFFE", -1))
		if err != nil {
			return nil, err
		}
		m, err := GetMetaPost(strings.Replace(reqURL.ResolveReference(u).String(), "%EF%BF%BE", "%", -1))
		if err != nil && i < len(links)-1 {
			continue
		}
		return m, err
	}
	panic("not reached")
}
<commit_msg>Add HTTPSOnly check to meta post<commit_after>package tent

import (
	"encoding/json"
	"net/url"
	"strings"
)

const RelMetaPost = "https://tent.io/rels/meta-post"

type MetaPostServer struct {
	Version    string `json:"version"`
	Preference int    `json:"preference"`

	URLs MetaPostServerURLs `json:"urls"`
}

type MetaPostServerURLs struct {
	OAuthAuth      string `json:"oauth_auth"`
	OAuthToken     string `json:"oauth_token"`
	PostsFeed      string `json:"posts_feed"`
	Post           string `json:"post"`
	NewPost        string `json:"new_post"`
	PostAttachment string `json:"post_attachment"`
	Attachment     string `json:"attachment"`
	Batch          string `json:"batch"`
	ServerInfo     string `json:"server_info"`
}

func (urls *MetaPostServerURLs) OAuthURL(appID string, state string) string {
	return appendQuery(urls.OAuthAuth, url.Values{"client_id": {appID}, "state": {state}}.Encode())
}

func (urls *MetaPostServerURLs) PostURL(entity, post, version string) string {
	u := strings.Replace(urls.Post, "{entity}", url.QueryEscape(entity), 1)
	u = strings.Replace(u, "{post}", post, 1)
	if version != "" {
		u = appendQuery(u, "version="+version)
	}
	return u
}

func (urls *MetaPostServerURLs) PostAttachmentURL(entity, post, version, name string) string {
	u := strings.Replace(urls.PostAttachment, "{entity}", url.QueryEscape(entity), 1)
	u = strings.Replace(u, "{post}", post, 1)
	u = strings.Replace(u, "{name}", url.QueryEscape(name), 1)
	if version != "" {
		u = appendQuery(u, "version="+version)
	}
	return u
}

func (urls *MetaPostServerURLs) AttachmentURL(entity, digest string) string {
	u := strings.Replace(urls.Attachment, "{entity}", url.QueryEscape(entity), 1)
	return strings.Replace(u, "{digest}", digest, 1)
}

func appendQuery(url, query string) string {
	if strings.Contains(url, "?") {
		return url + "&" + query
	}
	return url + "?" + query
}

type MetaPost struct {
	Entity  string           `json:"entity"`
	Profile MetaProfile      `json:"profile"`
	Servers []MetaPostServer `json:"servers"`
	Post    *Post            `json:"-"`
}

func (meta *MetaPost) HTTPSOnly() bool {
	v := func(s string) bool { return !strings.HasPrefix(s, "https://") }
	for _, s := range meta.Servers {
		if v(s.URLs.OAuthAuth) || v(s.URLs.OAuthToken) || v(s.URLs.PostsFeed) || v(s.URLs.Post) || v(s.URLs.NewPost) ||
			v(s.URLs.PostAttachment) || v(s.URLs.Attachment) || v(s.URLs.Batch) || v(s.URLs.ServerInfo) {
			return false
		}
	}
	return true
}

type MetaProfile struct {
	Name     string `json:"name,omitempty"`
	Bio      string `json:"bio,omitempty"`
	Website  string `json:"website,omitempty"`
	Location string `json:"location,omitempty"`

	AvatarDigest string `json:"avatar_digest,omitempty"`
}

func GetMetaPost(url string) (*MetaPost, error) {
	post, err := GetPost(url)
	if err != nil {
		return nil, err
	}
	metaPost, err := ParseMeta(post.Post.Content, post.Post.Attachments)
	metaPost.Post = post.Post
	return metaPost, err
}

func ParseMeta(content []byte, attachments []*PostAttachment) (*MetaPost, error) {
	meta := &MetaPost{}
	err := json.Unmarshal(content, meta)
	if len(attachments) > 0 {
		// TODO: make this more strict
		meta.Profile.AvatarDigest = attachments[0].Digest
	}
	return meta, err
}

func getMetaPost(links []string, reqURL *url.URL) (*MetaPost, error) {
	for i, l := range links {
		// replace percent symbols with a private unicode character so that the url doesn't get decoded
		u, err := url.Parse(strings.Replace(l, "%", "\uFFFE", -1))
		if err != nil {
			return nil, err
		}
		m, err := GetMetaPost(strings.Replace(reqURL.ResolveReference(u).String(), "%EF%BF%BE", "%", -1))
		if err != nil && i < len(links)-1 {
			continue
		}
		return m, err
	}
	panic("not reached")
}
<|endoftext|>


<commit_before><commit_msg>user model<commit_after><|endoftext|>