<commit_before>package antlr

import "sort"

// Collectable is an interface that a struct should implement if it is to be
// usable as a key in these collections.
type Collectable[T any] interface {
	Hash() int
	Equals(other Collectable[T]) bool
}

type Comparator[T any] interface {
	Hash1(o T) int
	Equals2(T, T) bool
}

// JStore implements a container that allows the use of a struct to calculate the key
// for a collection of values akin to map. This is not meant to be a full-blown HashMap but just
// serve the needs of the ANTLR Go runtime.
//
// For ease of porting the logic of the runtime from the master target (Java), this collection
// operates in a similar way to Java, in that it can use any struct that supplies a Hash() and Equals()
// function as the key. The values are stored in a standard go map which internally is a form of hashmap
// itself, the key for the go map is the hash supplied by the key object. The collection is able to deal with
// hash conflicts by using a simple slice of values associated with the hash code indexed bucket. That isn't
// particularly efficient, but it is simple, and it works. As this is specifically for the ANTLR runtime, and
// we understand the requirements, then this is fine - this is not a general purpose collection.
type JStore[T any, C Comparator[T]] struct {
	store      map[int][]T
	len        int
	comparator Comparator[T]
}

func NewJStore[T any, C Comparator[T]](comparator Comparator[T]) *JStore[T, C] {

	if comparator == nil {
		panic("comparator cannot be nil")
	}

	s := &JStore[T, C]{
		store:      make(map[int][]T),
		comparator: comparator,
	}
	return s
}

// Put will store given value in the collection. Note that the key for storage is generated from
// the value itself - this is specifically because that is what ANTLR needs - this would not be useful
// as any kind of general collection.
//
// If the key has a hash conflict, then the value will be added to the slice of values associated with the
// hash, unless the value is already in the slice, in which case the existing value is returned. Value equivalence is
// tested by calling the equals() method on the key.
//
// # If the given value is already present in the store, then the existing value is returned as v and exists is set to true
//
// If the given value is not present in the store, then the value is added to the store and returned as v and exists is set to false.
func (s *JStore[T, C]) Put(value T) (v T, exists bool) { //nolint:ireturn

	kh := s.comparator.Hash1(value)

	for _, v1 := range s.store[kh] {
		if s.comparator.Equals2(value, v1) {
			return v1, true
		}
	}
	s.store[kh] = append(s.store[kh], value)
	s.len++
	return value, false
}

// Get will return the value associated with the key - the type of the key is the same type as the value
// which would not generally be useful, but this is a specific thing for ANTLR where the key is
// generated using the object we are going to store.
func (s *JStore[T, C]) Get(key T) (T, bool) { //nolint:ireturn

	kh := s.comparator.Hash1(key)

	for _, v := range s.store[kh] {
		if s.comparator.Equals2(key, v) {
			return v, true
		}
	}
	return key, false
}

// Contains returns true if the given key is present in the store
func (s *JStore[T, C]) Contains(key T) bool { //nolint:ireturn

	_, present := s.Get(key)
	return present
}

func (s *JStore[T, C]) SortedSlice(less func(i, j T) bool) []T {
	vs := make([]T, 0, len(s.store))
	for _, v := range s.store {
		vs = append(vs, v...)
	}
	sort.Slice(vs, func(i, j int) bool {
		return less(vs[i], vs[j])
	})

	return vs
}

func (s *JStore[T, C]) Each(f func(T) bool) {
	for _, e := range s.store {
		for _, v := range e {
			f(v)
		}
	}
}

func (s *JStore[T, C]) Len() int {
	return s.len
}

func (s *JStore[T, C]) Values() []T {
	vs := make([]T, 0, len(s.store))
	for _, e := range s.store {
		for _, v := range e {
			vs = append(vs, v)
		}
	}
	return vs
}

type entry[K, V any] struct {
	key K
	val V
}

type JMap[K, V any, C Comparator[K]] struct {
	store      map[int][]*entry[K, V]
	len        int
	comparator Comparator[K]
}

func NewJMap[K, V any, C Comparator[K]](comparator Comparator[K]) *JMap[K, V, C] {
	return &JMap[K, V, C]{
		store:      make(map[int][]*entry[K, V]),
		comparator: comparator,
	}
}

func (m *JMap[K, V, C]) Put(key K, val V) {
	kh := m.comparator.Hash1(key)
	m.store[kh] = append(m.store[kh], &entry[K, V]{key, val})
	m.len++
}

func (m *JMap[K, V, C]) Values() []V {
	vs := make([]V, 0, len(m.store))
	for _, e := range m.store {
		for _, v := range e {
			vs = append(vs, v.val)
		}
	}
	return vs
}

func (m *JMap[K, V, C]) Get(key K) (V, bool) {

	var none V
	kh := m.comparator.Hash1(key)
	for _, e := range m.store[kh] {
		if m.comparator.Equals2(e.key, key) {
			return e.val, true
		}
	}
	return none, false
}

func (m *JMap[K, V, C]) Len() int {
	return len(m.store)
}

func (m *JMap[K, V, C]) Delete(key K) {
	kh := m.comparator.Hash1(key)
	for i, e := range m.store[kh] {
		if m.comparator.Equals2(e.key, key) {
			m.store[kh] = append(m.store[kh][:i], m.store[kh][i+1:]...)
			m.len--
			return
		}
	}
}

func (m *JMap[K, V, C]) Clear() {
	m.store = make(map[int][]*entry[K, V])
}
<commit_msg>feat: Reduce initial memory allocations for collections<commit_after>package antlr

import "sort"

// Collectable is an interface that a struct should implement if it is to be
// usable as a key in these collections.
type Collectable[T any] interface {
	Hash() int
	Equals(other Collectable[T]) bool
}

type Comparator[T any] interface {
	Hash1(o T) int
	Equals2(T, T) bool
}

// JStore implements a container that allows the use of a struct to calculate the key
// for a collection of values akin to map. This is not meant to be a full-blown HashMap but just
// serve the needs of the ANTLR Go runtime.
//
// For ease of porting the logic of the runtime from the master target (Java), this collection
// operates in a similar way to Java, in that it can use any struct that supplies a Hash() and Equals()
// function as the key. The values are stored in a standard go map which internally is a form of hashmap
// itself, the key for the go map is the hash supplied by the key object. The collection is able to deal with
// hash conflicts by using a simple slice of values associated with the hash code indexed bucket. That isn't
// particularly efficient, but it is simple, and it works. As this is specifically for the ANTLR runtime, and
// we understand the requirements, then this is fine - this is not a general purpose collection.
type JStore[T any, C Comparator[T]] struct {
	store      map[int][]T
	len        int
	comparator Comparator[T]
}

func NewJStore[T any, C Comparator[T]](comparator Comparator[T]) *JStore[T, C] {

	if comparator == nil {
		panic("comparator cannot be nil")
	}

	s := &JStore[T, C]{
		store:      make(map[int][]T, 1),
		comparator: comparator,
	}
	return s
}

// Put will store given value in the collection. Note that the key for storage is generated from
// the value itself - this is specifically because that is what ANTLR needs - this would not be useful
// as any kind of general collection.
//
// If the key has a hash conflict, then the value will be added to the slice of values associated with the
// hash, unless the value is already in the slice, in which case the existing value is returned. Value equivalence is
// tested by calling the equals() method on the key.
//
// # If the given value is already present in the store, then the existing value is returned as v and exists is set to true
//
// If the given value is not present in the store, then the value is added to the store and returned as v and exists is set to false.
func (s *JStore[T, C]) Put(value T) (v T, exists bool) { //nolint:ireturn

	kh := s.comparator.Hash1(value)

	for _, v1 := range s.store[kh] {
		if s.comparator.Equals2(value, v1) {
			return v1, true
		}
	}
	s.store[kh] = append(s.store[kh], value)
	s.len++
	return value, false
}

// Get will return the value associated with the key - the type of the key is the same type as the value
// which would not generally be useful, but this is a specific thing for ANTLR where the key is
// generated using the object we are going to store.
func (s *JStore[T, C]) Get(key T) (T, bool) { //nolint:ireturn

	kh := s.comparator.Hash1(key)

	for _, v := range s.store[kh] {
		if s.comparator.Equals2(key, v) {
			return v, true
		}
	}
	return key, false
}

// Contains returns true if the given key is present in the store
func (s *JStore[T, C]) Contains(key T) bool { //nolint:ireturn

	_, present := s.Get(key)
	return present
}

func (s *JStore[T, C]) SortedSlice(less func(i, j T) bool) []T {
	vs := make([]T, 0, len(s.store))
	for _, v := range s.store {
		vs = append(vs, v...)
	}
	sort.Slice(vs, func(i, j int) bool {
		return less(vs[i], vs[j])
	})

	return vs
}

func (s *JStore[T, C]) Each(f func(T) bool) {
	for _, e := range s.store {
		for _, v := range e {
			f(v)
		}
	}
}

func (s *JStore[T, C]) Len() int {
	return s.len
}

func (s *JStore[T, C]) Values() []T {
	vs := make([]T, 0, len(s.store))
	for _, e := range s.store {
		for _, v := range e {
			vs = append(vs, v)
		}
	}
	return vs
}

type entry[K, V any] struct {
	key K
	val V
}

type JMap[K, V any, C Comparator[K]] struct {
	store      map[int][]*entry[K, V]
	len        int
	comparator Comparator[K]
}

func NewJMap[K, V any, C Comparator[K]](comparator Comparator[K]) *JMap[K, V, C] {
	return &JMap[K, V, C]{
		store:      make(map[int][]*entry[K, V], 1),
		comparator: comparator,
	}
}

func (m *JMap[K, V, C]) Put(key K, val V) {
	kh := m.comparator.Hash1(key)
	m.store[kh] = append(m.store[kh], &entry[K, V]{key, val})
	m.len++
}

func (m *JMap[K, V, C]) Values() []V {
	vs := make([]V, 0, len(m.store))
	for _, e := range m.store {
		for _, v := range e {
			vs = append(vs, v.val)
		}
	}
	return vs
}

func (m *JMap[K, V, C]) Get(key K) (V, bool) {

	var none V
	kh := m.comparator.Hash1(key)
	for _, e := range m.store[kh] {
		if m.comparator.Equals2(e.key, key) {
			return e.val, true
		}
	}
	return none, false
}

func (m *JMap[K, V, C]) Len() int {
	return len(m.store)
}

func (m *JMap[K, V, C]) Delete(key K) {
	kh := m.comparator.Hash1(key)
	for i, e := range m.store[kh] {
		if m.comparator.Equals2(e.key, key) {
			m.store[kh] = append(m.store[kh][:i], m.store[kh][i+1:]...)
			m.len--
			return
		}
	}
}

func (m *JMap[K, V, C]) Clear() {
	m.store = make(map[int][]*entry[K, V])
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/matm/scytale"
)

// App version
const appVersion = "1.0"

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s -o output input password\n", os.Args[0])
		flag.PrintDefaults()
	}
	help := flag.Bool("h", false, "show help message")
	decrypt := flag.Bool("d", false, "decrypt file")
	output := flag.String("o", "", "output file name")
	version := flag.Bool("v", false, "show version")
	flag.Parse()

	if *help {
		flag.Usage()
		os.Exit(2)
	}
	if *version {
		fmt.Println("version:", appVersion)
		os.Exit(0)
	}
	if len(flag.Args()) != 2 {
		flag.Usage()
		os.Exit(2)
	}
	if *output == "" {
		log.Fatal("missing output file name (use -o)")
	}
	name := flag.Arg(0)
	a, err := scytale.NewAES(flag.Arg(1))
	if err != nil {
		log.Fatal("AES init:", err)
	}

	in, err := os.Open(name)
	if err != nil {
		log.Fatal(err)
	}
	defer in.Close()
	out, err := os.Create(*output)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	var action func(io.Reader, io.Writer) error
	action = a.EncryptFile
	if *decrypt {
		action = a.DecryptFile
	}

	if err := action(in, out); err != nil {
		log.Fatal(err)
	}
	fmt.Println("Wrote to", *output)
}
<commit_msg>aesenc: no password as argument, ask it more securely<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/matm/scytale"
)

const pwdMinLen = 4

// App version
const appVersion = "1.0"

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s -o output input\n", os.Args[0])
		flag.PrintDefaults()
	}
	help := flag.Bool("h", false, "show help message")
	decrypt := flag.Bool("d", false, "decrypt file")
	output := flag.String("o", "", "output file name")
	version := flag.Bool("v", false, "show version")
	flag.Parse()

	if *help {
		flag.Usage()
		os.Exit(2)
	}
	if *version {
		fmt.Println("version:", appVersion)
		os.Exit(0)
	}
	if len(flag.Args()) != 1 {
		flag.Usage()
		os.Exit(2)
	}
	if *output == "" {
		log.Fatal("missing output file name (use -o)")
	}
	// Read password twice
	twice := true
	if *decrypt {
		twice = false
	}
	pwd, err := scytale.ReadPassword(pwdMinLen, twice)
	if err != nil {
		log.Fatal(err)
	}
	name := flag.Arg(0)
	a, err := scytale.NewAES(pwd)
	if err != nil {
		log.Fatal("AES init:", err)
	}

	in, err := os.Open(name)
	if err != nil {
		log.Fatal(err)
	}
	defer in.Close()
	out, err := os.Create(*output)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	var action func(io.Reader, io.Writer) error
	action = a.EncryptFile
	if *decrypt {
		action = a.DecryptFile
	}

	if err := action(in, out); err != nil {
		log.Fatal(err)
	}
	fmt.Println("Wrote to", *output)
}
<|endoftext|>


<commit_before>//
// Copyright 2021, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"net/http"
)

// ReleaseLinksService handles communication with the release link methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html
type ReleaseLinksService struct {
	client *Client
}

// ReleaseLink represents a release link.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html
type ReleaseLink struct {
	ID             int    `json:"id"`
	Name           string `json:"name"`
	URL            string `json:"url"`
	External       bool   `json:"external"`
	DirectAssetURL string `json:"direct_asset_url,omitempty"`
	LinkType       string `json:"link_type,omitempty"`
}

// ListReleaseLinksOptions represents ListReleaseLinks() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#get-links
type ListReleaseLinksOptions ListOptions

// ListReleaseLinks gets assets as links from a Release.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#get-links
func (s *ReleaseLinksService) ListReleaseLinks(pid interface{}, tagName string, opt *ListReleaseLinksOptions, options ...RequestOptionFunc) ([]*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links", pathEscape(project), tagName)

	req, err := s.client.NewRequest(http.MethodGet, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var rls []*ReleaseLink
	resp, err := s.client.Do(req, &rls)
	if err != nil {
		return nil, resp, err
	}

	return rls, resp, err
}

// GetReleaseLink returns a link from release assets.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#get-a-link
func (s *ReleaseLinksService) GetReleaseLink(pid interface{}, tagName string, link int, options ...RequestOptionFunc) (*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links/%d",
		pathEscape(project),
		tagName,
		link)

	req, err := s.client.NewRequest(http.MethodGet, u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	rl := new(ReleaseLink)
	resp, err := s.client.Do(req, rl)
	if err != nil {
		return nil, resp, err
	}

	return rl, resp, err
}

// CreateReleaseLinkOptions represents CreateReleaseLink() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#create-a-link
type CreateReleaseLinkOptions struct {
	Name     *string `url:"name" json:"name"`
	URL      *string `url:"url" json:"url"`
	FilePath *string `url:"file_path,omitempty" json:"file_path,omitempty"`
	LinkType *string `url:"link_type,omitempty" json:"link_type,omitempty"`
}

// CreateReleaseLink creates a link.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#create-a-link
func (s *ReleaseLinksService) CreateReleaseLink(pid interface{}, tagName string, opt *CreateReleaseLinkOptions, options ...RequestOptionFunc) (*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links", pathEscape(project), tagName)

	req, err := s.client.NewRequest(http.MethodPost, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	rl := new(ReleaseLink)
	resp, err := s.client.Do(req, rl)
	if err != nil {
		return nil, resp, err
	}

	return rl, resp, err
}

// UpdateReleaseLinkOptions represents UpdateReleaseLink() options.
//
// You have to specify at least one of Name of URL.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#update-a-link
type UpdateReleaseLinkOptions struct {
	Name     *string `url:"name,omitempty" json:"name,omitempty"`
	URL      *string `url:"url,omitempty" json:"url,omitempty"`
	FilePath *string `url:"file_path,omitempty" json:"file_path,omitempty"`
	LinkType *string `url:"link_type,omitempty" json:"link_type,omitempty"`
}

// UpdateReleaseLink updates an asset link.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#update-a-link
func (s *ReleaseLinksService) UpdateReleaseLink(pid interface{}, tagName string, link int, opt *UpdateReleaseLinkOptions, options ...RequestOptionFunc) (*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links/%d",
		pathEscape(project),
		tagName,
		link)

	req, err := s.client.NewRequest(http.MethodPut, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	rl := new(ReleaseLink)
	resp, err := s.client.Do(req, rl)
	if err != nil {
		return nil, resp, err
	}

	return rl, resp, err
}

// DeleteReleaseLink deletes a link from release.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#delete-a-link
func (s *ReleaseLinksService) DeleteReleaseLink(pid interface{}, tagName string, link int, options ...RequestOptionFunc) (*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links/%d",
		pathEscape(project),
		tagName,
		link,
	)

	req, err := s.client.NewRequest(http.MethodDelete, u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	rl := new(ReleaseLink)
	resp, err := s.client.Do(req, rl)
	if err != nil {
		return nil, resp, err
	}

	return rl, resp, err
}
<commit_msg>Remove erroneous underscore from file path<commit_after>//
// Copyright 2021, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"net/http"
)

// ReleaseLinksService handles communication with the release link methods
// of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html
type ReleaseLinksService struct {
	client *Client
}

// ReleaseLink represents a release link.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html
type ReleaseLink struct {
	ID             int    `json:"id"`
	Name           string `json:"name"`
	URL            string `json:"url"`
	External       bool   `json:"external"`
	DirectAssetURL string `json:"direct_asset_url,omitempty"`
	LinkType       string `json:"link_type,omitempty"`
}

// ListReleaseLinksOptions represents ListReleaseLinks() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#get-links
type ListReleaseLinksOptions ListOptions

// ListReleaseLinks gets assets as links from a Release.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#get-links
func (s *ReleaseLinksService) ListReleaseLinks(pid interface{}, tagName string, opt *ListReleaseLinksOptions, options ...RequestOptionFunc) ([]*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links", pathEscape(project), tagName)

	req, err := s.client.NewRequest(http.MethodGet, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	var rls []*ReleaseLink
	resp, err := s.client.Do(req, &rls)
	if err != nil {
		return nil, resp, err
	}

	return rls, resp, err
}

// GetReleaseLink returns a link from release assets.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#get-a-link
func (s *ReleaseLinksService) GetReleaseLink(pid interface{}, tagName string, link int, options ...RequestOptionFunc) (*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links/%d",
		pathEscape(project),
		tagName,
		link)

	req, err := s.client.NewRequest(http.MethodGet, u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	rl := new(ReleaseLink)
	resp, err := s.client.Do(req, rl)
	if err != nil {
		return nil, resp, err
	}

	return rl, resp, err
}

// CreateReleaseLinkOptions represents CreateReleaseLink() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#create-a-link
type CreateReleaseLinkOptions struct {
	Name     *string `url:"name" json:"name"`
	URL      *string `url:"url" json:"url"`
	FilePath *string `url:"filepath,omitempty" json:"filepath,omitempty"`
	LinkType *string `url:"link_type,omitempty" json:"link_type,omitempty"`
}

// CreateReleaseLink creates a link.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#create-a-link
func (s *ReleaseLinksService) CreateReleaseLink(pid interface{}, tagName string, opt *CreateReleaseLinkOptions, options ...RequestOptionFunc) (*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links", pathEscape(project), tagName)

	req, err := s.client.NewRequest(http.MethodPost, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	rl := new(ReleaseLink)
	resp, err := s.client.Do(req, rl)
	if err != nil {
		return nil, resp, err
	}

	return rl, resp, err
}

// UpdateReleaseLinkOptions represents UpdateReleaseLink() options.
//
// You have to specify at least one of Name of URL.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#update-a-link
type UpdateReleaseLinkOptions struct {
	Name     *string `url:"name,omitempty" json:"name,omitempty"`
	URL      *string `url:"url,omitempty" json:"url,omitempty"`
	FilePath *string `url:"filepath,omitempty" json:"filepath,omitempty"`
	LinkType *string `url:"link_type,omitempty" json:"link_type,omitempty"`
}

// UpdateReleaseLink updates an asset link.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#update-a-link
func (s *ReleaseLinksService) UpdateReleaseLink(pid interface{}, tagName string, link int, opt *UpdateReleaseLinkOptions, options ...RequestOptionFunc) (*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links/%d",
		pathEscape(project),
		tagName,
		link)

	req, err := s.client.NewRequest(http.MethodPut, u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	rl := new(ReleaseLink)
	resp, err := s.client.Do(req, rl)
	if err != nil {
		return nil, resp, err
	}

	return rl, resp, err
}

// DeleteReleaseLink deletes a link from release.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/releases/links.html#delete-a-link
func (s *ReleaseLinksService) DeleteReleaseLink(pid interface{}, tagName string, link int, options ...RequestOptionFunc) (*ReleaseLink, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/releases/%s/assets/links/%d",
		pathEscape(project),
		tagName,
		link,
	)

	req, err := s.client.NewRequest(http.MethodDelete, u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	rl := new(ReleaseLink)
	resp, err := s.client.Do(req, rl)
	if err != nil {
		return nil, resp, err
	}

	return rl, resp, err
}
<|endoftext|>


<commit_before>package basis

import (
	"database/sql"
	"github.com/Cepave/open-falcon-backend/common/model"
	"github.com/Cepave/open-falcon-backend/modules/hbs/g"
	commonDb "github.com/Cepave/open-falcon-backend/common/db"
	log "github.com/sirupsen/logrus"
)

func UpdateAgent(agentInfo *model.AgentUpdateInfo) error {
	var err error = nil

	/**
	 * 如果 config 的 Hosts 有值，
	 * 只更新 host，不新增 host
	 */
	if g.Config().Hosts == "" {
		err = refreshAgent(agentInfo)
	} else {
		err = updateAgent(agentInfo)
	}
	// :~)

	if err != nil {
		log.Errorf("Refresh host info(by agent) has error: %v.", err)
	}

	return err
}

/**
 * Refresh agent or add a new one
 */
type refreshHostProcessor model.AgentReportRequest
func (self *refreshHostProcessor) BootCallback(tx *sql.Tx) bool {
	result := commonDb.ToTxExt(tx).Exec(
		`
		UPDATE host
		SET ip = ?,
			agent_version = ?,
			plugin_version = ?,
			update_at = NOW()
		WHERE hostname = ?
		`,
		self.IP,
		self.AgentVersion,
		self.PluginVersion,
		self.Hostname,
	)

	return commonDb.ToResultExt(result).RowsAffected() == 0
}
func (self *refreshHostProcessor) IfTrue(tx *sql.Tx) {
	commonDb.ToTxExt(tx).Exec(
		`
		INSERT INTO host(
			hostname,
			ip, agent_version, plugin_version
		)
		VALUES (?, ?, ?, ?)
		ON DUPLICATE KEY UPDATE
			ip = ?,
			agent_version = ?,
			plugin_version = ?
		`,
		self.Hostname,
		self.IP,
		self.AgentVersion,
		self.PluginVersion,
		self.IP,
		self.AgentVersion,
		self.PluginVersion,
	)
}

func refreshAgent(agentInfo *model.AgentUpdateInfo) (dbError error) {
	processor := refreshHostProcessor(*agentInfo.ReportRequest)

	DbFacade.SqlDbCtrl.InTxForIf(&processor)

	return
}
func updateAgent(agentInfo *model.AgentUpdateInfo) (dbError error) {
	DbFacade.SqlDbCtrl.Exec(
		`
		UPDATE host
		SET ip = ?,
			agent_version = ?,
			plugin_version = ?
		WHERE hostname = ?
		`,
		agentInfo.ReportRequest.IP,
		agentInfo.ReportRequest.AgentVersion,
		agentInfo.ReportRequest.PluginVersion,
		agentInfo.ReportRequest.Hostname,
	)

	return
}
<commit_msg>[OWL-1695] Remove the stale file. File is added accidentally due to merge conflict.<commit_after><|endoftext|>


<commit_before>package jira

import (
	// Stdlib
	"fmt"
	"net/url"
	"strings"

	// Internal
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/version"

	// Vendor
	"github.com/salsita/go-jira/v2/jira"
	"github.com/toqueteos/webbrowser"
)

const ServiceName = "JIRA"

type issueTracker struct {
	config       Config
	versionCache map[string]*jira.Version
}

func Factory() (common.IssueTracker, error) {
	config, err := LoadConfig()
	if err != nil {
		return nil, err
	}
	return &issueTracker{config, nil}, nil
}

func (tracker *issueTracker) ServiceName() string {
	return ServiceName
}

func (tracker *issueTracker) CurrentUser() (common.User, error) {
	data, _, err := newClient(tracker.config).Myself.Get()
	if err != nil {
		return nil, err
	}
	return &user{data}, nil
}

func (tracker *issueTracker) StartableStories() (stories []common.Story, err error) {
	query := fmt.Sprintf("(%v) AND (%v)",
		formatInRange("type", codingIssueTypeIds...),
		formatInRange("status", startableStateIds...))

	return tracker.searchStories(query)
}

func (tracker *issueTracker) StoriesInDevelopment() (stories []common.Story, err error) {
	query := fmt.Sprintf("(%v) AND (%v)",
		formatInRange("type", codingIssueTypeIds...),
		formatInRange("status", inDevelopmentStateIds...))

	return tracker.searchStories(query)
}

func (tracker *issueTracker) ReviewedStories() (stories []common.Story, err error) {
	query := fmt.Sprintf("(%v) AND (%v)",
		formatInRange("type", codingIssueTypeIds...),
		formatInRange("status", stateIdReviewed))

	return tracker.searchStories(query)
}

func (tracker *issueTracker) ListStoriesByTag(tags []string) (stories []common.Story, err error) {
	// Fetch issues by ID. Apparently, the tags are the same as issue keys for JIRA.
	issues, err := listStoriesByIdOrdered(newClient(tracker.config), tags)
	if err != nil {
		return nil, err
	}

	// Convert to []common.Story and return.
	return toCommonStories(issues, tracker), nil
}

func (tracker *issueTracker) ListStoriesByRelease(v *version.Version) ([]common.Story, error) {
	return tracker.storiesByRelease(v)
}

func (tracker *issueTracker) NextRelease(
	trunkVersion *version.Version,
	nextTrunkVersion *version.Version,
) (common.NextRelease, error) {

	return newNextRelease(tracker, trunkVersion, nextTrunkVersion)
}

func (tracker *issueTracker) RunningRelease(
	releaseVersion *version.Version,
) (common.RunningRelease, error) {

	return newRunningRelease(tracker, releaseVersion)
}

func (tracker *issueTracker) OpenStory(storyId string) error {
	relativeURL, _ := url.Parse("browse/" + storyId)
	return webbrowser.Open(tracker.config.ServerURL().ResolveReference(relativeURL).String())
}

func (tracker *issueTracker) StoryTagToReadableStoryId(tag string) (storyId string, err error) {
	prefix := fmt.Sprintf("%v-", tracker.config.ProjectKey())
	if !strings.HasPrefix(tag, prefix) {
		return "", fmt.Errorf("not a valid issue key: %v", tag)
	}
	return tag, nil
}

func (tracker *issueTracker) getVersionResource(ver *version.Version) (*jira.Version, error) {
	var (
		projectKey  = tracker.config.ProjectKey()
		versionName = ver.ReleaseTagString()
		api         = newClient(tracker.config)
	)

	// In case the resource cache is empty, fill it.
	if tracker.versionCache == nil {
		vs, _, err := api.Projects.ListVersions(projectKey)
		if err != nil {
			return nil, err
		}

		m := make(map[string]*jira.Version, len(vs))
		for _, v := range vs {
			m[v.Name] = v
		}
		tracker.versionCache = m
	}

	// Return the resource we are looking for.
	if res, ok := tracker.versionCache[versionName]; ok {
		return res, nil
	}
	return nil, nil
}

func (tracker *issueTracker) searchIssues(queryFormat string, v ...interface{}) ([]*jira.Issue, error) {
	query := fmt.Sprintf(queryFormat, v...)
	jql := fmt.Sprintf("project = \"%v\" AND (%v)", tracker.config.ProjectKey(), query)

	issues, _, err := newClient(tracker.config).Issues.Search(&jira.SearchOptions{
		JQL:        jql,
		MaxResults: 200,
	})
	if err != nil {
		return nil, err
	}
	return issues, nil
}

func (tracker *issueTracker) issuesByRelease(v *version.Version) ([]*jira.Issue, error) {
	label := v.ReleaseTagString()
	return tracker.searchIssues("labels = %v", label)
}

func (tracker *issueTracker) searchStories(queryFormat string, v ...interface{}) ([]common.Story, error) {
	issues, err := tracker.searchIssues(queryFormat, v...)
	if err != nil {
		return nil, err
	}
	return toCommonStories(issues, tracker), nil
}

func (tracker *issueTracker) storiesByRelease(v *version.Version) ([]common.Story, error) {
	issues, err := tracker.issuesByRelease(v)
	if err != nil {
		return nil, err
	}
	return toCommonStories(issues, tracker), nil
}

func toCommonStories(issues []*jira.Issue, tracker *issueTracker) []common.Story {
	stories := make([]common.Story, len(issues))
	for i := range issues {
		s, err := newStory(issues[i], tracker)
		if err != nil {
			panic(err)
		}
		stories[i] = s
	}
	return stories
}
<commit_msg>jira: Refactoring<commit_after>package jira

import (
	// Stdlib
	"fmt"
	"net/url"
	"strings"

	// Internal
	"github.com/salsaflow/salsaflow/modules/common"
	"github.com/salsaflow/salsaflow/version"

	// Vendor
	"github.com/salsita/go-jira/v2/jira"
	"github.com/toqueteos/webbrowser"
)

const ServiceName = "JIRA"

type issueTracker struct {
	config       Config
	versionCache map[string]*jira.Version
}

func Factory() (common.IssueTracker, error) {
	config, err := LoadConfig()
	if err != nil {
		return nil, err
	}
	return &issueTracker{config, nil}, nil
}

func (tracker *issueTracker) ServiceName() string {
	return ServiceName
}

func (tracker *issueTracker) CurrentUser() (common.User, error) {
	data, _, err := newClient(tracker.config).Myself.Get()
	if err != nil {
		return nil, err
	}
	return &user{data}, nil
}

func (tracker *issueTracker) StartableStories() (stories []common.Story, err error) {
	query := fmt.Sprintf("(%v) AND (%v)",
		formatInRange("type", codingIssueTypeIds...),
		formatInRange("status", startableStateIds...))

	return tracker.searchStories(query)
}

func (tracker *issueTracker) StoriesInDevelopment() (stories []common.Story, err error) {
	query := fmt.Sprintf("(%v) AND (%v)",
		formatInRange("type", codingIssueTypeIds...),
		formatInRange("status", inDevelopmentStateIds...))

	return tracker.searchStories(query)
}

func (tracker *issueTracker) ReviewedStories() (stories []common.Story, err error) {
	query := fmt.Sprintf("(%v) AND (%v)",
		formatInRange("type", codingIssueTypeIds...),
		formatInRange("status", stateIdReviewed))

	return tracker.searchStories(query)
}

func (tracker *issueTracker) ListStoriesByTag(tags []string) (stories []common.Story, err error) {
	// Fetch issues by ID. Apparently, the tags are the same as issue keys for JIRA.
	issues, err := listStoriesByIdOrdered(newClient(tracker.config), tags)
	if err != nil {
		return nil, err
	}

	// Convert to []common.Story and return.
	return toCommonStories(issues, tracker), nil
}

func (tracker *issueTracker) ListStoriesByRelease(v *version.Version) ([]common.Story, error) {
	issues, err := tracker.issuesByRelease(v)
	if err != nil {
		return nil, err
	}
	return toCommonStories(issues, tracker), nil
}

func (tracker *issueTracker) NextRelease(
	trunkVersion *version.Version,
	nextTrunkVersion *version.Version,
) (common.NextRelease, error) {

	return newNextRelease(tracker, trunkVersion, nextTrunkVersion)
}

func (tracker *issueTracker) RunningRelease(
	releaseVersion *version.Version,
) (common.RunningRelease, error) {

	return newRunningRelease(tracker, releaseVersion)
}

func (tracker *issueTracker) OpenStory(storyId string) error {
	relativeURL, _ := url.Parse("browse/" + storyId)
	return webbrowser.Open(tracker.config.ServerURL().ResolveReference(relativeURL).String())
}

func (tracker *issueTracker) StoryTagToReadableStoryId(tag string) (storyId string, err error) {
	prefix := fmt.Sprintf("%v-", tracker.config.ProjectKey())
	if !strings.HasPrefix(tag, prefix) {
		return "", fmt.Errorf("not a valid issue key: %v", tag)
	}
	return tag, nil
}

func (tracker *issueTracker) getVersionResource(ver *version.Version) (*jira.Version, error) {
	var (
		projectKey  = tracker.config.ProjectKey()
		versionName = ver.ReleaseTagString()
		api         = newClient(tracker.config)
	)

	// In case the resource cache is empty, fill it.
	if tracker.versionCache == nil {
		vs, _, err := api.Projects.ListVersions(projectKey)
		if err != nil {
			return nil, err
		}

		m := make(map[string]*jira.Version, len(vs))
		for _, v := range vs {
			m[v.Name] = v
		}
		tracker.versionCache = m
	}

	// Return the resource we are looking for.
	if res, ok := tracker.versionCache[versionName]; ok {
		return res, nil
	}
	return nil, nil
}

func (tracker *issueTracker) searchIssues(queryFormat string, v ...interface{}) ([]*jira.Issue, error) {
	query := fmt.Sprintf(queryFormat, v...)
	jql := fmt.Sprintf("project = \"%v\" AND (%v)", tracker.config.ProjectKey(), query)

	issues, _, err := newClient(tracker.config).Issues.Search(&jira.SearchOptions{
		JQL:        jql,
		MaxResults: 200,
	})
	if err != nil {
		return nil, err
	}
	return issues, nil
}

func (tracker *issueTracker) issuesByRelease(v *version.Version) ([]*jira.Issue, error) {
	label := v.ReleaseTagString()
	return tracker.searchIssues("labels = %v", label)
}

func (tracker *issueTracker) searchStories(queryFormat string, v ...interface{}) ([]common.Story, error) {
	issues, err := tracker.searchIssues(queryFormat, v...)
	if err != nil {
		return nil, err
	}
	return toCommonStories(issues, tracker), nil
}

func toCommonStories(issues []*jira.Issue, tracker *issueTracker) []common.Story {
	stories := make([]common.Story, len(issues))
	for i := range issues {
		s, err := newStory(issues[i], tracker)
		if err != nil {
			panic(err)
		}
		stories[i] = s
	}
	return stories
}
<|endoftext|>


<commit_before>// Package brimtext contains tools for working with text. Probably the most
// complex of these tools is Align, which allows for formatting "pretty
// tables".
package brimtext

import (
	"bytes"
	"fmt"
	"math"
	"strconv"
	"strings"
	"unicode/utf8"
)

// OrdinalSuffix returns "st", "nd", "rd", etc. for the number given (1st, 2nd,
// 3rd, etc.).
func OrdinalSuffix(number int) string {
	if (number/10)%10 == 1 || number%10 > 3 {
		return "th"
	} else if number%10 == 1 {
		return "st"
	} else if number%10 == 2 {
		return "nd"
	} else if number%10 == 3 {
		return "rd"
	}
	return "th"
}

// ThousandsSep returns the number formatted using the separator at each
// thousands position, such as ThousandsSep(1234567, ",") giving 1,234,567.
func ThousandsSep(v int64, sep string) string {
	s := strconv.FormatInt(v, 10)
	for i := len(s) - 3; i > 0; i -= 3 {
		s = s[:i] + "," + s[i:]
	}
	return s
}

// ThousandsSepU returns the number formatted using the separator at each
// thousands position, such as ThousandsSepU(1234567, ",") giving 1,234,567.
func ThousandsSepU(v uint64, sep string) string {
	s := strconv.FormatUint(v, 10)
	for i := len(s) - 3; i > 0; i -= 3 {
		s = s[:i] + "," + s[i:]
	}
	return s
}

func humanSize(v float64, u float64, s []string) string {
	n := v
	i := 0
	for ; i < len(s); i++ {
		if math.Ceil(n) < 1000 {
			break
		}
		n = n / u
	}
	if i >= len(s) {
		return fmt.Sprintf("%.0f%s", n*u, s[len(s)-1])
	}
	if i == 0 {
		return fmt.Sprintf("%.4g", n)
	}
	if n < 1 {
		return fmt.Sprintf("%.2g%s", n, s[i])
	}
	return fmt.Sprintf("%.3g%s", n, s[i])
}

// HumanSize1000 returns a more readable size format, such as
// HumanSize1000(1234567) giving "1.23m".
// These are 1,000 unit based: 1k = 1000, 1m = 1000000, etc.
func HumanSize1000(v float64) string {
	return humanSize(v, 1000, []string{"", "k", "m", "g", "t", "p", "e", "z", "y"})
}

// HumanSize1024 returns a more readable size format, such as
// HumanSize1024(1234567) giving "1.18m".
// These are 1,024 unit based: 1k = 1024, 1m = 1048576, etc.
func HumanSize1024(v float64) string {
	return humanSize(v, 1024, []string{"", "K", "M", "G", "T", "P", "E", "Z", "Y"})
}

// Sentence converts the value into a sentence, uppercasing the first character
// and ensuring the string ends with a period. Useful to output better looking
// error.Error() messages, which are all lower case with no trailing period by
// convention.
func Sentence(value string) string {
	if value != "" {
		if value[len(value)-1] != '.' {
			value = strings.ToUpper(value[:1]) + value[1:] + "."
		} else {
			value = strings.ToUpper(value[:1]) + value[1:]
		}
	}
	return value
}

// StringSliceToLowerSort provides a sort.Interface that will sort a []string
// by their strings.ToLower values. This isn't exactly a case insensitive sort
// due to Unicode situations, but is usually good enough.
type StringSliceToLowerSort []string

func (s StringSliceToLowerSort) Len() int {
	return len(s)
}

func (s StringSliceToLowerSort) Swap(x int, y int) {
	s[x], s[y] = s[y], s[x]
}

func (s StringSliceToLowerSort) Less(x int, y int) bool {
	return strings.ToLower(s[x]) < strings.ToLower(s[y])
}

// Wrap wraps text for more readable output.
//
// The width can be a positive int for a specific width, 0 for the default
// width (attempted to get from terminal, 79 otherwise), or a negative number
// for a width relative to the default.
//
// The indent1 is the prefix for the first line.
//
// The indent2 is the prefix for any second or subsequent lines.
func Wrap(text string, width int, indent1 string, indent2 string) string {
	if width < 1 {
		width = GetTTYWidth() - 1 + width
	}
	bs := []byte(text)
	bs = wrap(bs, width, []byte(indent1), []byte(indent2))
	return string(bytes.Trim(bs, "\n"))
}

func wrap(text []byte, width int, indent1 []byte, indent2 []byte) []byte {
	if utf8.RuneCount(text) == 0 {
		return text
	}
	text = bytes.Replace(text, []byte{'\r', '\n'}, []byte{'\n'}, -1)
	var out bytes.Buffer
	for _, par := range bytes.Split([]byte(text), []byte{'\n', '\n'}) {
		par = bytes.Replace(par, []byte{'\n'}, []byte{' '}, -1)
		lineLen := 0
		start := true
		for _, word := range bytes.Split(par, []byte{' '}) {
			wordLen := utf8.RuneCount(word)
			if wordLen == 0 {
				continue
			}
			scan := word
			for len(scan) > 1 {
				i := bytes.IndexByte(scan, '\x1b')
				if i == -1 {
					break
				}
				j := bytes.IndexByte(scan[i+1:], 'm')
				if j == -1 {
					i++
				} else {
					j += 2
					wordLen -= j
					scan = scan[i+j:]
				}
			}
			if start {
				out.Write(indent1)
				lineLen += utf8.RuneCount(indent1)
				out.Write(word)
				lineLen += wordLen
				start = false
			} else if lineLen+1+wordLen > width {
				out.WriteByte('\n')
				out.Write(indent2)
				out.Write(word)
				lineLen = utf8.RuneCount(indent2) + wordLen
			} else {
				out.WriteByte(' ')
				out.Write(word)
				lineLen += 1 + wordLen
			}
		}
		out.WriteByte('\n')
		out.WriteByte('\n')
	}
	return out.Bytes()
}

// AllEqual returns true if all the values are equal strings; no strings,
// AllEqual() or AllEqual([]string{}...), are considered AllEqual.
func AllEqual(values ...string) bool {
	if len(values) < 2 {
		return true
	}
	compare := values[0]
	for _, v := range values[1:] {
		if v != compare {
			return false
		}
	}
	return true
}

// TrueString returns true if the string contains a recognized true value, such
// as "true", "True", "TRUE", "yes", "on", etc. Yes, there is already
// strconv.ParseBool, but this function is often easier to work with since it
// just returns true or false instead of (bool, error) like ParseBool does. If
// you need to differentiate between true, false, and unknown, ParseBool should
// be your choice. Although I suppose you could use TrueString(s),
// FalseString(s), and !TrueString(s) && !FalseString(s).
func TrueString(value string) bool {
	v := strings.ToLower(value)
	switch v {
	case "true":
		return true
	case "yes":
		return true
	case "on":
		return true
	case "1":
		return true
	}
	return false
}

// FalseString returns true if the string contains a recognized false value,
// such as "false", "False", "FALSE", "no", "off", etc. Yes, there is already
// strconv.ParseBool, but this function is often easier to work with since it
// just returns true or false instead of (bool, error) like ParseBool does. If
// you need to differentiate between true, false, and unknown, ParseBool should
// be your choice. Although I suppose you could use TrueString(s),
// FalseString(s), and !TrueString(s) && !FalseString(s).
func FalseString(value string) bool {
	v := strings.ToLower(value)
	switch v {
	case "false":
		return true
	case "no":
		return true
	case "off":
		return true
	case "0":
		return true
	}
	return false
}
<commit_msg>Fixed more typos<commit_after>// Package brimtext contains tools for working with text. Probably the most
// complex of these tools is Align, which allows for formatting "pretty
// tables".
package brimtext

import (
	"bytes"
	"fmt"
	"math"
	"strconv"
	"strings"
	"unicode/utf8"
)

// OrdinalSuffix returns "st", "nd", "rd", etc. for the number given (1st, 2nd,
// 3rd, etc.).
func OrdinalSuffix(number int) string {
	if (number/10)%10 == 1 || number%10 > 3 {
		return "th"
	} else if number%10 == 1 {
		return "st"
	} else if number%10 == 2 {
		return "nd"
	} else if number%10 == 3 {
		return "rd"
	}
	return "th"
}

// ThousandsSep returns the number formatted using the separator at each
// thousands position, such as ThousandsSep(1234567, ",") giving 1,234,567.
func ThousandsSep(v int64, sep string) string {
	s := strconv.FormatInt(v, 10)
	for i := len(s) - 3; i > 0; i -= 3 {
		s = s[:i] + "," + s[i:]
	}
	return s
}

// ThousandsSepU returns the number formatted using the separator at each
// thousands position, such as ThousandsSepU(1234567, ",") giving 1,234,567.
func ThousandsSepU(v uint64, sep string) string {
	s := strconv.FormatUint(v, 10)
	for i := len(s) - 3; i > 0; i -= 3 {
		s = s[:i] + "," + s[i:]
	}
	return s
}

func humanSize(v float64, u float64, s []string) string {
	n := v
	i := 0
	for ; i < len(s); i++ {
		if math.Ceil(n) < 1000 {
			break
		}
		n = n / u
	}
	if i >= len(s) {
		return fmt.Sprintf("%.0f%s", n*u, s[len(s)-1])
	}
	if i == 0 {
		return fmt.Sprintf("%.4g", n)
	}
	if n < 1 {
		return fmt.Sprintf("%.2g%s", n, s[i])
	}
	return fmt.Sprintf("%.3g%s", n, s[i])
}

// HumanSize1000 returns a more readable size format, such as
// HumanSize1000(1234567) giving "1.23m".
// These are 1,000 unit based: 1k = 1000, 1m = 1000000, etc.
func HumanSize1000(v float64) string {
	return humanSize(v, 1000, []string{"", "k", "m", "g", "t", "p", "e", "z", "y"})
}

// HumanSize1024 returns a more readable size format, such as
// HumanSize1024(1234567) giving "1.18M".
// These are 1,024 unit based: 1K = 1024, 1M = 1048576, etc.
func HumanSize1024(v float64) string {
	return humanSize(v, 1024, []string{"", "K", "M", "G", "T", "P", "E", "Z", "Y"})
}

// Sentence converts the value into a sentence, uppercasing the first character
// and ensuring the string ends with a period. Useful to output better looking
// error.Error() messages, which are all lower case with no trailing period by
// convention.
func Sentence(value string) string {
	if value != "" {
		if value[len(value)-1] != '.' {
			value = strings.ToUpper(value[:1]) + value[1:] + "."
		} else {
			value = strings.ToUpper(value[:1]) + value[1:]
		}
	}
	return value
}

// StringSliceToLowerSort provides a sort.Interface that will sort a []string
// by their strings.ToLower values. This isn't exactly a case insensitive sort
// due to Unicode situations, but is usually good enough.
type StringSliceToLowerSort []string

func (s StringSliceToLowerSort) Len() int {
	return len(s)
}

func (s StringSliceToLowerSort) Swap(x int, y int) {
	s[x], s[y] = s[y], s[x]
}

func (s StringSliceToLowerSort) Less(x int, y int) bool {
	return strings.ToLower(s[x]) < strings.ToLower(s[y])
}

// Wrap wraps text for more readable output.
//
// The width can be a positive int for a specific width, 0 for the default
// width (attempted to get from terminal, 79 otherwise), or a negative number
// for a width relative to the default.
//
// The indent1 is the prefix for the first line.
//
// The indent2 is the prefix for any second or subsequent lines.
func Wrap(text string, width int, indent1 string, indent2 string) string {
	if width < 1 {
		width = GetTTYWidth() - 1 + width
	}
	bs := []byte(text)
	bs = wrap(bs, width, []byte(indent1), []byte(indent2))
	return string(bytes.Trim(bs, "\n"))
}

func wrap(text []byte, width int, indent1 []byte, indent2 []byte) []byte {
	if utf8.RuneCount(text) == 0 {
		return text
	}
	text = bytes.Replace(text, []byte{'\r', '\n'}, []byte{'\n'}, -1)
	var out bytes.Buffer
	for _, par := range bytes.Split([]byte(text), []byte{'\n', '\n'}) {
		par = bytes.Replace(par, []byte{'\n'}, []byte{' '}, -1)
		lineLen := 0
		start := true
		for _, word := range bytes.Split(par, []byte{' '}) {
			wordLen := utf8.RuneCount(word)
			if wordLen == 0 {
				continue
			}
			scan := word
			for len(scan) > 1 {
				i := bytes.IndexByte(scan, '\x1b')
				if i == -1 {
					break
				}
				j := bytes.IndexByte(scan[i+1:], 'm')
				if j == -1 {
					i++
				} else {
					j += 2
					wordLen -= j
					scan = scan[i+j:]
				}
			}
			if start {
				out.Write(indent1)
				lineLen += utf8.RuneCount(indent1)
				out.Write(word)
				lineLen += wordLen
				start = false
			} else if lineLen+1+wordLen > width {
				out.WriteByte('\n')
				out.Write(indent2)
				out.Write(word)
				lineLen = utf8.RuneCount(indent2) + wordLen
			} else {
				out.WriteByte(' ')
				out.Write(word)
				lineLen += 1 + wordLen
			}
		}
		out.WriteByte('\n')
		out.WriteByte('\n')
	}
	return out.Bytes()
}

// AllEqual returns true if all the values are equal strings; no strings,
// AllEqual() or AllEqual([]string{}...), are considered AllEqual.
func AllEqual(values ...string) bool {
	if len(values) < 2 {
		return true
	}
	compare := values[0]
	for _, v := range values[1:] {
		if v != compare {
			return false
		}
	}
	return true
}

// TrueString returns true if the string contains a recognized true value, such
// as "true", "True", "TRUE", "yes", "on", etc. Yes, there is already
// strconv.ParseBool, but this function is often easier to work with since it
// just returns true or false instead of (bool, error) like ParseBool does. If
// you need to differentiate between true, false, and unknown, ParseBool should
// be your choice. Although I suppose you could use TrueString(s),
// FalseString(s), and !TrueString(s) && !FalseString(s).
func TrueString(value string) bool {
	v := strings.ToLower(value)
	switch v {
	case "true":
		return true
	case "yes":
		return true
	case "on":
		return true
	case "1":
		return true
	}
	return false
}

// FalseString returns true if the string contains a recognized false value,
// such as "false", "False", "FALSE", "no", "off", etc. Yes, there is already
// strconv.ParseBool, but this function is often easier to work with since it
// just returns true or false instead of (bool, error) like ParseBool does. If
// you need to differentiate between true, false, and unknown, ParseBool should
// be your choice. Although I suppose you could use TrueString(s),
// FalseString(s), and !TrueString(s) && !FalseString(s).
func FalseString(value string) bool {
	v := strings.ToLower(value)
	switch v {
	case "false":
		return true
	case "no":
		return true
	case "off":
		return true
	case "0":
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package bsondiff

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/kylelemons/godebug/pretty"
	"gopkg.in/mgo.v2/bson"
)

type Program struct {
	JSON   bool
	Stdout io.Writer
}

func (p *Program) Run(f *flag.FlagSet, args []string) error {
	f.BoolVar(&p.JSON, "json", false, "")

	if err := f.Parse(args); err != nil {
		return err
	}

	if p.JSON {
		cols, err := p.readCollections(f.Args()...)
		if err != nil {
			return err
		}

		enc := json.NewEncoder(p.Stdout)
		enc.SetIndent("", "\t")

		return enc.Encode(cols)
	}

	if f.NArg() != 2 {
		return errors.New("usage: bsondiff <old dump dir> <new dump dir>")
	}

	oldCols, err := p.readCollectionDir(f.Arg(0))
	if err != nil {
		return fmt.Errorf("reading %q: %s", f.Arg(0), err)
	}

	newCols, err := p.readCollectionDir(f.Arg(1))
	if err != nil {
		return fmt.Errorf("reading %q: %s", f.Arg(1), err)
	}

	fmt.Fprintln(p.Stdout, pretty.Compare(oldCols, newCols))

	return nil
}

func (p *Program) readFiles(path string) (files []string, err error) {
	fi, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	if fi.IsDir() {
		fis, err := ioutil.ReadDir(path)
		if err != nil {
			return nil, err
		}

		for _, fi := range fis {
			files = append(files, filepath.Join(path, fi.Name()))
		}
	} else {
		files = append(files, path)
	}

	return files, nil
}

func (p *Program) readCollections(files ...string) (map[string]interface{}, error) {
	cols := make(map[string]interface{})

	for _, file := range files {
		name := filepath.Base(file)

		i := strings.Index(name, ".bson")
		if i == -1 {
			continue
		}

		name = name[:i]

		p, err := ioutil.ReadFile(file)
		if err != nil {
			return nil, err
		}

		var v interface{}

		if err := bson.Unmarshal(p, &v); err != nil {
			return nil, fmt.Errorf("reading %q: %s", file, err)
		}

		cols[name] = v
	}

	return cols, nil
}

func (p *Program) readCollectionDir(dir string) (map[string]interface{}, error) {
	files, err := p.readFiles(dir)
	if err != nil {
		return nil, err
	}

	return p.readCollections(files...)
}
<commit_msg>bsondiff: fix "Document corrupted" error<commit_after>package bsondiff

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/kylelemons/godebug/pretty"
	"gopkg.in/mgo.v2/bson"
)

type Program struct {
	JSON   bool
	Stdout io.Writer
}

func (p *Program) Run(f *flag.FlagSet, args []string) error {
	f.BoolVar(&p.JSON, "json", false, "")

	if err := f.Parse(args); err != nil {
		return err
	}

	if p.JSON {
		cols, err := p.readCollections(f.Args()...)
		if err != nil {
			return err
		}

		enc := json.NewEncoder(p.Stdout)
		enc.SetIndent("", "\t")

		return enc.Encode(cols)
	}

	if f.NArg() != 2 {
		return errors.New("usage: bsondiff <old dump dir> <new dump dir>")
	}

	oldCols, err := p.readCollectionDir(f.Arg(0))
	if err != nil {
		return fmt.Errorf("reading %q: %s", f.Arg(0), err)
	}

	newCols, err := p.readCollectionDir(f.Arg(1))
	if err != nil {
		return fmt.Errorf("reading %q: %s", f.Arg(1), err)
	}

	fmt.Fprintln(p.Stdout, pretty.Compare(oldCols, newCols))

	return nil
}

func (p *Program) readFiles(path string) (files []string, err error) {
	fi, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	if fi.IsDir() {
		fis, err := ioutil.ReadDir(path)
		if err != nil {
			return nil, err
		}

		for _, fi := range fis {
			files = append(files, filepath.Join(path, fi.Name()))
		}
	} else {
		files = append(files, path)
	}

	return files, nil
}

func (p *Program) readCollections(files ...string) (map[string]interface{}, error) {
	cols := make(map[string]interface{})

	for _, file := range files {
		name := filepath.Base(file)

		i := strings.Index(name, ".bson")
		if i == -1 {
			continue
		}

		name = name[:i]

		p, err := ioutil.ReadFile(file)
		if err != nil {
			return nil, err
		}

		if len(bytes.TrimSpace(p)) == 0 {
			continue
		}

		var v interface{}

		if err := bson.Unmarshal(p, &v); err != nil {
			return nil, fmt.Errorf("reading %q: %s", file, err)
		}

		cols[name] = v
	}

	return cols, nil
}

func (p *Program) readCollectionDir(dir string) (map[string]interface{}, error) {
	files, err := p.readFiles(dir)
	if err != nil {
		return nil, err
	}

	return p.readCollections(files...)
}
<|endoftext|>


<commit_before>/* Copyright 2020 The FedLearner Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"context"
	"flag"
	"fmt"
	"net/http"
	"net/http/pprof"
	"os"
	"os/signal"
	"syscall"
	"time"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/informers"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	"k8s.io/klog"

	"github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/apis/fedlearner.k8s.io/v1alpha1"
	crdclientset "github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/client/clientset/versioned"
	crdinformers "github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/client/informers/externalversions"
	"github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/operator"
	"github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/server"
)

var (
	master                      = flag.String("master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
	kubeConfig                  = flag.String("kube-config", "", "Path to a kube config. Only required if out-of-cluster.")
	port                        = flag.String("port", "8080", "The http port controller listening.")
	debugPort                   = flag.String("debug-port", "8081", "The debug http port controller listening.")
	workerNum                   = flag.Int("worker-num", 10, "Number of worker threads used by the fedlearner controller.")
	resyncInterval              = flag.Int("resync-interval", 30, "Informer resync interval in seconds.")
	namespace                   = flag.String("namespace", "default", "The namespace to which controller listen FLApps.")
	ingressExtraHostSuffix      = flag.String("ingress-extra-host-suffix", "", "The extra suffix of hosts when creating ingress.")
	ingressSecretName           = flag.String("ingress-secret-name", "", "The secret name used for tls, only one secret supported now.")
	ingressEnableClientAuth     = flag.Bool("ingress-enabled-client-auth", true, "Whether enable client auth for created ingress.")
	ingressClientAuthSecretName = flag.String("ingress-client-auth-secret-name", "", "The secret name used for client auth, only one secret supported now.")
	enableLeaderElection        = flag.Bool("leader-election", false, "Enable fedlearner controller leader election.")
	leaderElectionLockNamespace = flag.String("leader-election-lock-namespace", "fedlearner-system", "Namespace in which to create the Endpoints for leader election.")
	leaderElectionLockName      = flag.String("leader-election-lock-name", "fedlearner-kubernetes-operator-lock", "Name of the Endpoint for leader election.")
	leaderElectionLeaseDuration = flag.Duration("leader-election-lease-duration", 15*time.Second, "Leader election lease duration.")
	leaderElectionRenewDeadline = flag.Duration("leader-election-renew-deadline", 5*time.Second, "Leader election renew deadline.")
	leaderElectionRetryPeriod   = flag.Duration("leader-election-retry-period", 4*time.Second, "Leader election retry period.")
	grpcClientTimeout           = flag.Duration("grpc-client-timeout", 15*time.Second, "GRPC Client timetout")
)

func buildConfig(masterURL string, kubeConfig string) (*rest.Config, error) {
	if kubeConfig != "" {
		return clientcmd.BuildConfigFromFlags(masterURL, kubeConfig)
	}
	config, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}

	if masterURL != "" {
		config.Host = masterURL
	}
	return config, nil
}

func buildClientset(masterURL string, kubeConfig string) (*clientset.Clientset, *crdclientset.Clientset, error) {
	config, err := buildConfig(masterURL, kubeConfig)
	if err != nil {
		return nil, nil, err
	}

	kubeClient, err := clientset.NewForConfig(config)
	if err != nil {
		return nil, nil, err
	}

	crdClient, err := crdclientset.NewForConfig(config)
	if err != nil {
		return nil, nil, err
	}
	return kubeClient, crdClient, err
}

func startLeaderElection(
	kubeClient *clientset.Clientset,
	recorder record.EventRecorder,
	startCh chan struct{},
	stopCh chan struct{},
) {
	hostName, err := os.Hostname()
	if err != nil {
		klog.Error("failed to get hostname")
		return
	}
	hostName = hostName + "_" + string(uuid.NewUUID())

	resourceLock, err := resourcelock.New(resourcelock.EndpointsResourceLock,
		*leaderElectionLockNamespace,
		*leaderElectionLockName,
		kubeClient.CoreV1(),
		nil,
		resourcelock.ResourceLockConfig{
			Identity:      hostName,
			EventRecorder: recorder,
		})
	if err != nil {
		return
	}

	electionCfg := leaderelection.LeaderElectionConfig{
		Lock:          resourceLock,
		LeaseDuration: *leaderElectionLeaseDuration,
		RenewDeadline: *leaderElectionRenewDeadline,
		RetryPeriod:   *leaderElectionRetryPeriod,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: func(c context.Context) {
				close(startCh)
			},
			OnStoppedLeading: func() {
				close(stopCh)
			},
		},
	}
	elector, err := leaderelection.NewLeaderElector(electionCfg)
	if err != nil {
		klog.Fatal(err)
	}

	go elector.Run(context.Background())
}

func main() {
	flag.Parse()

	kubeClient, crdClient, err := buildClientset(*master, *kubeConfig)
	if err != nil {
		klog.Fatalf("failed to build clientset, err = %v", err)
	}
	if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil {
		klog.Fatalf("Failed to add flapp scheme: %v", err)
	}

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
	stopCh := make(chan struct{}, 1)
	startCh := make(chan struct{}, 1)

	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(klog.Infof)
	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{
		Interface: kubeClient.CoreV1().Events(""),
	})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "fedlearner-operator"})

	if *namespace == metav1.NamespaceAll {
		klog.Fatalf("cluster scoped operator is not supported")
	}
	klog.Infof("scoping operator to namespace %s", *namespace)

	kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(
		kubeClient,
		time.Duration(*resyncInterval)*time.Second,
		informers.WithNamespace(*namespace),
	)
	crdInformerFactory := crdinformers.NewSharedInformerFactoryWithOptions(
		crdClient,
		time.Duration(*resyncInterval)*time.Second,
		crdinformers.WithNamespace(*namespace),
	)

	appEventHandler := operator.NewAppEventHandlerWithClientTimeout(*namespace, crdClient, *grpcClientTimeout)
	flController := operator.NewFLController(
		*namespace,
		recorder,
		*resyncInterval,
		*ingressExtraHostSuffix,
		*ingressSecretName,
		*ingressEnableClientAuth,
		*ingressClientAuthSecretName,
		kubeClient,
		crdClient,
		kubeInformerFactory,
		crdInformerFactory,
		appEventHandler,
		stopCh)

	go func() {
		klog.Infof("starting adapter listening %v", *port)
		server.ServeGrpc("0.0.0.0", *port, appEventHandler)
	}()
	go func() {
		mux := http.NewServeMux()
		mux.HandleFunc("/debug/pprof/", pprof.Index)
		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
		klog.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%s", *debugPort), mux))
	}()

	if *enableLeaderElection {
		startLeaderElection(kubeClient, recorder, startCh, stopCh)
	}

	klog.Info("starting the fedlearner operator")
	if *enableLeaderElection {
		klog.Info("waiting to be elected leader before starting application controller goroutines")
		<-startCh
	}

	go kubeInformerFactory.Start(stopCh)
	go crdInformerFactory.Start(stopCh)

	klog.Info("starting application controller goroutines")
	if err := flController.Start(*workerNum); err != nil {
		klog.Fatal(err)
	}

	select {
	case <-signalCh:
		close(stopCh)
	case <-stopCh:
	}

	klog.Info("shutting down the fedlearner kubernetes operator")
	flController.Stop()
}
<commit_msg>feat(operator): refactor config and add qps/burst args (#606)<commit_after>/* Copyright 2020 The FedLearner Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"context"
	"flag"
	"fmt"
	"net/http"
	"net/http/pprof"
	"os"
	"os/signal"
	"syscall"
	"time"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/informers"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	"k8s.io/klog"

	"github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/apis/fedlearner.k8s.io/v1alpha1"
	crdclientset "github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/client/clientset/versioned"
	crdinformers "github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/client/informers/externalversions"
	"github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/operator"
	"github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/server"
)

type config struct {
	master                      string
	kubeConfig                  string
	port                        string
	debugPort                   string
	workerNum                   int
	resyncInterval              int
	namespace                   string
	qps                         float64
	burst                       int
	ingressExtraHostSuffix      string
	ingressSecretName           string
	ingressEnableClientAuth     bool
	ingressClientAuthSecretName string
	enableLeaderElection        bool
	leaderElectionLockNamespace string
	leaderElectionLockName      string
	leaderElectionLeaseDuration time.Duration
	leaderElectionRenewDeadline time.Duration
	leaderElectionRetryPeriod   time.Duration
	grpcClientTimeout           time.Duration
}

func newConfig() *config {
	config := config{}
	config.port = "8080"
	config.debugPort = "8081"
	config.workerNum = 10
	config.resyncInterval = 30
	config.namespace = "default"
	config.qps = 20.0
	config.burst = 30
	config.ingressEnableClientAuth = true
	config.leaderElectionLockNamespace = "fedlearner-system"
	config.leaderElectionLockName = "fedlearner-kubernetes-operator-lock"
	config.leaderElectionLeaseDuration = 15 * time.Second
	config.leaderElectionRenewDeadline = 5 * time.Second
	config.leaderElectionRetryPeriod = 4 * time.Second
	config.grpcClientTimeout = 15 * time.Second
	return &config
}

func initFlags() *config {
	fl := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	config := newConfig()
	fl.StringVar(&config.master, "master", config.master, "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
	fl.StringVar(&config.kubeConfig, "kube-config", config.kubeConfig, "Path to a kube config. Only required if out-of-cluster.")
	fl.StringVar(&config.port, "port", config.port, "The http port controller listening.")
	fl.StringVar(&config.debugPort, "debug-port", config.debugPort, "The debug http port controller listening.")
	fl.IntVar(&config.workerNum, "worker-num", config.workerNum, "Number of worker threads used by the fedlearner controller.")
	fl.IntVar(&config.resyncInterval, "resync-interval", config.resyncInterval, "Informer resync interval in seconds.")
	fl.StringVar(&config.namespace, "namespace", config.namespace, "The namespace to which controller listen FLApps.")
	fl.Float64Var(&config.qps, "qps", config.qps, "The clientset config QPS")
	fl.IntVar(&config.burst, "burst", config.burst, "The clientset config Burst")
	fl.StringVar(&config.ingressExtraHostSuffix, "ingress-extra-host-suffix", config.ingressExtraHostSuffix, "The extra suffix of hosts when creating ingress.")
	fl.StringVar(&config.ingressSecretName, "ingress-secret-name", config.ingressSecretName, "The secret name used for tls, only one secret supported now.")
	fl.BoolVar(&config.ingressEnableClientAuth, "ingress-enabled-client-auth", config.ingressEnableClientAuth, "Whether enable client auth for created ingress.")
	fl.StringVar(&config.ingressClientAuthSecretName, "ingress-client-auth-secret-name", config.ingressClientAuthSecretName, "The secret name used for client auth, only one secret supported now.")
	fl.BoolVar(&config.enableLeaderElection, "leader-election", config.enableLeaderElection, "Enable fedlearner controller leader election.")
	fl.StringVar(&config.leaderElectionLockNamespace, "leader-election-lock-namespace", config.leaderElectionLockNamespace, "Namespace in which to create the Endpoints for leader election.")
	fl.StringVar(&config.leaderElectionLockName, "leader-election-lock-name", config.leaderElectionLockName, "Name of the Endpoint for leader election.")
	fl.DurationVar(&config.leaderElectionLeaseDuration, "leader-election-lease-duration", config.leaderElectionLeaseDuration, "Leader election lease duration.")
	fl.DurationVar(&config.leaderElectionRenewDeadline, "leader-election-renew-deadline", config.leaderElectionRenewDeadline, "Leader election renew deadline.")
	fl.DurationVar(&config.leaderElectionRetryPeriod, "leader-election-retry-period", config.leaderElectionRetryPeriod, "Leader election retry period.")
	fl.DurationVar(&config.grpcClientTimeout, "grpc-client-timeout", config.grpcClientTimeout, "GRPC Client timetout")
	klog.InitFlags(fl)
	fl.Parse(os.Args[1:])
	return config
}

func buildConfig(masterURL string, kubeConfig string, qps float32, burst int) (*rest.Config, error) {
	if kubeConfig != "" {
		return clientcmd.BuildConfigFromFlags(masterURL, kubeConfig)
	}
	config, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}
	config.QPS = qps
	config.Burst = burst
	klog.V(6).Infof("clientset config = %v", config)

	if masterURL != "" {
		config.Host = masterURL
	}
	return config, nil
}

func buildClientset(masterURL string, kubeConfig string, qps float64, burst int) (*clientset.Clientset, *crdclientset.Clientset, error) {
	config, err := buildConfig(masterURL, kubeConfig, float32(qps), burst)
	if err != nil {
		return nil, nil, err
	}

	kubeClient, err := clientset.NewForConfig(config)
	if err != nil {
		return nil, nil, err
	}

	crdClient, err := crdclientset.NewForConfig(config)
	if err != nil {
		return nil, nil, err
	}
	return kubeClient, crdClient, err
}

func startLeaderElection(
	config *config,
	kubeClient *clientset.Clientset,
	recorder record.EventRecorder,
	startCh chan struct{},
	stopCh chan struct{},
) {
	hostName, err := os.Hostname()
	if err != nil {
		klog.Error("failed to get hostname")
		return
	}
	hostName = hostName + "_" + string(uuid.NewUUID())

	resourceLock, err := resourcelock.New(resourcelock.EndpointsResourceLock,
		config.leaderElectionLockNamespace,
		config.leaderElectionLockName,
		kubeClient.CoreV1(),
		nil,
		resourcelock.ResourceLockConfig{
			Identity:      hostName,
			EventRecorder: recorder,
		})
	if err != nil {
		return
	}

	electionCfg := leaderelection.LeaderElectionConfig{
		Lock:          resourceLock,
		LeaseDuration: config.leaderElectionLeaseDuration,
		RenewDeadline: config.leaderElectionRenewDeadline,
		RetryPeriod:   config.leaderElectionRetryPeriod,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: func(c context.Context) {
				close(startCh)
			},
			OnStoppedLeading: func() {
				close(stopCh)
			},
		},
	}
	elector, err := leaderelection.NewLeaderElector(electionCfg)
	if err != nil {
		klog.Fatal(err)
	}

	go elector.Run(context.Background())
}

func main() {
	config := initFlags()

	kubeClient, crdClient, err := buildClientset(config.master, config.kubeConfig, config.qps, config.burst)
	if err != nil {
		klog.Fatalf("failed to build clientset, err = %v", err)
	}
	if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil {
		klog.Fatalf("Failed to add flapp scheme: %v", err)
	}

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
	stopCh := make(chan struct{}, 1)
	startCh := make(chan struct{}, 1)

	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(klog.Infof)
	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{
		Interface: kubeClient.CoreV1().Events(""),
	})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "fedlearner-operator"})

	if config.namespace == metav1.NamespaceAll {
		klog.Fatalf("cluster scoped operator is not supported")
	}
	klog.Infof("scoping operator to namespace %s", config.namespace)

	kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(
		kubeClient,
		time.Duration(config.resyncInterval)*time.Second,
		informers.WithNamespace(config.namespace),
	)
	crdInformerFactory := crdinformers.NewSharedInformerFactoryWithOptions(
		crdClient,
		time.Duration(config.resyncInterval)*time.Second,
		crdinformers.WithNamespace(config.namespace),
	)

	appEventHandler := operator.NewAppEventHandlerWithClientTimeout(config.namespace, crdClient, config.grpcClientTimeout)
	flController := operator.NewFLController(
		config.namespace,
		recorder,
		config.resyncInterval,
		config.ingressExtraHostSuffix,
		config.ingressSecretName,
		config.ingressEnableClientAuth,
		config.ingressClientAuthSecretName,
		kubeClient,
		crdClient,
		kubeInformerFactory,
		crdInformerFactory,
		appEventHandler,
		stopCh)

	go func() {
		klog.Infof("starting adapter listening %v", config.port)
		server.ServeGrpc("0.0.0.0", config.port, appEventHandler)
	}()
	go func() {
		mux := http.NewServeMux()
		mux.HandleFunc("/debug/pprof/", pprof.Index)
		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
		klog.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%s", config.debugPort), mux))
	}()

	if config.enableLeaderElection {
		startLeaderElection(config, kubeClient, recorder, startCh, stopCh)
	}

	klog.Info("starting the fedlearner operator")
	if config.enableLeaderElection {
		klog.Info("waiting to be elected leader before starting application controller goroutines")
		<-startCh
	}

	go kubeInformerFactory.Start(stopCh)
	go crdInformerFactory.Start(stopCh)

	klog.Info("starting application controller goroutines")
	if err := flController.Start(config.workerNum); err != nil {
		klog.Fatal(err)
	}

	select {
	case <-signalCh:
		close(stopCh)
	case <-stopCh:
	}

	klog.Info("shutting down the fedlearner kubernetes operator")
	flController.Stop()
}
<|endoftext|>


<commit_before>package render

import (
	"image"

	"github.com/oakmound/oak/oakerr"
)

// Sheet is a 2D array of image rgbas
type Sheet [][]*image.RGBA

// SubSprite gets a sprite from a sheet at the given location
func (sh *Sheet) SubSprite(x, y int) *Sprite {
	return NewSprite(0, 0, (*sh)[x][y])
}

// ToSprites returns this sheet as a 2D array of Sprites
func (sh *Sheet) ToSprites() [][]*Sprite {
	sprites := make([][]*Sprite, len(*sh))
	for x, row := range *sh {
		sprites[x] = make([]*Sprite, len(row))
		for y := range row {
			sprites[x][y] = sh.SubSprite(x, y)
		}
	}
	return sprites
}

// NewSheetSequence creates a Sequence from a sheet and a list of x,y frame coordinates.
// A sequence will be created by getting the sheet's [i][i+1]th elements incrementally
// from the input frames. If the number of input frames is uneven, an error is returned.
func NewSheetSequence(sheet *Sheet, fps float64, frames ...int) (*Sequence, error) {
	if len(frames)%2 != 0 {
		return nil, oakerr.IndivisibleInput{
			InputName:    "frames",
			IsList:       true,
			MustDivideBy: 2,
		}
	}

	sh := *sheet

	mods := make([]Modifiable, len(frames)/2)
	for i := 0; i < len(frames); i += 2 {
		if len(sh) < frames[i] || len(sh[frames[i]]) < frames[i+1] {
			return nil, oakerr.InvalidInput{InputName: "Frame requested does not exist"}
		}
		mods[i/2] = NewSprite(0, 0, sh[frames[i]][frames[i+1]])
	}
	return NewSequence(fps, mods...), nil
}
<commit_msg>Updating error in sheets to have even better reporting and actually correct condition.<commit_after>package render

import (
	"image"

	"github.com/oakmound/oak/dlog"
	"github.com/oakmound/oak/oakerr"
)

// Sheet is a 2D array of image rgbas
type Sheet [][]*image.RGBA

// SubSprite gets a sprite from a sheet at the given location
func (sh *Sheet) SubSprite(x, y int) *Sprite {
	return NewSprite(0, 0, (*sh)[x][y])
}

// ToSprites returns this sheet as a 2D array of Sprites
func (sh *Sheet) ToSprites() [][]*Sprite {
	sprites := make([][]*Sprite, len(*sh))
	for x, row := range *sh {
		sprites[x] = make([]*Sprite, len(row))
		for y := range row {
			sprites[x][y] = sh.SubSprite(x, y)
		}
	}
	return sprites
}

// NewSheetSequence creates a Sequence from a sheet and a list of x,y frame coordinates.
// A sequence will be created by getting the sheet's [i][i+1]th elements incrementally
// from the input frames. If the number of input frames is uneven, an error is returned.
func NewSheetSequence(sheet *Sheet, fps float64, frames ...int) (*Sequence, error) {
	if len(frames)%2 != 0 {
		return nil, oakerr.IndivisibleInput{
			InputName:    "frames",
			IsList:       true,
			MustDivideBy: 2,
		}
	}

	sh := *sheet

	mods := make([]Modifiable, len(frames)/2)
	for i := 0; i < len(frames); i += 2 {
		if len(sh) <= frames[i] || len(sh[frames[i]]) <= frames[i+1] {
			dlog.Error("Frame ", frames[i], frames[i+1], "requested but sheet has dimensions ", len(sh), len(sh[frames[i]]))
			return nil, oakerr.InvalidInput{InputName: "Frame requested does not exist "}
		}
		mods[i/2] = NewSprite(0, 0, sh[frames[i]][frames[i+1]])
	}
	return NewSequence(fps, mods...), nil
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"strconv"
	"sync"
	"time"

	"github.com/OpenBazaar/openbazaar-go/repo"
)

type ChatDB struct {
	modelStore
}

func NewChatStore(db *sql.DB, lock *sync.Mutex) repo.ChatStore {
	return &ChatDB{modelStore{db, lock}}
}

func (c *ChatDB) Put(messageId string, peerId string, subject string, message string, timestamp time.Time, read bool, outgoing bool) error {
	c.lock.Lock()
	defer c.lock.Unlock()

	// timestamp.UnixNano() is undefined when time has a zero value
	if timestamp.IsZero() {
		log.Warningf("putting chat message (%s): recieved zero timestamp, using current time", messageId)
		timestamp = time.Now()
	}

	tx, err := c.db.Begin()
	if err != nil {
		return err
	}
	stm := `insert into chat(messageID, peerID, subject, message, read, timestamp, outgoing) values(?,?,?,?,?,?,?)`
	stmt, err := tx.Prepare(stm)
	if err != nil {
		return err
	}
	readInt := 0
	if read {
		readInt = 1
	}

	outgoingInt := 0
	if outgoing {
		outgoingInt = 1
	}

	defer stmt.Close()
	_, err = stmt.Exec(
		messageId,
		peerId,
		subject,
		message,
		readInt,
		int(timestamp.UnixNano()),
		outgoingInt,
	)
	if err != nil {
		tx.Rollback()
		return err
	}
	tx.Commit()
	return nil
}

func (c *ChatDB) GetConversations() []repo.ChatConversation {
	c.lock.Lock()
	defer c.lock.Unlock()
	var ret []repo.ChatConversation

	stm := "select distinct peerID from chat where subject='' order by timestamp desc;"
	rows, err := c.db.Query(stm)
	if err != nil {
		return ret
	}
	var ids []string
	for rows.Next() {
		var peerId string
		if err := rows.Scan(&peerId); err != nil {
			continue
		}
		ids = append(ids, peerId)

	}
	defer rows.Close()
	for _, peerId := range ids {
		var (
			count  int
			m      string
			ts     int64
			outInt int
			stm    = "select Count(*) from chat where peerID='" + peerId + "' and read=0 and subject='' and outgoing=0;"
			row    = c.db.QueryRow(stm)
		)
		row.Scan(&count)
		stm = "select max(timestamp), message, outgoing from chat where peerID='" + peerId + "' and subject=''"
		row = c.db.QueryRow(stm)
		row.Scan(&ts, &m, &outInt)
		outgoing := false
		if outInt > 0 {
			outgoing = true
		}
		timestamp := repo.NewAPITime(time.Unix(0, ts))
		convo := repo.ChatConversation{
			PeerId:    peerId,
			Unread:    count,
			Last:      m,
			Timestamp: timestamp,
			Outgoing:  outgoing,
		}
		ret = append(ret, convo)
	}
	return ret
}

func (c *ChatDB) GetMessages(peerID string, subject string, offsetId string, limit int) []repo.ChatMessage {
	c.lock.Lock()
	defer c.lock.Unlock()
	var ret []repo.ChatMessage

	var peerStm string
	if peerID != "" {
		peerStm = " and peerID='" + peerID + "'"
	}

	var stm string
	if offsetId != "" {
		stm = "select messageID, peerID, message, read, timestamp, outgoing from chat where subject='" + subject + "'" + peerStm + " and timestamp<(select timestamp from chat where messageID='" + offsetId + "') order by timestamp desc limit " + strconv.Itoa(limit) + " ;"
	} else {
		stm = "select messageID, peerID, message, read, timestamp, outgoing from chat where subject='" + subject + "'" + peerStm + " order by timestamp desc limit " + strconv.Itoa(limit) + ";"
	}
	rows, err := c.db.Query(stm)
	if err != nil {
		log.Error(err)
		return ret
	}
	for rows.Next() {
		var (
			msgID        string
			pid          string
			message      string
			readInt      int
			timestampInt int64
			outgoingInt  int
		)
		if err := rows.Scan(&msgID, &pid, &message, &readInt, &timestampInt, &outgoingInt); err != nil {
			continue
		}
		var read bool
		if readInt == 1 {
			read = true
		}
		var outgoing bool
		if outgoingInt == 1 {
			outgoing = true
		}
		timestamp := repo.NewAPITime(time.Unix(0, timestampInt))
		chatMessage := repo.ChatMessage{
			PeerId:    pid,
			MessageId: msgID,
			Subject:   subject,
			Message:   message,
			Read:      read,
			Timestamp: timestamp,
			Outgoing:  outgoing,
		}
		ret = append(ret, chatMessage)
	}
	return ret
}

func (c *ChatDB) MarkAsRead(peerID string, subject string, outgoing bool, messageId string) (string, bool, error) {
	c.lock.Lock()
	defer c.lock.Unlock()
	updated := false
	outgoingInt := 0
	if outgoing {
		outgoingInt = 1
	}
	var stmt *sql.Stmt
	var tx *sql.Tx
	var err error
	if messageId != "" {
		stm := "select messageID from chat where peerID=? and subject=? and outgoing=? and read=0 and timestamp<=(select timestamp from chat where messageID=?) limit 1"
		rows, err := c.db.Query(stm, peerID, subject, outgoingInt, messageId)
		if err != nil {
			return "", updated, err
		}
		if rows.Next() {
			updated = true
		}
		rows.Close()
		tx, err = c.db.Begin()
		if err != nil {
			return "", updated, err
		}
		stmt, err = tx.Prepare("update chat set read=1 where peerID=? and subject=? and outgoing=? and timestamp<=(select timestamp from chat where messageID=?)")
		if err != nil {
			return "", updated, err
		}
		_, err = stmt.Exec(peerID, subject, outgoingInt, messageId)
		if err != nil {
			return "", updated, err
		}
	} else {
		var peerStm string
		if peerID != "" {
			peerStm = " and peerID=?"
		}

		stm := "select messageID from chat where subject=?" + peerStm + " and outgoing=? and read=0 limit 1"
		var rows *sql.Rows
		var err error
		if peerID != "" {
			rows, err = c.db.Query(stm, subject, peerID, outgoingInt)
		} else {
			rows, err = c.db.Query(stm, subject, outgoingInt)
		}
		if err != nil {
			return "", updated, err
		}
		if rows.Next() {
			updated = true
		}
		rows.Close()
		tx, err = c.db.Begin()
		if err != nil {
			return "", updated, err
		}
		stmt, err = tx.Prepare("update chat set read=1 where subject=?" + peerStm + " and outgoing=?")
		if err != nil {
			return "", updated, err
		}
		if peerID != "" {
			_, err = stmt.Exec(subject, peerID, outgoingInt)
		} else {
			_, err = stmt.Exec(subject, outgoingInt)
		}
		if err != nil {
			return "", updated, err
		}
	}
	defer stmt.Close()
	if err != nil {
		tx.Rollback()
		return "", updated, err
	}
	tx.Commit()

	var peerStm string

	if peerID != "" {
		peerStm = " and peerID=?"
	}
	stmt2, err := c.db.Prepare("select max(timestamp), messageID from chat where subject=?" + peerStm + " and outgoing=?")
	if err != nil {
		return "", updated, err
	}
	defer stmt2.Close()
	var (
		timestamp sql.NullInt64
		msgId     sql.NullString
	)
	if peerID != "" {
		err = stmt2.QueryRow(subject, peerID, outgoingInt).Scan(&timestamp, &msgId)
	} else {
		err = stmt2.QueryRow(subject, outgoingInt).Scan(&timestamp, &msgId)
	}
	if err != nil {
		return "", updated, err
	}
	return msgId.String, updated, nil
}

func (c *ChatDB) GetUnreadCount(subject string) (int, error) {
	stm := "select Count(*) from chat where read=0 and subject=? and outgoing=0;"
	row := c.db.QueryRow(stm, subject)
	var count int
	err := row.Scan(&count)
	if err != nil {
		return 0, err
	}
	return count, nil
}

func (c *ChatDB) DeleteMessage(msgID string) error {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.db.Exec("delete from chat where messageID=?", msgID)
	return nil
}

func (c *ChatDB) DeleteConversation(peerId string) error {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.db.Exec("delete from chat where peerId=? and subject=''", peerId)
	return nil
}
<commit_msg>[#1169] Make transaction and error handling self-contained in repo/db/chat<commit_after>package db

import (
	"database/sql"
	"fmt"
	"strconv"
	"sync"
	"time"

	"github.com/OpenBazaar/openbazaar-go/repo"
)

type ChatDB struct {
	modelStore
}

func NewChatStore(db *sql.DB, lock *sync.Mutex) repo.ChatStore {
	return &ChatDB{modelStore{db, lock}}
}

func (c *ChatDB) Put(messageId string, peerId string, subject string, message string, timestamp time.Time, read bool, outgoing bool) error {
	c.lock.Lock()
	defer c.lock.Unlock()

	// timestamp.UnixNano() is undefined when time has a zero value
	if timestamp.IsZero() {
		log.Warningf("putting chat message (%s): recieved zero timestamp, using current time", messageId)
		timestamp = time.Now()
	}

	tx, err := c.db.Begin()
	if err != nil {
		return err
	}
	stm := `insert into chat(messageID, peerID, subject, message, read, timestamp, outgoing) values(?,?,?,?,?,?,?)`
	stmt, err := tx.Prepare(stm)
	if err != nil {
		return err
	}
	readInt := 0
	if read {
		readInt = 1
	}

	outgoingInt := 0
	if outgoing {
		outgoingInt = 1
	}

	defer stmt.Close()
	_, err = stmt.Exec(
		messageId,
		peerId,
		subject,
		message,
		readInt,
		int(timestamp.UnixNano()),
		outgoingInt,
	)
	if err != nil {
		tx.Rollback()
		return err
	}
	tx.Commit()
	return nil
}

func (c *ChatDB) GetConversations() []repo.ChatConversation {
	c.lock.Lock()
	defer c.lock.Unlock()
	var ret []repo.ChatConversation

	stm := "select distinct peerID from chat where subject='' order by timestamp desc;"
	rows, err := c.db.Query(stm)
	if err != nil {
		return ret
	}
	var ids []string
	for rows.Next() {
		var peerId string
		if err := rows.Scan(&peerId); err != nil {
			continue
		}
		ids = append(ids, peerId)

	}
	defer rows.Close()
	for _, peerId := range ids {
		var (
			count  int
			m      string
			ts     int64
			outInt int
			stm    = "select Count(*) from chat where peerID='" + peerId + "' and read=0 and subject='' and outgoing=0;"
			row    = c.db.QueryRow(stm)
		)
		row.Scan(&count)
		stm = "select max(timestamp), message, outgoing from chat where peerID='" + peerId + "' and subject=''"
		row = c.db.QueryRow(stm)
		row.Scan(&ts, &m, &outInt)
		outgoing := false
		if outInt > 0 {
			outgoing = true
		}
		timestamp := repo.NewAPITime(time.Unix(0, ts))
		convo := repo.ChatConversation{
			PeerId:    peerId,
			Unread:    count,
			Last:      m,
			Timestamp: timestamp,
			Outgoing:  outgoing,
		}
		ret = append(ret, convo)
	}
	return ret
}

func (c *ChatDB) GetMessages(peerID string, subject string, offsetId string, limit int) []repo.ChatMessage {
	c.lock.Lock()
	defer c.lock.Unlock()
	var ret []repo.ChatMessage

	var peerStm string
	if peerID != "" {
		peerStm = " and peerID='" + peerID + "'"
	}

	var stm string
	if offsetId != "" {
		stm = "select messageID, peerID, message, read, timestamp, outgoing from chat where subject='" + subject + "'" + peerStm + " and timestamp<(select timestamp from chat where messageID='" + offsetId + "') order by timestamp desc limit " + strconv.Itoa(limit) + " ;"
	} else {
		stm = "select messageID, peerID, message, read, timestamp, outgoing from chat where subject='" + subject + "'" + peerStm + " order by timestamp desc limit " + strconv.Itoa(limit) + ";"
	}
	rows, err := c.db.Query(stm)
	if err != nil {
		log.Error(err)
		return ret
	}
	for rows.Next() {
		var (
			msgID        string
			pid          string
			message      string
			readInt      int
			timestampInt int64
			outgoingInt  int
		)
		if err := rows.Scan(&msgID, &pid, &message, &readInt, &timestampInt, &outgoingInt); err != nil {
			continue
		}
		var read bool
		if readInt == 1 {
			read = true
		}
		var outgoing bool
		if outgoingInt == 1 {
			outgoing = true
		}
		timestamp := repo.NewAPITime(time.Unix(0, timestampInt))
		chatMessage := repo.ChatMessage{
			PeerId:    pid,
			MessageId: msgID,
			Subject:   subject,
			Message:   message,
			Read:      read,
			Timestamp: timestamp,
			Outgoing:  outgoing,
		}
		ret = append(ret, chatMessage)
	}
	return ret
}

func (c *ChatDB) MarkAsRead(peerID string, subject string, outgoing bool, messageId string) (string, bool, error) {
	c.lock.Lock()
	defer c.lock.Unlock()
	updated := false
	outgoingInt := 0
	if outgoing {
		outgoingInt = 1
	}
	var err error
	if messageId != "" {
		stm := "select messageID from chat where peerID=? and subject=? and outgoing=? and read=0 and timestamp<=(select timestamp from chat where messageID=?) limit 1"
		rows, err := c.db.Query(stm, peerID, subject, outgoingInt, messageId)
		if err != nil {
			return "", updated, err
		}
		if rows.Next() {
			updated = true
		}
		rows.Close()
		var tx *sql.Tx
		tx, err = c.db.Begin()
		if err != nil {
			if rErr := tx.Rollback(); rErr != nil {
				return "", updated, fmt.Errorf("create tx: %s (w rollback error: %s)", err, rErr)
			}
			return "", updated, fmt.Errorf("create tx: %s", err)
		}
		var stmt *sql.Stmt
		stmt, err = tx.Prepare("update chat set read=1 where peerID=? and subject=? and outgoing=? and timestamp<=(select timestamp from chat where messageID=?)")
		if err != nil {
			if rErr := tx.Rollback(); rErr != nil {
				return "", updated, fmt.Errorf("prepare chat update: %s (w rollback error: %s)", err, rErr)
			}
			return "", updated, fmt.Errorf("prepare chat update: %s", err)
		}
		_, err = stmt.Exec(peerID, subject, outgoingInt, messageId)
		if err != nil {
			if rErr := tx.Rollback(); rErr != nil {
				return "", updated, fmt.Errorf("update chat record: %s (w rollback error: %s)", err, rErr)
			}
			return "", updated, fmt.Errorf("update chat record: %s", err)
		}
		if err := tx.Commit(); err != nil {
			return "", updated, fmt.Errorf("commit tx: %s", err)
		}
	} else {
		var peerStm string
		if peerID != "" {
			peerStm = " and peerID=?"
		}

		stm := "select messageID from chat where subject=?" + peerStm + " and outgoing=? and read=0 limit 1"
		var rows *sql.Rows
		var err error
		if peerID != "" {
			rows, err = c.db.Query(stm, subject, peerID, outgoingInt)
		} else {
			rows, err = c.db.Query(stm, subject, outgoingInt)
		}
		if err != nil {
			return "", updated, err
		}
		if rows.Next() {
			updated = true
		}
		rows.Close()
		var tx *sql.Tx
		tx, err = c.db.Begin()
		if err != nil {
			if rErr := tx.Rollback(); rErr != nil {
				return "", updated, fmt.Errorf("create tx: %s (w rollback error: %s)", err, rErr)
			}
			return "", updated, fmt.Errorf("create tx: %s", err)
		}
		var stmt *sql.Stmt
		stmt, err = tx.Prepare("update chat set read=1 where subject=?" + peerStm + " and outgoing=?")
		if err != nil {
			if rErr := tx.Rollback(); rErr != nil {
				return "", updated, fmt.Errorf("prepare chat update: %s (w rollback error: %s)", err, rErr)
			}
			return "", updated, fmt.Errorf("prepare chat update: %s", err)
		}
		if peerID != "" {
			_, err = stmt.Exec(subject, peerID, outgoingInt)
		} else {
			_, err = stmt.Exec(subject, outgoingInt)
		}
		if err != nil {
			if rErr := tx.Rollback(); rErr != nil {
				return "", updated, fmt.Errorf("update chat record: %s (w rollback error: %s)", err, rErr)
			}
			return "", updated, fmt.Errorf("update chat record: %s", err)
		}
		if err := tx.Commit(); err != nil {
			return "", updated, fmt.Errorf("commit tx: %s", err)
		}
	}
	var peerStm string

	if peerID != "" {
		peerStm = " and peerID=?"
	}
	stmt2, err := c.db.Prepare("select max(timestamp), messageID from chat where subject=?" + peerStm + " and outgoing=?")
	if err != nil {
		return "", updated, err
	}
	defer stmt2.Close()
	var (
		timestamp sql.NullInt64
		msgId     sql.NullString
	)
	if peerID != "" {
		err = stmt2.QueryRow(subject, peerID, outgoingInt).Scan(&timestamp, &msgId)
	} else {
		err = stmt2.QueryRow(subject, outgoingInt).Scan(&timestamp, &msgId)
	}
	if err != nil {
		return "", updated, err
	}
	return msgId.String, updated, nil
}

func (c *ChatDB) GetUnreadCount(subject string) (int, error) {
	stm := "select Count(*) from chat where read=0 and subject=? and outgoing=0;"
	row := c.db.QueryRow(stm, subject)
	var count int
	err := row.Scan(&count)
	if err != nil {
		return 0, err
	}
	return count, nil
}

func (c *ChatDB) DeleteMessage(msgID string) error {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.db.Exec("delete from chat where messageID=?", msgID)
	return nil
}

func (c *ChatDB) DeleteConversation(peerId string) error {
	c.lock.Lock()
	defer c.lock.Unlock()
	c.db.Exec("delete from chat where peerId=? and subject=''", peerId)
	return nil
}
<|endoftext|>


<commit_before>package ratelimit

import (
	"io/ioutil"
	"net/http"
	"testing"
)

func TestGetRemoteIP(t *testing.T) {

	tests := []struct {
		input    map[string]string
		expected string
	}{
		{
			map[string]string{
				"X-Forwarded-For": "192.168.1.1, 192.168.1.2",
			},
			"192.168.1.1",
		},
		{
			map[string]string{
				"Real-Ip": "192.168.1.1",
			},
			"192.168.1.1",
		},
		{
			map[string]string{
				"X-Forwarded-For": "192.168.1.2,192.168.1.1",
				"Real-Ip":         "192.168.1.2",
			},
			"192.169.1.2",
		},
		{
			map[string]string{
				"X-Forwarded-For": "",
			},
			"127.0.0.1",
		},
		{
			make(map[string]string),
			"127.0.0.1",
		},
	}

	defaultClient := &http.Client{}
	var (
		req  *http.Request
		resp *http.Response
		err  error
	)
	for i, test := range tests {
		req, err = http.NewRequest("GET", server.URL, nil)
		if err != nil {
			t.Fatalf("Test %d errored: '%v'", i, err)
		}
		for k, v := range test.input {
			req.Header.Add(k, v)
		}
		resp, err = defaultClient.Do(req)
		if err != nil {
			t.Errorf("Test %d errored: '%v'", i, err)
		}
		ip, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Errorf("Test %d errored: '%v'", i, err)
		}
		if string(ip) != test.expected {
			t.Errorf("Test %d Expected %s, get %s", i, test.expected, ip)
		}
		resp.Body.Close()
	}
}
<commit_msg>fix typo in test :neutral_face:<commit_after>package ratelimit

import (
	"io/ioutil"
	"net/http"
	"testing"
)

func TestGetRemoteIP(t *testing.T) {

	tests := []struct {
		input    map[string]string
		expected string
	}{
		{
			map[string]string{
				"X-Forwarded-For": "192.168.1.1, 192.168.1.2",
			},
			"192.168.1.1",
		},
		{
			map[string]string{
				"X-Real-Ip": "192.168.1.1",
			},
			"192.168.1.1",
		},
		{
			map[string]string{
				"X-Forwarded-For": "192.168.1.2,192.168.1.1",
				"X-Real-Ip":       "192.168.1.2",
			},
			"192.168.1.2",
		},
		{
			map[string]string{
				"X-Forwarded-For": "",
			},
			"127.0.0.1",
		},
		{
			make(map[string]string),
			"127.0.0.1",
		},
	}

	defaultClient := &http.Client{}
	var (
		req  *http.Request
		resp *http.Response
		err  error
	)
	for i, test := range tests {
		req, err = http.NewRequest("GET", server.URL, nil)
		if err != nil {
			t.Fatalf("Test %d errored: '%v'", i, err)
		}
		for k, v := range test.input {
			req.Header.Add(k, v)
		}
		resp, err = defaultClient.Do(req)
		if err != nil {
			t.Errorf("Test %d errored: '%v'", i, err)
		}
		ip, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Errorf("Test %d errored: '%v'", i, err)
		}
		if string(ip) != test.expected {
			t.Errorf("Test %d Expected %s, get %s", i, test.expected, ip)
		}
		resp.Body.Close()
	}
}
<|endoftext|>