<commit_before>// Copyright 2016 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/gorilla/websocket"

	"github.com/la5nta/pat/internal/osutil"
	"github.com/la5nta/wl2k-go/mailbox"
)

const KeepaliveInterval = 4 * time.Minute

// WSConn represent one connection in the WSHub pool
type WSConn struct {
	conn *websocket.Conn
	out  chan interface{}
}

// WSHub is a hub for broadcasting data to several websocket connections
type WSHub struct {
	mu   sync.Mutex
	pool map[*WSConn]struct{}
}

func NewWSHub() *WSHub {
	w := &WSHub{pool: map[*WSConn]struct{}{}}
	go w.watchMBox()
	return w
}

func (w *WSHub) UpdateStatus()            { w.WriteJSON(struct{ Status Status }{getStatus()}) }
func (w *WSHub) WriteProgress(p Progress) { w.WriteJSON(struct{ Progress Progress }{p}) }
func (w *WSHub) WriteNotification(n Notification) {
	w.WriteJSON(struct{ Notification Notification }{n})
}

func (w *WSHub) Prompt(p Prompt) {
	w.WriteJSON(struct{ Prompt Prompt }{p})
	go func() { <-p.cancel; w.WriteJSON(struct{ PromptAbort Prompt }{p}) }()
}

func (w *WSHub) WriteJSON(v interface{}) {
	if w == nil {
		return
	}

	w.mu.Lock()
	for c := range w.pool {
		select {
		case c.out <- v:
		case <-time.After(3 * time.Second):
			log.Println("Closing one unresponsive web socket")
			c.conn.Close()
			delete(w.pool, c)
		}
	}
	w.mu.Unlock()
}

func (w *WSHub) ClientAddrs() []string {
	if w == nil {
		return nil
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	addrs := make([]string, 0, len(w.pool))
	for c := range w.pool {
		addrs = append(addrs, c.conn.RemoteAddr().String())
	}
	return addrs
}

func (w *WSHub) watchMBox() {
	// Maximise ulimit -n:
	//   fsnotify opens a file descriptor for every file in the directories it watches, which
	//   may more files than the current soft limit. The is especially a problem on macOS which
	//   has a default soft limit of only 256 files. Windows does not have a such a limit.
	if runtime.GOOS != "windows" {
		if err := osutil.RaiseOpenFileLimit(4096); err != nil {
			log.Printf("Unable to raise open file limit: %v", err)
		}
	}

	fsWatcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Println("Unable to start fs watcher: ", err)
	} else {
		p := path.Join(mbox.MBoxPath, mailbox.DIR_INBOX)
		if err := fsWatcher.Add(p); err != nil {
			log.Printf("Unable to add path '%s' to fs watcher: %s", p, err)
		}

		// These will probably fail if the first failed, but it's not important to log all.
		fsWatcher.Add(path.Join(mbox.MBoxPath, mailbox.DIR_OUTBOX))
		fsWatcher.Add(path.Join(mbox.MBoxPath, mailbox.DIR_SENT))
		fsWatcher.Add(path.Join(mbox.MBoxPath, mailbox.DIR_ARCHIVE))
		defer fsWatcher.Close()
	}

	for {
		select {
		// Filesystem events
		case <-fsWatcher.Events:
			drainEvents(fsWatcher)
			websocketHub.WriteJSON(struct {
				UpdateMailbox bool
			}{true})
		case err := <-fsWatcher.Errors:
			log.Println(err)
		}
	}
}

// Handle adds a new websocket to the hub
//
// It will block until the client either stops responding or closes the connection.
func (w *WSHub) Handle(conn *websocket.Conn) {
	c := &WSConn{
		conn: conn,
		out:  make(chan interface{}, 1),
	}

	w.mu.Lock()
	w.pool[c] = struct{}{}
	w.mu.Unlock()

	// Initial status update
	// (broadcasted as it includes info to other clients about this new one)
	w.UpdateStatus()

	quit := wsReadLoop(conn)

	lines, done, err := tailFile(fOptions.LogPath)
	if err != nil {
		log.Println(err)
		return
	}
	defer close(done)

	for {
		select {
		case <-time.After(KeepaliveInterval):
			c.conn.WriteJSON(struct {
				Ping bool
			}{true})
		case line := <-lines:
			c.conn.WriteJSON(struct {
				LogLine string
			}{string(line)})
		case v := <-c.out:
			err := c.conn.WriteJSON(v)
			if err != nil {
				log.Println(err)
			}
		case <-quit:
			// The read loop failed/disconnected. Remove from hub.
			c.conn.Close()
			w.mu.Lock()
			delete(w.pool, c)
			w.mu.Unlock()
			w.UpdateStatus()
			return
		}
	}
}

func drainEvents(w *fsnotify.Watcher) {
	for {
		select {
		case <-w.Events:
		default:
			return
		}
	}
}

// Expects the file to never get renamed/truncated or deleted
func tailFile(path string) (<-chan []byte, chan<- struct{}, error) {
	lines := make(chan []byte)
	done := make(chan struct{})
	file, err := os.Open(path)
	if err != nil {
		return nil, nil, err
	}

	go func() {
		rd := bufio.NewReader(file)
		for {
			data, _, err := rd.ReadLine()
			if errors.Is(err, io.EOF) {
				time.Sleep(time.Millisecond * 100)
				continue
			}

			select {
			case <-done:
				file.Close()
				return
			case lines <- data:
			}
		}
	}()

	return lines, done, nil
}

func handleWSMessage(v map[string]json.RawMessage) {
	raw, ok := v["prompt_response"]
	if !ok {
		return
	}
	var resp PromptResponse
	json.Unmarshal(raw, &resp)
	promptHub.Respond(resp.ID, resp.Value, resp.Err)
}

func wsReadLoop(c *websocket.Conn) <-chan struct{} {
	quit := make(chan struct{})
	go func() {
		for {
			v := map[string]json.RawMessage{}
			err := c.ReadJSON(&v)
			if err != nil {
				close(quit)
				return
			}
			if _, ok := v["Pong"]; ok {
				// For Keepalive only. Could use a delayed ping response to detect dead connections in the future.
				continue
			}
			go handleWSMessage(v)
		}
	}()
	return quit
}
<commit_msg>Improve mailbox fsnotify handling<commit_after>// Copyright 2016 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/gorilla/websocket"

	"github.com/la5nta/pat/internal/osutil"
	"github.com/la5nta/wl2k-go/mailbox"
)

const KeepaliveInterval = 4 * time.Minute

// WSConn represent one connection in the WSHub pool
type WSConn struct {
	conn *websocket.Conn
	out  chan interface{}
}

// WSHub is a hub for broadcasting data to several websocket connections
type WSHub struct {
	mu   sync.Mutex
	pool map[*WSConn]struct{}
}

func NewWSHub() *WSHub {
	w := &WSHub{pool: map[*WSConn]struct{}{}}
	go w.watchMBox()
	return w
}

func (w *WSHub) UpdateStatus()            { w.WriteJSON(struct{ Status Status }{getStatus()}) }
func (w *WSHub) WriteProgress(p Progress) { w.WriteJSON(struct{ Progress Progress }{p}) }
func (w *WSHub) WriteNotification(n Notification) {
	w.WriteJSON(struct{ Notification Notification }{n})
}

func (w *WSHub) Prompt(p Prompt) {
	w.WriteJSON(struct{ Prompt Prompt }{p})
	go func() { <-p.cancel; w.WriteJSON(struct{ PromptAbort Prompt }{p}) }()
}

func (w *WSHub) WriteJSON(v interface{}) {
	if w == nil {
		return
	}

	w.mu.Lock()
	for c := range w.pool {
		select {
		case c.out <- v:
		case <-time.After(3 * time.Second):
			log.Println("Closing one unresponsive web socket")
			c.conn.Close()
			delete(w.pool, c)
		}
	}
	w.mu.Unlock()
}

func (w *WSHub) ClientAddrs() []string {
	if w == nil {
		return nil
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	addrs := make([]string, 0, len(w.pool))
	for c := range w.pool {
		addrs = append(addrs, c.conn.RemoteAddr().String())
	}
	return addrs
}

func (w *WSHub) watchMBox() {
	// Maximise ulimit -n:
	//   fsnotify opens a file descriptor for every file in the directories it watches, which
	//   may more files than the current soft limit. The is especially a problem on macOS which
	//   has a default soft limit of only 256 files. Windows does not have a such a limit.
	if runtime.GOOS != "windows" {
		if err := osutil.RaiseOpenFileLimit(4096); err != nil {
			log.Printf("Unable to raise open file limit: %v", err)
		}
	}

	fsWatcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Println("Unable to start fs watcher: ", err)
	} else {
		p := path.Join(mbox.MBoxPath, mailbox.DIR_INBOX)
		if err := fsWatcher.Add(p); err != nil {
			log.Printf("Unable to add path '%s' to fs watcher: %s", p, err)
		}

		// These will probably fail if the first failed, but it's not important to log all.
		fsWatcher.Add(path.Join(mbox.MBoxPath, mailbox.DIR_OUTBOX))
		fsWatcher.Add(path.Join(mbox.MBoxPath, mailbox.DIR_SENT))
		fsWatcher.Add(path.Join(mbox.MBoxPath, mailbox.DIR_ARCHIVE))
		defer fsWatcher.Close()
	}

	for {
		select {
		// Filesystem events
		case e := <-fsWatcher.Events:
			if e.Op == fsnotify.Chmod {
				continue
			}
			// Make sure we don't send many of these events over a short period.
			drainUntilSilence(fsWatcher, 100*time.Millisecond)
			websocketHub.WriteJSON(struct {
				UpdateMailbox bool
			}{true})
		case err := <-fsWatcher.Errors:
			log.Println(err)
		}
	}
}

// Handle adds a new websocket to the hub
//
// It will block until the client either stops responding or closes the connection.
func (w *WSHub) Handle(conn *websocket.Conn) {
	c := &WSConn{
		conn: conn,
		out:  make(chan interface{}, 1),
	}

	w.mu.Lock()
	w.pool[c] = struct{}{}
	w.mu.Unlock()

	// Initial status update
	// (broadcasted as it includes info to other clients about this new one)
	w.UpdateStatus()

	quit := wsReadLoop(conn)

	lines, done, err := tailFile(fOptions.LogPath)
	if err != nil {
		log.Println(err)
		return
	}
	defer close(done)

	for {
		select {
		case <-time.After(KeepaliveInterval):
			c.conn.WriteJSON(struct {
				Ping bool
			}{true})
		case line := <-lines:
			c.conn.WriteJSON(struct {
				LogLine string
			}{string(line)})
		case v := <-c.out:
			err := c.conn.WriteJSON(v)
			if err != nil {
				log.Println(err)
			}
		case <-quit:
			// The read loop failed/disconnected. Remove from hub.
			c.conn.Close()
			w.mu.Lock()
			delete(w.pool, c)
			w.mu.Unlock()
			w.UpdateStatus()
			return
		}
	}
}

// drainEvents reads from w.Events and blocks until the channel has been silent for at least 50 ms.
func drainUntilSilence(w *fsnotify.Watcher, silenceDur time.Duration) {
	timer := time.NewTimer(silenceDur)
	defer timer.Stop()
	for {
		select {
		case <-w.Events:
			if !timer.Stop() {
				<-timer.C
			}
			timer.Reset(silenceDur)
		case <-timer.C:
			return
		}
	}
}

// Expects the file to never get renamed/truncated or deleted
func tailFile(path string) (<-chan []byte, chan<- struct{}, error) {
	lines := make(chan []byte)
	done := make(chan struct{})
	file, err := os.Open(path)
	if err != nil {
		return nil, nil, err
	}

	go func() {
		rd := bufio.NewReader(file)
		for {
			data, _, err := rd.ReadLine()
			if errors.Is(err, io.EOF) {
				time.Sleep(time.Millisecond * 100)
				continue
			}

			select {
			case <-done:
				file.Close()
				return
			case lines <- data:
			}
		}
	}()

	return lines, done, nil
}

func handleWSMessage(v map[string]json.RawMessage) {
	raw, ok := v["prompt_response"]
	if !ok {
		return
	}
	var resp PromptResponse
	json.Unmarshal(raw, &resp)
	promptHub.Respond(resp.ID, resp.Value, resp.Err)
}

func wsReadLoop(c *websocket.Conn) <-chan struct{} {
	quit := make(chan struct{})
	go func() {
		for {
			v := map[string]json.RawMessage{}
			err := c.ReadJSON(&v)
			if err != nil {
				close(quit)
				return
			}
			if _, ok := v["Pong"]; ok {
				// For Keepalive only. Could use a delayed ping response to detect dead connections in the future.
				continue
			}
			go handleWSMessage(v)
		}
	}()
	return quit
}
<|endoftext|>


<commit_before>package main

// from time to time see https://godoc.org/golang.org/x/crypto/acme/autocert
// it isn't ready for usage now, but can simple code in future.

import (
	"crypto/rsa"

	cryptorand "crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"errors"
	"github.com/Sirupsen/logrus"
	"github.com/hlandau/acme/acmeapi"
	"github.com/hlandau/acme/acmeapi/acmeutils"
	"context"
	"net"
	"strings"
	"sync"
	"time"
	"encoding/pem"
)

const (
	SNI01_EXPIRE_TOKEN time.Duration = time.Minute * 10
	ACME_DOMAIN_SUFFIX = ".acme.invalid"
)

type acmeStruct struct {
	serverAddress string
	privateKey    *rsa.PrivateKey
	client        *acmeapi.Client

	mutex                *sync.Mutex
	acmeauthDomainsMutex *sync.Mutex
	acmeAuthDomains      map[string]time.Time
}

func (this *acmeStruct) Init() {
	this.client = &acmeapi.Client{
		AccountKey:   this.privateKey,
		DirectoryURL: this.serverAddress,
	}

	this.mutex = &sync.Mutex{}

	this.acmeauthDomainsMutex = &sync.Mutex{}
	this.acmeAuthDomains = make(map[string]time.Time)
	this.CleanupTimer()
}

func (this *acmeStruct) RegisterEnsure(ctx context.Context) (err error) {
	reg := &acmeapi.Registration{}
	for i := 0; i < TRY_COUNT+1; i++ { // +1 count need for request latest agreement uri
		reg.AgreementURI = reg.LatestAgreementURI
		if reg.AgreementURI != "" {
			logrus.Info("Auto agree with terms:", reg.LatestAgreementURI)
		}
		err = this.client.UpsertRegistration(reg, ctx)
		if reg.AgreementURI != "" && err == nil {
			return
		}
	}
	return err
}

func (this *acmeStruct) Cleanup() {
	this.mutex.Lock()
	defer this.mutex.Unlock()

	now := time.Now()
	for token, expire := range this.acmeAuthDomains {
		if expire.Before(now) {
			delete(this.acmeAuthDomains, token)
		}
	}
}

func (this *acmeStruct) CleanupTimer() {
	this.Cleanup()
	time.AfterFunc(SNI01_EXPIRE_TOKEN, this.Cleanup)
}

func (this *acmeStruct) CreateCertificate(domain string) (cert *tls.Certificate, err error) {
	// Check suffix for avoid mutex sync in DeleteAcmeAuthDomain
	if strings.HasSuffix(domain, ACME_DOMAIN_SUFFIX) {
		logrus.Debugf("Detect auth-domain mode for domain '%v'", domain)
		if this.DeleteAcmeAuthDomain(domain) {
			logrus.Debugf("Return self-signed certificate for domain '%v'", domain)
			return this.createCertificateSelfSigned(domain)
		} else {
			logrus.Debugf("Detect auth-domain is not present in list '%v'", domain)
			return nil, errors.New("Now allowed auth-domain")
		}
	}

	// check about we serve the domain
	ips, err := net.LookupIP(domain)
	if err != nil {
		logrus.Warnf("Can't lookup ip for domain '%v': %v", domain, err)
		return nil, errors.New("Can't lookup ip of the domain")
	}
	isLocalIP := false
checkLocalIP:
	for _, ip := range ips {
		for _, localIP := range localIPs {
			if ip.Equal(localIP) {
				isLocalIP = true
				break checkLocalIP
			}
		}
	}
	if !isLocalIP {
		logrus.Warnf("Domain have ip of other server. Domain '%v', Domain ips: %v, Server ips: %v", domain, ips, localIPs)
		return nil, errors.New("Domain have ip of other server.")
	}

	return this.createCertificateAcme(domain)
}

func (this *acmeStruct) createCertificateAcme(domain string) (cert *tls.Certificate, err error) {
	this.mutex.Lock()
	defer this.mutex.Unlock()

	var auth *acmeapi.Authorization

	ctx, cancelFunc := context.WithTimeout(context.Background(), LETSENCRYPT_CREATE_CERTIFICATE_TIMEOUT)
	defer cancelFunc()

	for i := 0; i < TRY_COUNT; i++ {
		auth, err = this.client.NewAuthorization(domain, ctx)
		if err == nil {
			break
		} else {
			logrus.Infof("Can't create new authorization for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Infof("Create authorization for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create new authorization for domain '%v': %v", domain, err)
		return nil, errors.New("Can't create new authorization for domain")
	}

	if logrus.GetLevel() >= logrus.DebugLevel {
		challengeTypes := make([]string, len(auth.Challenges))
		for i := range auth.Challenges {
			challengeTypes[i] = auth.Challenges[i].Type
		}
		logrus.Debugf("Challenge types for domain '%v': %v. Challenge combinations: %v", domain, challengeTypes, auth.Combinations)
	}

	canAuthorize := false
	var challenge *acmeapi.Challenge
	for _, cmb := range auth.Combinations {
		if len(cmb) == 1 && auth.Challenges[cmb[0]].Type == "tls-sni-01" {
			canAuthorize = true
			challenge = auth.Challenges[cmb[0]]
			break
		}
	}
	if !canAuthorize {
		logrus.Errorf("Can't find good challange combination for domain: '%v'", domain)
		return nil, errors.New("Can't find good challange combination")
	}

	acmeHostName, err := acmeutils.TLSSNIHostname(this.privateKey, challenge.Token)
	if err == nil {
		logrus.Debugf("Create acme-auth hostname for domain '%v': %v", domain, acmeHostName)
	} else {
		logrus.Errorf("Can't create acme domain for domain '%v' token '%v': %v", domain, challenge.Token, err)
		return nil, errors.New("Can't create acme domain")
	}
	this.PutAcmeAuthDomain(acmeHostName)

	challengeResponse, err := acmeutils.ChallengeResponseJSON(this.privateKey, challenge.Token, challenge.Type)
	if err == nil {
		logrus.Debugf("Create challenge response for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create challenge response for domain '%v', token '%v', challenge type %v: %v",
			domain, challenge.Token, challenge.Type, err)
		return nil, errors.New("Can't create challenge response")
	}
	for i := 0; i < TRY_COUNT; i++ {
		err = this.client.RespondToChallenge(challenge, challengeResponse, this.privateKey, ctx)
		if err == nil {
			break
		} else {
			logrus.Info("Can't send response for challenge of domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Debugf("Send challenge response for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't send response for challenge of domain '%v': %v", domain, err)
		return nil, errors.New("Can't send response for challenge")
	}

	for i := 0; i < TRY_COUNT; i++ {
		err = this.client.LoadChallenge(challenge, ctx)
		if err != nil {
			logrus.Infof("Can't load challenge for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Debugf("Load challenge for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't load challenge for domain '%v': %v", domain, err)
		return nil, errors.New("Can't load challenge")
	}

	// Generate CSR
	certKey, err := rsa.GenerateKey(cryptorand.Reader, PRIVATE_KEY_BITS)
	if err == nil {
		logrus.Debugf("Create private key for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create rsa key for domain '%v': %v", err)
		return nil, errors.New("Can't create rsa key")
	}
	certRequest := &x509.CertificateRequest{
		SignatureAlgorithm: x509.SHA256WithRSA,
		PublicKeyAlgorithm: x509.RSA,
		PublicKey:          &certKey.PublicKey,
		Subject:            pkix.Name{CommonName: domain},
		DNSNames:           []string{domain},
	}
	csrDER, err := x509.CreateCertificateRequest(cryptorand.Reader, certRequest, certKey)
	if err == nil {
		logrus.Debugf("Create CSR for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create csr for domain '%v': %v", domain, err)
		return nil, errors.New("Can't create csr")
	}

	var certResponse *acmeapi.Certificate
	for i := 0; i < TRY_COUNT; i++ {
		certResponse, err = this.client.RequestCertificate(csrDER, ctx)
		if err != nil {
			logrus.Infof("Can't get certificate for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Infof("Get certificate for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't get certificate for domain '%v': %v", domain, err)
		return nil, errors.New("Can't request certificate")
	}

	pemEncode := func(b []byte, t string) []byte {
		return pem.EncodeToMemory(&pem.Block{Bytes: b, Type: t})
	}
	certPEM := pem.EncodeToMemory(&pem.Block{Bytes: certResponse.Certificate, Type: "CERTIFICATE"})
	for _, extraCert := range certResponse.ExtraCertificates {
		extraCertPEM := pem.EncodeToMemory(&pem.Block{Bytes: extraCert, Type: "CERTIFICATE"})
		certPEM = append(certPEM, '\n')
		certPEM = append(certPEM, extraCertPEM...)
	}
	logrus.Debugf("CERT PEM:\n%s", certPEM)
	certKeyPEM := pemEncode(x509.MarshalPKCS1PrivateKey(certKey), "RSA PRIVATE KEY")

	tmpCert, err := tls.X509KeyPair(certPEM, certKeyPEM)
	logrus.Debugf("Parsed cert count for domain '%v':", len(tmpCert.Certificate))
	if err == nil {
		logrus.Infof("Cert for domain '%v' parsed.", domain)
		cert = &tmpCert
	} else {
		logrus.Errorf("Can't parse cert for domain '%v': %v", domain, err)
		return nil, errors.New("Can't parse cert for domain")
	}

	return cert, nil
}

func (this *acmeStruct) createCertificateSelfSigned(domain string) (cert *tls.Certificate, err error) {
	derCert, privateKey, err := acmeutils.CreateTLSSNICertificate(domain)
	if err != nil {
		logrus.Errorf("Can't create tls-sni-01 self-signed certificate for '%v': %v", domain, err)
		return nil, err
	}

	cert = &tls.Certificate{}
	cert.Certificate = [][]byte{derCert}
	cert.PrivateKey = privateKey
	return cert, nil
}

func (this *acmeStruct) PutAcmeAuthDomain(domain string) {
	this.acmeauthDomainsMutex.Lock()
	defer this.acmeauthDomainsMutex.Unlock()

	logrus.Debug("Put acme auth domain:", domain)
	this.acmeAuthDomains[domain] = time.Now().Add(SNI01_EXPIRE_TOKEN)
}

func (this *acmeStruct) DeleteAcmeAuthDomain(domain string) bool {
	this.acmeauthDomainsMutex.Lock()
	defer this.acmeauthDomainsMutex.Unlock()

	logrus.Debug("Delete acme auth domain:", domain)
	_, ok := this.acmeAuthDomains[domain]
	if ok {
		delete(this.acmeAuthDomains, domain)
	}
	return ok
}
<commit_msg>allow check auth domain many times<commit_after>package main

// from time to time see https://godoc.org/golang.org/x/crypto/acme/autocert
// it isn't ready for usage now, but can simple code in future.

import (
	"crypto/rsa"

	cryptorand "crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"errors"
	"github.com/Sirupsen/logrus"
	"github.com/hlandau/acme/acmeapi"
	"github.com/hlandau/acme/acmeapi/acmeutils"
	"context"
	"net"
	"strings"
	"sync"
	"time"
	"encoding/pem"
)

const (
	SNI01_EXPIRE_TOKEN time.Duration = time.Minute * 10
	ACME_DOMAIN_SUFFIX = ".acme.invalid"
)

type acmeStruct struct {
	serverAddress    string
	privateKey       *rsa.PrivateKey
	client           *acmeapi.Client

	mutex            *sync.Mutex
	authDomainsMutex *sync.Mutex
	authDomains      map[string]time.Time
}

func (this *acmeStruct) authDomainPut(domain string) {
	this.authDomainsMutex.Lock()
	defer this.authDomainsMutex.Unlock()

	logrus.Debug("Put acme auth domain:", domain)
	this.authDomains[domain] = time.Now().Add(SNI01_EXPIRE_TOKEN)
}
func (this *acmeStruct) authDomainCheck(domain string)bool {
	this.authDomainsMutex.Lock()
	defer this.authDomainsMutex.Unlock()

	logrus.Debug("Check acme auth domain:", domain)
	_, ok := this.authDomains[domain]
	return ok
}

func (this *acmeStruct) authDomainDelete(domain string) {
	this.authDomainsMutex.Lock()
	defer this.authDomainsMutex.Unlock()

	logrus.Debug("Delete acme auth domain:", domain)
	delete(this.authDomains, domain)
}

func (this *acmeStruct) Init() {
	this.client = &acmeapi.Client{
		AccountKey:   this.privateKey,
		DirectoryURL: this.serverAddress,
	}

	this.mutex = &sync.Mutex{}

	this.authDomainsMutex = &sync.Mutex{}
	this.authDomains = make(map[string]time.Time)
	this.CleanupTimer()
}

func (this *acmeStruct) RegisterEnsure(ctx context.Context) (err error) {
	reg := &acmeapi.Registration{}
	for i := 0; i < TRY_COUNT+1; i++ { // +1 count need for request latest agreement uri
		reg.AgreementURI = reg.LatestAgreementURI
		if reg.AgreementURI != "" {
			logrus.Info("Auto agree with terms:", reg.LatestAgreementURI)
		}
		err = this.client.UpsertRegistration(reg, ctx)
		if reg.AgreementURI != "" && err == nil {
			return
		}
	}
	return err
}

func (this *acmeStruct) Cleanup() {
	this.mutex.Lock()
	defer this.mutex.Unlock()

	now := time.Now()
	for token, expire := range this.authDomains {
		if expire.Before(now) {
			delete(this.authDomains, token)
		}
	}
}

func (this *acmeStruct) CleanupTimer() {
	this.Cleanup()
	time.AfterFunc(SNI01_EXPIRE_TOKEN, this.Cleanup)
}

func (this *acmeStruct) CreateCertificate(domain string) (cert *tls.Certificate, err error) {
	// Check suffix for avoid mutex sync in DeleteAcmeAuthDomain
	if strings.HasSuffix(domain, ACME_DOMAIN_SUFFIX) {
		logrus.Debugf("Detect auth-domain mode for domain '%v'", domain)
		if this.authDomainCheck(domain) {
			logrus.Debugf("Return self-signed certificate for domain '%v'", domain)
			return this.createCertificateSelfSigned(domain)
		} else {
			logrus.Debugf("Detect auth-domain is not present in list '%v'", domain)
			return nil, errors.New("Now allowed auth-domain")
		}
	}

	// check about we serve the domain
	ips, err := net.LookupIP(domain)
	if err != nil {
		logrus.Warnf("Can't lookup ip for domain '%v': %v", domain, err)
		return nil, errors.New("Can't lookup ip of the domain")
	}
	isLocalIP := false
checkLocalIP:
	for _, ip := range ips {
		for _, localIP := range localIPs {
			if ip.Equal(localIP) {
				isLocalIP = true
				break checkLocalIP
			}
		}
	}
	if !isLocalIP {
		logrus.Warnf("Domain have ip of other server. Domain '%v', Domain ips: %v, Server ips: %v", domain, ips, localIPs)
		return nil, errors.New("Domain have ip of other server.")
	}

	return this.createCertificateAcme(domain)
}

func (this *acmeStruct) createCertificateAcme(domain string) (cert *tls.Certificate, err error) {
	this.mutex.Lock()
	defer this.mutex.Unlock()

	var auth *acmeapi.Authorization

	ctx, cancelFunc := context.WithTimeout(context.Background(), LETSENCRYPT_CREATE_CERTIFICATE_TIMEOUT)
	defer cancelFunc()

	for i := 0; i < TRY_COUNT; i++ {
		auth, err = this.client.NewAuthorization(domain, ctx)
		if err == nil {
			break
		} else {
			logrus.Infof("Can't create new authorization for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Infof("Create authorization for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create new authorization for domain '%v': %v", domain, err)
		return nil, errors.New("Can't create new authorization for domain")
	}

	if logrus.GetLevel() >= logrus.DebugLevel {
		challengeTypes := make([]string, len(auth.Challenges))
		for i := range auth.Challenges {
			challengeTypes[i] = auth.Challenges[i].Type
		}
		logrus.Debugf("Challenge types for domain '%v': %v. Challenge combinations: %v", domain, challengeTypes, auth.Combinations)
	}

	canAuthorize := false
	var challenge *acmeapi.Challenge
	for _, cmb := range auth.Combinations {
		if len(cmb) == 1 && auth.Challenges[cmb[0]].Type == "tls-sni-01" {
			canAuthorize = true
			challenge = auth.Challenges[cmb[0]]
			break
		}
	}
	if !canAuthorize {
		logrus.Errorf("Can't find good challange combination for domain: '%v'", domain)
		return nil, errors.New("Can't find good challange combination")
	}

	acmeHostName, err := acmeutils.TLSSNIHostname(this.privateKey, challenge.Token)
	if err == nil {
		logrus.Debugf("Create acme-auth hostname for domain '%v': %v", domain, acmeHostName)
	} else {
		logrus.Errorf("Can't create acme domain for domain '%v' token '%v': %v", domain, challenge.Token, err)
		return nil, errors.New("Can't create acme domain")
	}
	this.authDomainPut(acmeHostName)
	defer this.authDomainDelete(acmeHostName)

	challengeResponse, err := acmeutils.ChallengeResponseJSON(this.privateKey, challenge.Token, challenge.Type)
	if err == nil {
		logrus.Debugf("Create challenge response for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create challenge response for domain '%v', token '%v', challenge type %v: %v",
			domain, challenge.Token, challenge.Type, err)
		return nil, errors.New("Can't create challenge response")
	}
	for i := 0; i < TRY_COUNT; i++ {
		err = this.client.RespondToChallenge(challenge, challengeResponse, this.privateKey, ctx)
		if err == nil {
			break
		} else {
			logrus.Info("Can't send response for challenge of domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Debugf("Send challenge response for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't send response for challenge of domain '%v': %v", domain, err)
		return nil, errors.New("Can't send response for challenge")
	}

	for i := 0; i < TRY_COUNT; i++ {
		err = this.client.LoadChallenge(challenge, ctx)
		if err != nil {
			logrus.Infof("Can't load challenge for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Debugf("Load challenge for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't load challenge for domain '%v': %v", domain, err)
		return nil, errors.New("Can't load challenge")
	}

	// Generate CSR
	certKey, err := rsa.GenerateKey(cryptorand.Reader, PRIVATE_KEY_BITS)
	if err == nil {
		logrus.Debugf("Create private key for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create rsa key for domain '%v': %v", err)
		return nil, errors.New("Can't create rsa key")
	}
	certRequest := &x509.CertificateRequest{
		SignatureAlgorithm: x509.SHA256WithRSA,
		PublicKeyAlgorithm: x509.RSA,
		PublicKey:          &certKey.PublicKey,
		Subject:            pkix.Name{CommonName: domain},
		DNSNames:           []string{domain},
	}
	csrDER, err := x509.CreateCertificateRequest(cryptorand.Reader, certRequest, certKey)
	if err == nil {
		logrus.Debugf("Create CSR for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create csr for domain '%v': %v", domain, err)
		return nil, errors.New("Can't create csr")
	}

	var certResponse *acmeapi.Certificate
	for i := 0; i < TRY_COUNT; i++ {
		certResponse, err = this.client.RequestCertificate(csrDER, ctx)
		if err != nil {
			logrus.Infof("Can't get certificate for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Infof("Get certificate for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't get certificate for domain '%v': %v", domain, err)
		return nil, errors.New("Can't request certificate")
	}

	pemEncode := func(b []byte, t string) []byte {
		return pem.EncodeToMemory(&pem.Block{Bytes: b, Type: t})
	}
	certPEM := pem.EncodeToMemory(&pem.Block{Bytes: certResponse.Certificate, Type: "CERTIFICATE"})
	for _, extraCert := range certResponse.ExtraCertificates {
		extraCertPEM := pem.EncodeToMemory(&pem.Block{Bytes: extraCert, Type: "CERTIFICATE"})
		certPEM = append(certPEM, '\n')
		certPEM = append(certPEM, extraCertPEM...)
	}
	logrus.Debugf("CERT PEM:\n%s", certPEM)
	certKeyPEM := pemEncode(x509.MarshalPKCS1PrivateKey(certKey), "RSA PRIVATE KEY")

	tmpCert, err := tls.X509KeyPair(certPEM, certKeyPEM)
	logrus.Debugf("Parsed cert count for domain '%v':", len(tmpCert.Certificate))
	if err == nil {
		logrus.Infof("Cert for domain '%v' parsed.", domain)
		cert = &tmpCert
	} else {
		logrus.Errorf("Can't parse cert for domain '%v': %v", domain, err)
		return nil, errors.New("Can't parse cert for domain")
	}

	return cert, nil
}

func (this *acmeStruct) createCertificateSelfSigned(domain string) (cert *tls.Certificate, err error) {
	derCert, privateKey, err := acmeutils.CreateTLSSNICertificate(domain)
	if err != nil {
		logrus.Errorf("Can't create tls-sni-01 self-signed certificate for '%v': %v", domain, err)
		return nil, err
	}

	cert = &tls.Certificate{}
	cert.Certificate = [][]byte{derCert}
	cert.PrivateKey = privateKey
	return cert, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rsa"

	cryptorand "crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"errors"
	"github.com/Sirupsen/logrus"
	"github.com/hlandau/acme/acmeapi"
	"github.com/hlandau/acme/acmeapi/acmeutils"
	"context"
	"net"
	"strings"
	"sync"
	"time"
	"encoding/pem"
)

const (
	SNI01_EXPIRE_TOKEN time.Duration = time.Minute * 10
)

type acmeStruct struct {
	serverAddress string
	privateKey    *rsa.PrivateKey
	client        *acmeapi.Client

	mutex                *sync.Mutex
	acmeauthDomainsMutex *sync.Mutex
	acmeAuthDomains      map[string]time.Time
}

func (this *acmeStruct) Init() {
	this.client = &acmeapi.Client{
		AccountKey:   this.privateKey,
		DirectoryURL: this.serverAddress,
	}

	this.mutex = &sync.Mutex{}

	this.acmeauthDomainsMutex = &sync.Mutex{}
	this.acmeAuthDomains = make(map[string]time.Time)
	this.CleanupTimer()
}

func (this *acmeStruct) RegisterEnsure(ctx context.Context) (err error) {
	reg := &acmeapi.Registration{}
	for i := 0; i < TRY_COUNT+1; i++ { // +1 count need for request latest agreement uri
		reg.AgreementURI = reg.LatestAgreementURI
		if reg.AgreementURI != "" {
			logrus.Info("Auto agree with terms:", reg.LatestAgreementURI)
		}
		err = this.client.UpsertRegistration(reg, ctx)
		if reg.AgreementURI != "" && err == nil {
			return
		}
	}
	return err
}

func (this *acmeStruct) Cleanup() {
	this.mutex.Lock()
	defer this.mutex.Unlock()

	now := time.Now()
	for token, expire := range this.acmeAuthDomains {
		if expire.Before(now) {
			delete(this.acmeAuthDomains, token)
		}
	}
}

func (this *acmeStruct) CleanupTimer() {
	this.Cleanup()
	time.AfterFunc(SNI01_EXPIRE_TOKEN, this.Cleanup)
}

func (this *acmeStruct) CreateCertificate(domain string) (cert *tls.Certificate, err error) {
	// Check suffix for avoid mutex sync in DeleteAcmeAuthDomain
	if strings.HasSuffix(domain, ".acme.invalid") {
		logrus.Debugf("Detect auth-domain mode for domain '%v'", domain)
		if this.DeleteAcmeAuthDomain(domain) {
			logrus.Debugf("Return self-signed certificate for domain '%v'", domain)
			return this.createCertificateSelfSigned(domain)
		} else {
			logrus.Debugf("Detect auth-domain is not present in list '%v'", domain)
			return nil, errors.New("Now allowed auth-domain")
		}
	}

	// check about we serve the domain
	ips, err := net.LookupIP(domain)
	if err != nil {
		logrus.Warnf("Can't lookup ip for domain '%v': %v", domain, err)
		return nil, errors.New("Can't lookup ip of the domain")
	}
	isLocalIP := false
checkLocalIP:
	for _, ip := range ips {
		for _, localIP := range localIPs {
			if ip.Equal(localIP) {
				isLocalIP = true
				break checkLocalIP
			}
		}
	}
	if !isLocalIP {
		logrus.Warnf("Domain have ip of other server. Domain '%v', Domain ips: %v, Server ips: %v", domain, ips, localIPs)
		return nil, errors.New("Domain have ip of other server.")
	}

	return this.createCertificateAcme(domain)
}

func (this *acmeStruct) createCertificateAcme(domain string) (cert *tls.Certificate, err error) {
	this.mutex.Lock()
	defer this.mutex.Unlock()

	var auth *acmeapi.Authorization

	ctx, cancelFunc := context.WithTimeout(context.Background(), LETSENCRYPT_CREATE_CERTIFICATE_TIMEOUT)
	defer cancelFunc()

	for i := 0; i < TRY_COUNT; i++ {
		auth, err = this.client.NewAuthorization(domain, ctx)
		if err == nil {
			break
		} else {
			logrus.Infof("Can't create new authorization for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Infof("Create authorization for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create new authorization for domain '%v': %v", domain, err)
		return nil, errors.New("Can't create new authorization for domain")
	}

	if logrus.GetLevel() >= logrus.DebugLevel {
		challengeTypes := make([]string, len(auth.Challenges))
		for i := range auth.Challenges {
			challengeTypes[i] = auth.Challenges[i].Type
		}
		logrus.Debugf("Challenge types for domain '%v': %v. Challenge combinations: %v", domain, challengeTypes, auth.Combinations)
	}

	canAuthorize := false
	var challenge *acmeapi.Challenge
	for _, cmb := range auth.Combinations {
		if len(cmb) == 1 && auth.Challenges[cmb[0]].Type == "tls-sni-01" {
			canAuthorize = true
			challenge = auth.Challenges[cmb[0]]
			break
		}
	}
	if !canAuthorize {
		logrus.Errorf("Can't find good challange combination for domain: '%v'", domain)
		return nil, errors.New("Can't find good challange combination")
	}

	acmeHostName, err := acmeutils.TLSSNIHostname(this.privateKey, challenge.Token)
	if err == nil {
		logrus.Debugf("Create acme-auth hostname for domain '%v': %v", domain, acmeHostName)
	} else {
		logrus.Errorf("Can't create acme domain for domain '%v' token '%v': %v", domain, challenge.Token, err)
		return nil, errors.New("Can't create acme domain")
	}
	this.PutAcmeAuthDomain(acmeHostName)

	challengeResponse, err := acmeutils.ChallengeResponseJSON(this.privateKey, challenge.Token, challenge.Type)
	if err == nil {
		logrus.Debugf("Create challenge response for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create challenge response for domain '%v', token '%v', challenge type %v: %v",
			domain, challenge.Token, challenge.Type, err)
		return nil, errors.New("Can't create challenge response")
	}
	for i := 0; i < TRY_COUNT; i++ {
		err = this.client.RespondToChallenge(challenge, challengeResponse, this.privateKey, ctx)
		if err == nil {
			break
		} else {
			logrus.Info("Can't send response for challenge of domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Debugf("Send challenge response for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't send response for challenge of domain '%v': %v", domain, err)
		return nil, errors.New("Can't send response for challenge")
	}

	for i := 0; i < TRY_COUNT; i++ {
		err = this.client.LoadChallenge(challenge, ctx)
		if err != nil {
			logrus.Infof("Can't load challenge for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Debugf("Load challenge for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't load challenge for domain '%v': %v", domain, err)
		return nil, errors.New("Can't load challenge")
	}

	// Generate CSR
	certKey, err := rsa.GenerateKey(cryptorand.Reader, PRIVATE_KEY_BITS)
	if err == nil {
		logrus.Debugf("Create private key for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create rsa key for domain '%v': %v", err)
		return nil, errors.New("Can't create rsa key")
	}
	certRequest := &x509.CertificateRequest{
		SignatureAlgorithm: x509.SHA256WithRSA,
		PublicKeyAlgorithm: x509.RSA,
		PublicKey:          &certKey.PublicKey,
		Subject:            pkix.Name{CommonName: domain},
		DNSNames:           []string{domain},
	}
	csrDER, err := x509.CreateCertificateRequest(cryptorand.Reader, certRequest, certKey)
	if err == nil {
		logrus.Debugf("Create CSR for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create csr for domain '%v': %v", domain, err)
		return nil, errors.New("Can't create csr")
	}

	var certResponse *acmeapi.Certificate
	for i := 0; i < TRY_COUNT; i++ {
		certResponse, err = this.client.RequestCertificate(csrDER, ctx)
		if err != nil {
			logrus.Infof("Can't get certificate for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Infof("Get certificate for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't get certificate for domain '%v': %v", domain, err)
		return nil, errors.New("Can't request certificate")
	}

	pemEncode := func(b []byte, t string) []byte {
		return pem.EncodeToMemory(&pem.Block{Bytes: b, Type: t})
	}
	certPEM := pem.EncodeToMemory(&pem.Block{Bytes: certResponse.Certificate, Type: "CERTIFICATE"})
	for _, extraCert := range certResponse.ExtraCertificates {
		extraCertPEM := pem.EncodeToMemory(&pem.Block{Bytes: extraCert, Type: "CERTIFICATE"})
		certPEM = append(certPEM, '\n')
		certPEM = append(certPEM, extraCertPEM...)
	}
	logrus.Debugf("CERT PEM:\n%s", certPEM)
	certKeyPEM := pemEncode(x509.MarshalPKCS1PrivateKey(certKey), "RSA PRIVATE KEY")

	tmpCert, err := tls.X509KeyPair(certPEM, certKeyPEM)
	if err == nil {
		logrus.Infof("Cert for domain '%v' parsed.", domain)
		cert = &tmpCert
	} else {
		logrus.Errorf("Can't parse cert for domain '%v': %v", domain, err)
		return nil, errors.New("Can't parse cert for domain")
	}

	return cert, nil
}

func (this *acmeStruct) createCertificateSelfSigned(domain string) (cert *tls.Certificate, err error) {
	derCert, privateKey, err := acmeutils.CreateTLSSNICertificate(domain)
	if err != nil {
		logrus.Errorf("Can't create tls-sni-01 self-signed certificate for '%v': %v", domain, err)
		return nil, err
	}

	cert = &tls.Certificate{}
	cert.Certificate = [][]byte{derCert}
	cert.PrivateKey = privateKey
	return cert, nil
}

func (this *acmeStruct) PutAcmeAuthDomain(domain string) {
	this.acmeauthDomainsMutex.Lock()
	defer this.acmeauthDomainsMutex.Unlock()

	logrus.Debug("Put acme auth domain:", domain)
	this.acmeAuthDomains[domain] = time.Now().Add(SNI01_EXPIRE_TOKEN)
}

func (this *acmeStruct) DeleteAcmeAuthDomain(domain string) bool {
	this.acmeauthDomainsMutex.Lock()
	defer this.acmeauthDomainsMutex.Unlock()

	logrus.Debug("Delete acme auth domain:", domain)
	_, ok := this.acmeAuthDomains[domain]
	if ok {
		delete(this.acmeAuthDomains, domain)
	}
	return ok
}
<commit_msg>paced cert count<commit_after>package main

import (
	"crypto/rsa"

	cryptorand "crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"errors"
	"github.com/Sirupsen/logrus"
	"github.com/hlandau/acme/acmeapi"
	"github.com/hlandau/acme/acmeapi/acmeutils"
	"context"
	"net"
	"strings"
	"sync"
	"time"
	"encoding/pem"
)

const (
	SNI01_EXPIRE_TOKEN time.Duration = time.Minute * 10
)

type acmeStruct struct {
	serverAddress string
	privateKey    *rsa.PrivateKey
	client        *acmeapi.Client

	mutex                *sync.Mutex
	acmeauthDomainsMutex *sync.Mutex
	acmeAuthDomains      map[string]time.Time
}

func (this *acmeStruct) Init() {
	this.client = &acmeapi.Client{
		AccountKey:   this.privateKey,
		DirectoryURL: this.serverAddress,
	}

	this.mutex = &sync.Mutex{}

	this.acmeauthDomainsMutex = &sync.Mutex{}
	this.acmeAuthDomains = make(map[string]time.Time)
	this.CleanupTimer()
}

func (this *acmeStruct) RegisterEnsure(ctx context.Context) (err error) {
	reg := &acmeapi.Registration{}
	for i := 0; i < TRY_COUNT+1; i++ { // +1 count need for request latest agreement uri
		reg.AgreementURI = reg.LatestAgreementURI
		if reg.AgreementURI != "" {
			logrus.Info("Auto agree with terms:", reg.LatestAgreementURI)
		}
		err = this.client.UpsertRegistration(reg, ctx)
		if reg.AgreementURI != "" && err == nil {
			return
		}
	}
	return err
}

func (this *acmeStruct) Cleanup() {
	this.mutex.Lock()
	defer this.mutex.Unlock()

	now := time.Now()
	for token, expire := range this.acmeAuthDomains {
		if expire.Before(now) {
			delete(this.acmeAuthDomains, token)
		}
	}
}

func (this *acmeStruct) CleanupTimer() {
	this.Cleanup()
	time.AfterFunc(SNI01_EXPIRE_TOKEN, this.Cleanup)
}

func (this *acmeStruct) CreateCertificate(domain string) (cert *tls.Certificate, err error) {
	// Check suffix for avoid mutex sync in DeleteAcmeAuthDomain
	if strings.HasSuffix(domain, ".acme.invalid") {
		logrus.Debugf("Detect auth-domain mode for domain '%v'", domain)
		if this.DeleteAcmeAuthDomain(domain) {
			logrus.Debugf("Return self-signed certificate for domain '%v'", domain)
			return this.createCertificateSelfSigned(domain)
		} else {
			logrus.Debugf("Detect auth-domain is not present in list '%v'", domain)
			return nil, errors.New("Now allowed auth-domain")
		}
	}

	// check about we serve the domain
	ips, err := net.LookupIP(domain)
	if err != nil {
		logrus.Warnf("Can't lookup ip for domain '%v': %v", domain, err)
		return nil, errors.New("Can't lookup ip of the domain")
	}
	isLocalIP := false
checkLocalIP:
	for _, ip := range ips {
		for _, localIP := range localIPs {
			if ip.Equal(localIP) {
				isLocalIP = true
				break checkLocalIP
			}
		}
	}
	if !isLocalIP {
		logrus.Warnf("Domain have ip of other server. Domain '%v', Domain ips: %v, Server ips: %v", domain, ips, localIPs)
		return nil, errors.New("Domain have ip of other server.")
	}

	return this.createCertificateAcme(domain)
}

func (this *acmeStruct) createCertificateAcme(domain string) (cert *tls.Certificate, err error) {
	this.mutex.Lock()
	defer this.mutex.Unlock()

	var auth *acmeapi.Authorization

	ctx, cancelFunc := context.WithTimeout(context.Background(), LETSENCRYPT_CREATE_CERTIFICATE_TIMEOUT)
	defer cancelFunc()

	for i := 0; i < TRY_COUNT; i++ {
		auth, err = this.client.NewAuthorization(domain, ctx)
		if err == nil {
			break
		} else {
			logrus.Infof("Can't create new authorization for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Infof("Create authorization for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create new authorization for domain '%v': %v", domain, err)
		return nil, errors.New("Can't create new authorization for domain")
	}

	if logrus.GetLevel() >= logrus.DebugLevel {
		challengeTypes := make([]string, len(auth.Challenges))
		for i := range auth.Challenges {
			challengeTypes[i] = auth.Challenges[i].Type
		}
		logrus.Debugf("Challenge types for domain '%v': %v. Challenge combinations: %v", domain, challengeTypes, auth.Combinations)
	}

	canAuthorize := false
	var challenge *acmeapi.Challenge
	for _, cmb := range auth.Combinations {
		if len(cmb) == 1 && auth.Challenges[cmb[0]].Type == "tls-sni-01" {
			canAuthorize = true
			challenge = auth.Challenges[cmb[0]]
			break
		}
	}
	if !canAuthorize {
		logrus.Errorf("Can't find good challange combination for domain: '%v'", domain)
		return nil, errors.New("Can't find good challange combination")
	}

	acmeHostName, err := acmeutils.TLSSNIHostname(this.privateKey, challenge.Token)
	if err == nil {
		logrus.Debugf("Create acme-auth hostname for domain '%v': %v", domain, acmeHostName)
	} else {
		logrus.Errorf("Can't create acme domain for domain '%v' token '%v': %v", domain, challenge.Token, err)
		return nil, errors.New("Can't create acme domain")
	}
	this.PutAcmeAuthDomain(acmeHostName)

	challengeResponse, err := acmeutils.ChallengeResponseJSON(this.privateKey, challenge.Token, challenge.Type)
	if err == nil {
		logrus.Debugf("Create challenge response for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create challenge response for domain '%v', token '%v', challenge type %v: %v",
			domain, challenge.Token, challenge.Type, err)
		return nil, errors.New("Can't create challenge response")
	}
	for i := 0; i < TRY_COUNT; i++ {
		err = this.client.RespondToChallenge(challenge, challengeResponse, this.privateKey, ctx)
		if err == nil {
			break
		} else {
			logrus.Info("Can't send response for challenge of domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Debugf("Send challenge response for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't send response for challenge of domain '%v': %v", domain, err)
		return nil, errors.New("Can't send response for challenge")
	}

	for i := 0; i < TRY_COUNT; i++ {
		err = this.client.LoadChallenge(challenge, ctx)
		if err != nil {
			logrus.Infof("Can't load challenge for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Debugf("Load challenge for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't load challenge for domain '%v': %v", domain, err)
		return nil, errors.New("Can't load challenge")
	}

	// Generate CSR
	certKey, err := rsa.GenerateKey(cryptorand.Reader, PRIVATE_KEY_BITS)
	if err == nil {
		logrus.Debugf("Create private key for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create rsa key for domain '%v': %v", err)
		return nil, errors.New("Can't create rsa key")
	}
	certRequest := &x509.CertificateRequest{
		SignatureAlgorithm: x509.SHA256WithRSA,
		PublicKeyAlgorithm: x509.RSA,
		PublicKey:          &certKey.PublicKey,
		Subject:            pkix.Name{CommonName: domain},
		DNSNames:           []string{domain},
	}
	csrDER, err := x509.CreateCertificateRequest(cryptorand.Reader, certRequest, certKey)
	if err == nil {
		logrus.Debugf("Create CSR for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't create csr for domain '%v': %v", domain, err)
		return nil, errors.New("Can't create csr")
	}

	var certResponse *acmeapi.Certificate
	for i := 0; i < TRY_COUNT; i++ {
		certResponse, err = this.client.RequestCertificate(csrDER, ctx)
		if err != nil {
			logrus.Infof("Can't get certificate for domain '%v': %v", domain, err)
			time.Sleep(RETRY_SLEEP)
		}
	}
	if err == nil {
		logrus.Infof("Get certificate for domain '%v'", domain)
	} else {
		logrus.Errorf("Can't get certificate for domain '%v': %v", domain, err)
		return nil, errors.New("Can't request certificate")
	}

	pemEncode := func(b []byte, t string) []byte {
		return pem.EncodeToMemory(&pem.Block{Bytes: b, Type: t})
	}
	certPEM := pem.EncodeToMemory(&pem.Block{Bytes: certResponse.Certificate, Type: "CERTIFICATE"})
	for _, extraCert := range certResponse.ExtraCertificates {
		extraCertPEM := pem.EncodeToMemory(&pem.Block{Bytes: extraCert, Type: "CERTIFICATE"})
		certPEM = append(certPEM, '\n')
		certPEM = append(certPEM, extraCertPEM...)
	}
	logrus.Debugf("CERT PEM:\n%s", certPEM)
	certKeyPEM := pemEncode(x509.MarshalPKCS1PrivateKey(certKey), "RSA PRIVATE KEY")

	tmpCert, err := tls.X509KeyPair(certPEM, certKeyPEM)
	logrus.Debugf("Parsed cert count for domain '%v':", len(tmpCert.Certificate))
	if err == nil {
		logrus.Infof("Cert for domain '%v' parsed.", domain)
		cert = &tmpCert
	} else {
		logrus.Errorf("Can't parse cert for domain '%v': %v", domain, err)
		return nil, errors.New("Can't parse cert for domain")
	}

	return cert, nil
}

func (this *acmeStruct) createCertificateSelfSigned(domain string) (cert *tls.Certificate, err error) {
	derCert, privateKey, err := acmeutils.CreateTLSSNICertificate(domain)
	if err != nil {
		logrus.Errorf("Can't create tls-sni-01 self-signed certificate for '%v': %v", domain, err)
		return nil, err
	}

	cert = &tls.Certificate{}
	cert.Certificate = [][]byte{derCert}
	cert.PrivateKey = privateKey
	return cert, nil
}

func (this *acmeStruct) PutAcmeAuthDomain(domain string) {
	this.acmeauthDomainsMutex.Lock()
	defer this.acmeauthDomainsMutex.Unlock()

	logrus.Debug("Put acme auth domain:", domain)
	this.acmeAuthDomains[domain] = time.Now().Add(SNI01_EXPIRE_TOKEN)
}

func (this *acmeStruct) DeleteAcmeAuthDomain(domain string) bool {
	this.acmeauthDomainsMutex.Lock()
	defer this.acmeauthDomainsMutex.Unlock()

	logrus.Debug("Delete acme auth domain:", domain)
	_, ok := this.acmeAuthDomains[domain]
	if ok {
		delete(this.acmeAuthDomains, domain)
	}
	return ok
}
<|endoftext|>


<commit_before>// Random odds and ends.

package fuse

import (
	"os"
	"fmt"
	"log"
	"math"
	"reflect"
	"regexp"
	"syscall"
	"unsafe"
	"io/ioutil"
)

func (code Status) String() string {
	if code == OK {
		return "OK"
	}
	return fmt.Sprintf("%d=%v", int(code), os.Errno(code))
}

// Make a temporary directory securely.
func MakeTempDir() string {
	nm, err := ioutil.TempDir("", "go-fuse")
	if err != nil {
		panic("TempDir() failed: " + err.String())
	}
	return nm
}

// Convert os.Error back to Errno based errors.
func OsErrorToErrno(err os.Error) Status {
	if err != nil {
		switch t := err.(type) {
		case os.Errno:
			return Status(t)
		case *os.SyscallError:
			return Status(t.Errno)
		case *os.PathError:
			return OsErrorToErrno(t.Error)
		case *os.LinkError:
			return OsErrorToErrno(t.Error)
		default:
			log.Println("can't convert error type:", err)
			return ENOSYS
		}
	}
	return OK
}

func SplitNs(time float64, secs *uint64, nsecs *uint32) {
	*nsecs = uint32(1e9 * (time - math.Trunc(time)))
	*secs = uint64(math.Trunc(time))
}

func CopyFileInfo(fi *os.FileInfo, attr *Attr) {
	attr.Ino = uint64(fi.Ino)
	attr.Size = uint64(fi.Size)
	attr.Blocks = uint64(fi.Blocks)

	attr.Atime = uint64(fi.Atime_ns / 1e9)
	attr.Atimensec = uint32(fi.Atime_ns % 1e9)

	attr.Mtime = uint64(fi.Mtime_ns / 1e9)
	attr.Mtimensec = uint32(fi.Mtime_ns % 1e9)

	attr.Ctime = uint64(fi.Ctime_ns / 1e9)
	attr.Ctimensec = uint32(fi.Ctime_ns % 1e9)

	attr.Mode = fi.Mode
	attr.Nlink = uint32(fi.Nlink)
	attr.Uid = uint32(fi.Uid)
	attr.Gid = uint32(fi.Gid)
	attr.Rdev = uint32(fi.Rdev)
	attr.Blksize = uint32(fi.Blksize)
}

func writev(fd int, iovecs *syscall.Iovec, cnt int) (n int, errno int) {
	n1, _, e1 := syscall.Syscall(
		syscall.SYS_WRITEV,
		uintptr(fd), uintptr(unsafe.Pointer(iovecs)), uintptr(cnt))
	return int(n1), int(e1)
}

func Writev(fd int, packet [][]byte) (n int, err os.Error) {
	iovecs := make([]syscall.Iovec, 0, len(packet))

	for _, v := range packet {
		if v == nil || len(v) == 0 {
			continue
		}
		vec := syscall.Iovec{
			Base: &v[0],
		}
		vec.SetLen(len(v))
		iovecs = append(iovecs, vec)
	}

	if len(iovecs) == 0 {
		return 0, nil
	}

	n, errno := writev(fd, &iovecs[0], len(iovecs))
	if errno != 0 {
		err = os.NewSyscallError("writev", errno)
	}
	return n, err
}

func CountCpus() int {
	var contents [10240]byte

	f, err := os.Open("/proc/stat")
	defer f.Close()
	if err != nil {
		return 1
	}
	n, _ := f.Read(contents[:])
	re, _ := regexp.Compile("\ncpu[0-9]")

	return len(re.FindAllString(string(contents[:n]), 100))
}

// Creates a return entry for a non-existent path.
func NegativeEntry(time float64) *EntryOut {
	out := new(EntryOut)
	out.NodeId = 0
	SplitNs(time, &out.EntryValid, &out.EntryValidNsec)
	return out
}

func ModeToType(mode uint32) uint32 {
	return (mode & 0170000) >> 12
}

func CheckSuccess(e os.Error) {
	if e != nil {
		panic(fmt.Sprintf("Unexpected error: %v", e))
	}
}

func MyPID() string {
	v, _ := os.Readlink("/proc/self")
	return v
}

// Thanks to Andrew Gerrand for this hack.
func asSlice(ptr unsafe.Pointer, byteCount int) []byte {
	h := &reflect.SliceHeader{uintptr(ptr), byteCount, byteCount}
	return *(*[]byte)(unsafe.Pointer(h))
}
<commit_msg>Introduce Status.Ok().<commit_after>// Random odds and ends.

package fuse

import (
	"os"
	"fmt"
	"log"
	"math"
	"reflect"
	"regexp"
	"syscall"
	"unsafe"
	"io/ioutil"
)

func (code Status) String() string {
	if code == OK {
		return "OK"
	}
	return fmt.Sprintf("%d=%v", int(code), os.Errno(code))
}

func (code Status) Ok() bool {
	return code == OK
}

// Make a temporary directory securely.
func MakeTempDir() string {
	nm, err := ioutil.TempDir("", "go-fuse")
	if err != nil {
		panic("TempDir() failed: " + err.String())
	}
	return nm
}

// Convert os.Error back to Errno based errors.
func OsErrorToErrno(err os.Error) Status {
	if err != nil {
		switch t := err.(type) {
		case os.Errno:
			return Status(t)
		case *os.SyscallError:
			return Status(t.Errno)
		case *os.PathError:
			return OsErrorToErrno(t.Error)
		case *os.LinkError:
			return OsErrorToErrno(t.Error)
		default:
			log.Println("can't convert error type:", err)
			return ENOSYS
		}
	}
	return OK
}

func SplitNs(time float64, secs *uint64, nsecs *uint32) {
	*nsecs = uint32(1e9 * (time - math.Trunc(time)))
	*secs = uint64(math.Trunc(time))
}

func CopyFileInfo(fi *os.FileInfo, attr *Attr) {
	attr.Ino = uint64(fi.Ino)
	attr.Size = uint64(fi.Size)
	attr.Blocks = uint64(fi.Blocks)

	attr.Atime = uint64(fi.Atime_ns / 1e9)
	attr.Atimensec = uint32(fi.Atime_ns % 1e9)

	attr.Mtime = uint64(fi.Mtime_ns / 1e9)
	attr.Mtimensec = uint32(fi.Mtime_ns % 1e9)

	attr.Ctime = uint64(fi.Ctime_ns / 1e9)
	attr.Ctimensec = uint32(fi.Ctime_ns % 1e9)

	attr.Mode = fi.Mode
	attr.Nlink = uint32(fi.Nlink)
	attr.Uid = uint32(fi.Uid)
	attr.Gid = uint32(fi.Gid)
	attr.Rdev = uint32(fi.Rdev)
	attr.Blksize = uint32(fi.Blksize)
}

func writev(fd int, iovecs *syscall.Iovec, cnt int) (n int, errno int) {
	n1, _, e1 := syscall.Syscall(
		syscall.SYS_WRITEV,
		uintptr(fd), uintptr(unsafe.Pointer(iovecs)), uintptr(cnt))
	return int(n1), int(e1)
}

func Writev(fd int, packet [][]byte) (n int, err os.Error) {
	iovecs := make([]syscall.Iovec, 0, len(packet))

	for _, v := range packet {
		if v == nil || len(v) == 0 {
			continue
		}
		vec := syscall.Iovec{
			Base: &v[0],
		}
		vec.SetLen(len(v))
		iovecs = append(iovecs, vec)
	}

	if len(iovecs) == 0 {
		return 0, nil
	}

	n, errno := writev(fd, &iovecs[0], len(iovecs))
	if errno != 0 {
		err = os.NewSyscallError("writev", errno)
	}
	return n, err
}

func CountCpus() int {
	var contents [10240]byte

	f, err := os.Open("/proc/stat")
	defer f.Close()
	if err != nil {
		return 1
	}
	n, _ := f.Read(contents[:])
	re, _ := regexp.Compile("\ncpu[0-9]")

	return len(re.FindAllString(string(contents[:n]), 100))
}

// Creates a return entry for a non-existent path.
func NegativeEntry(time float64) *EntryOut {
	out := new(EntryOut)
	out.NodeId = 0
	SplitNs(time, &out.EntryValid, &out.EntryValidNsec)
	return out
}

func ModeToType(mode uint32) uint32 {
	return (mode & 0170000) >> 12
}

func CheckSuccess(e os.Error) {
	if e != nil {
		panic(fmt.Sprintf("Unexpected error: %v", e))
	}
}

func MyPID() string {
	v, _ := os.Readlink("/proc/self")
	return v
}

// Thanks to Andrew Gerrand for this hack.
func asSlice(ptr unsafe.Pointer, byteCount int) []byte {
	h := &reflect.SliceHeader{uintptr(ptr), byteCount, byteCount}
	return *(*[]byte)(unsafe.Pointer(h))
}
<|endoftext|>


<commit_before>package model

import (
	"errors"
	"fmt"

	"github.com/satori/go.uuid"
)

type Account struct {
	ID   int       `json:"id" example:"1" format:"int64"`
	Name string    `json:"name" example:"account name"`
	UUID uuid.UUID `json:"uuid" example:"550e8400-e29b-41d4-a716-446655440000" format:"uuid"`
}

var (
	ErrNameInvalid = errors.New("name is empty")
)

type AddAccount struct {
	Name string `json:"name" example:"account name"`
}

func (a AddAccount) Validation() error {
	switch {
	case len(a.Name) == 0:
		return ErrNameInvalid
	default:
		return nil
	}
}

type UpdateAccount struct {
	Name string `json:"name" example:"account name"`
}

func (a UpdateAccount) Validation() error {
	switch {
	case len(a.Name) == 0:
		return ErrNameInvalid
	default:
		return nil
	}
}

func AccountsAll(q string) ([]Account, error) {
	if q == "" {
		return accounts, nil
	}
	as := []Account{}
	for k, v := range accounts {
		if q == v.Name {
			as = append(as, accounts[k])
		}
	}
	return as, nil
}

func AccountOne(id int) (Account, error) {
	for _, v := range accounts {
		if id == v.ID {
			return v, nil
		}
	}
	return Account{}, ErrNoRow
}

func (a Account) Insert() (int, error) {
	accountMaxID++
	a.ID = accountMaxID
	accounts = append(accounts, a)
	return accountMaxID, nil
}

func Delete(id int) error {
	for k, v := range accounts {
		if id == v.ID {
			accounts = append(accounts[:k], accounts[k+1:]...)
			return nil
		}
	}
	return fmt.Errorf("account id=%d is not found", id)
}

func (a Account) Update() error {
	for k, v := range accounts {
		if a.ID == v.ID {
			accounts[k].Name = a.Name
			return nil
		}
	}
	return fmt.Errorf("account id=%d is not found", a.ID)
}

var accountMaxID = 3
var accounts = []Account{
	Account{ID: 1, Name: "account_1"},
	Account{ID: 2, Name: "account_2"},
	Account{ID: 3, Name: "account_3"},
}
<commit_msg>example: update account.go (#196)<commit_after>package model

import (
	"errors"
	"fmt"

	"github.com/satori/go.uuid"
)

type Account struct {
	ID   int       `json:"id" example:"1" format:"int64"`
	Name string    `json:"name" example:"account name"`
	UUID uuid.UUID `json:"uuid" example:"550e8400-e29b-41d4-a716-446655440000" format:"uuid"`
}

var (
	ErrNameInvalid = errors.New("name is empty")
)

type AddAccount struct {
	Name string `json:"name" example:"account name"`
}

func (a AddAccount) Validation() error {
	switch {
	case len(a.Name) == 0:
		return ErrNameInvalid
	default:
		return nil
	}
}

type UpdateAccount struct {
	Name string `json:"name" example:"account name"`
}

func (a UpdateAccount) Validation() error {
	switch {
	case len(a.Name) == 0:
		return ErrNameInvalid
	default:
		return nil
	}
}

func AccountsAll(q string) ([]Account, error) {
	if q == "" {
		return accounts, nil
	}
	as := []Account{}
	for k, v := range accounts {
		if q == v.Name {
			as = append(as, accounts[k])
		}
	}
	return as, nil
}

func AccountOne(id int) (Account, error) {
	for _, v := range accounts {
		if id == v.ID {
			return v, nil
		}
	}
	return Account{}, ErrNoRow
}

func (a Account) Insert() (int, error) {
	accountMaxID++
	a.ID = accountMaxID
	a.Name = fmt.Sprintf("account_%d", accountMaxID)
	accounts = append(accounts, a)
	return accountMaxID, nil
}

func Delete(id int) error {
	for k, v := range accounts {
		if id == v.ID {
			accounts = append(accounts[:k], accounts[k+1:]...)
			return nil
		}
	}
	return fmt.Errorf("account id=%d is not found", id)
}

func (a Account) Update() error {
	for k, v := range accounts {
		if a.ID == v.ID {
			accounts[k].Name = a.Name
			return nil
		}
	}
	return fmt.Errorf("account id=%d is not found", a.ID)
}

var accountMaxID = 3
var accounts = []Account{
	Account{ID: 1, Name: "account_1"},
	Account{ID: 2, Name: "account_2"},
	Account{ID: 3, Name: "account_3"},
}
<|endoftext|>


<commit_before>package jwd

import (
	"math"
	"strings"

	"github.com/9uuso/unidecode"
)

// According to this tool: http://www.csun.edu/english/edit_distance.php
// parameter order should have no difference in the result. Therefore,
// to avoid panicing later on, we will order the strings according to
// their length.
func sort(s1, s2 string) (string, string) {
	if strings.Count(s1, "")-1 <= strings.Count(s2, "")-1 {
		return s1, s2
	}
	return s2, s1
}

// Calculate calculates Jaro-Winkler distance of two strings. The function lowercases and sorts the parameters
// so that that the longest string is evaluated against the shorter one.
func Calculate(s1, s2 string) float64 {

	s1 = unidecode.Unidecode(s1)
	s2 = unidecode.Unidecode(s2)

	s1, s2 = sort(strings.ToLower(s1), strings.ToLower(s2))

	// This avoids the function to return NaN.
	if strings.Count(s1, "") == 1 || strings.Count(s2, "") == 1 {
		return float64(0)
	}

	// m as `matching characters`
	// t as `transposition`
	// l as `the length of common prefix at the start of the string up to a maximum of 4 characters`.
	// See more: https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance
	m := 0
	t := 0
	l := 0

	window := math.Floor(float64(math.Max(float64(len(s1)), float64(len(s2)))/2) - 1)

	//debug:
	//fmt.Println("s1:", s1, "s2:", s2)
	//fmt.Println("Match window:", window)
	//fmt.Println("len(s1):", len(s1), "len(s2):", len(s2))

	for i := 0; i < len(s1); i++ {
		// Exact match
		if s1[i] == s2[i] {
			m++
			// Common prefix limitter
			if i == l && i < 4 {
				l++
			}
		} else {
			if strings.Contains(s2, string(s1[i])) {
				// The character is also considered matching if the amount of characters between the occurances in s1 and s2
				// is less than match window
				gap := strings.Index(s2, string(s1[i])) - strings.Index(s1, string(s1[i]))
				if gap <= int(window) {
					m++
					// Check if transposition is in reach of window
					for k := i; k < len(s1); k++ {
						if strings.Index(s2, string(s1[k])) <= i {
							t++
						}
					}
				}
			}
		}
	}

	distance := (float64(m)/float64(len(s1)) + float64(m)/float64(len(s2)) + (float64(m)-math.Floor(float64(t)/float64(2)))/float64(m)) / float64(3)
	jwd := distance + (float64(l) * float64(0.1) * (float64(1) - distance))

	//debug:
	//fmt.Println("- transpositions:", t)
	//fmt.Println("- matches:", m)
	//fmt.Println("- l:", l)
	//fmt.Println(jwd)

	return jwd

}
<commit_msg>update unidecode package import path<commit_after>package jwd

import (
	"math"
	"strings"

	"gopkgs.com/unidecode.v1"
)

// According to this tool: http://www.csun.edu/english/edit_distance.php
// parameter order should have no difference in the result. Therefore,
// to avoid panicing later on, we will order the strings according to
// their length.
func sort(s1, s2 string) (string, string) {
	if strings.Count(s1, "")-1 <= strings.Count(s2, "")-1 {
		return s1, s2
	}
	return s2, s1
}

// Calculate calculates Jaro-Winkler distance of two strings. The function lowercases and sorts the parameters
// so that that the longest string is evaluated against the shorter one.
func Calculate(s1, s2 string) float64 {

	s1 = unidecode.Unidecode(s1)
	s2 = unidecode.Unidecode(s2)

	s1, s2 = sort(strings.ToLower(s1), strings.ToLower(s2))

	// This avoids the function to return NaN.
	if strings.Count(s1, "") == 1 || strings.Count(s2, "") == 1 {
		return float64(0)
	}

	// m as `matching characters`
	// t as `transposition`
	// l as `the length of common prefix at the start of the string up to a maximum of 4 characters`.
	// See more: https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance
	m := 0
	t := 0
	l := 0

	window := math.Floor(float64(math.Max(float64(len(s1)), float64(len(s2)))/2) - 1)

	//debug:
	//fmt.Println("s1:", s1, "s2:", s2)
	//fmt.Println("Match window:", window)
	//fmt.Println("len(s1):", len(s1), "len(s2):", len(s2))

	for i := 0; i < len(s1); i++ {
		// Exact match
		if s1[i] == s2[i] {
			m++
			// Common prefix limitter
			if i == l && i < 4 {
				l++
			}
		} else {
			if strings.Contains(s2, string(s1[i])) {
				// The character is also considered matching if the amount of characters between the occurances in s1 and s2
				// is less than match window
				gap := strings.Index(s2, string(s1[i])) - strings.Index(s1, string(s1[i]))
				if gap <= int(window) {
					m++
					// Check if transposition is in reach of window
					for k := i; k < len(s1); k++ {
						if strings.Index(s2, string(s1[k])) <= i {
							t++
						}
					}
				}
			}
		}
	}

	distance := (float64(m)/float64(len(s1)) + float64(m)/float64(len(s2)) + (float64(m)-math.Floor(float64(t)/float64(2)))/float64(m)) / float64(3)
	jwd := distance + (float64(l) * float64(0.1) * (float64(1) - distance))

	//debug:
	//fmt.Println("- transpositions:", t)
	//fmt.Println("- matches:", m)
	//fmt.Println("- l:", l)
	//fmt.Println(jwd)

	return jwd

}
<|endoftext|>


<commit_before>// Copyright 2014 gandalf authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/tsuru/config"
	"github.com/tsuru/gandalf/api"
	"log"
	"net/http"
)

const version = "0.4.1"

func main() {
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	configFile := flag.String("config", "/etc/gandalf.conf", "Gandalf configuration file")
	gVersion := flag.Bool("version", false, "Print version and exit")
	flag.Parse()

	if *gVersion {
		fmt.Printf("gandalf-webserver version %s\n", version)
		return
	}

	err := config.ReadAndWatchConfigFile(*configFile)
	if err != nil {
		msg := `Could not find gandalf config file. Searched on %s.
For an example conf check gandalf/etc/gandalf.conf file.\n %s`
		log.Panicf(msg, *configFile, err)
	}
	router := pat.New()
	router.Post("/user/:name/key", http.HandlerFunc(api.AddKey))
	router.Del("/user/:name/key/:keyname", http.HandlerFunc(api.RemoveKey))
	router.Get("/user/:name/keys", http.HandlerFunc(api.ListKeys))
	router.Post("/user", http.HandlerFunc(api.NewUser))
	router.Del("/user/:name", http.HandlerFunc(api.RemoveUser))
	router.Post("/repository", http.HandlerFunc(api.NewRepository))
	router.Post("/repository/grant", http.HandlerFunc(api.GrantAccess))
	router.Del("/repository/revoke", http.HandlerFunc(api.RevokeAccess))
	router.Del("/repository/:name", http.HandlerFunc(api.RemoveRepository))
	router.Get("/repository/:name", http.HandlerFunc(api.GetRepository))
	router.Put("/repository/:name", http.HandlerFunc(api.RenameRepository))
	router.Get("/repository/:name/archive/:ref.:format", http.HandlerFunc(api.GetArchive))
	router.Get("/repository/:name/contents", http.HandlerFunc(api.GetFileContents))
	router.Get("/repository/:name/tree/:path", http.HandlerFunc(api.GetTree))
	router.Get("/repository/:name/tree", http.HandlerFunc(api.GetTree))
	router.Get("/repository/:name/branch", http.HandlerFunc(api.GetTree))
	router.Get("/healthcheck/", http.HandlerFunc(api.HealthCheck))
	router.Post("/hook/:name", http.HandlerFunc(api.AddHook))

	bind, err := config.GetString("bind")
	if err != nil {
		var perr error
		bind, perr = config.GetString("webserver:port")
		if perr != nil {
			panic(err)
		}
	}
	if !*dry {
		log.Fatal(http.ListenAndServe(bind, router))
	}
}
<commit_msg>Main: Fixing Branch handler<commit_after>// Copyright 2014 gandalf authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/tsuru/config"
	"github.com/tsuru/gandalf/api"
	"log"
	"net/http"
)

const version = "0.4.1"

func main() {
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	configFile := flag.String("config", "/etc/gandalf.conf", "Gandalf configuration file")
	gVersion := flag.Bool("version", false, "Print version and exit")
	flag.Parse()

	if *gVersion {
		fmt.Printf("gandalf-webserver version %s\n", version)
		return
	}

	err := config.ReadAndWatchConfigFile(*configFile)
	if err != nil {
		msg := `Could not find gandalf config file. Searched on %s.
For an example conf check gandalf/etc/gandalf.conf file.\n %s`
		log.Panicf(msg, *configFile, err)
	}
	router := pat.New()
	router.Post("/user/:name/key", http.HandlerFunc(api.AddKey))
	router.Del("/user/:name/key/:keyname", http.HandlerFunc(api.RemoveKey))
	router.Get("/user/:name/keys", http.HandlerFunc(api.ListKeys))
	router.Post("/user", http.HandlerFunc(api.NewUser))
	router.Del("/user/:name", http.HandlerFunc(api.RemoveUser))
	router.Post("/repository", http.HandlerFunc(api.NewRepository))
	router.Post("/repository/grant", http.HandlerFunc(api.GrantAccess))
	router.Del("/repository/revoke", http.HandlerFunc(api.RevokeAccess))
	router.Del("/repository/:name", http.HandlerFunc(api.RemoveRepository))
	router.Get("/repository/:name", http.HandlerFunc(api.GetRepository))
	router.Put("/repository/:name", http.HandlerFunc(api.RenameRepository))
	router.Get("/repository/:name/archive/:ref.:format", http.HandlerFunc(api.GetArchive))
	router.Get("/repository/:name/contents", http.HandlerFunc(api.GetFileContents))
	router.Get("/repository/:name/tree/:path", http.HandlerFunc(api.GetTree))
	router.Get("/repository/:name/tree", http.HandlerFunc(api.GetTree))
	router.Get("/repository/:name/branch", http.HandlerFunc(api.GetBranch))
	router.Get("/healthcheck/", http.HandlerFunc(api.HealthCheck))
	router.Post("/hook/:name", http.HandlerFunc(api.AddHook))

	bind, err := config.GetString("bind")
	if err != nil {
		var perr error
		bind, perr = config.GetString("webserver:port")
		if perr != nil {
			panic(err)
		}
	}
	if !*dry {
		log.Fatal(http.ListenAndServe(bind, router))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/currantlabs/ble"
	"github.com/currantlabs/ble/examples/lib/dev"
)

var (
	device = flag.String("device", "default", "implementation of ble")
	name   = flag.String("name", "Gopher", "name of remote peripheral")
	addr   = flag.String("addr", "", "address of remote peripheral (MAC on Linux, UUID on OS X)")
	sub    = flag.Duration("sub", 0, "subscribe to notification and indication for a specified period")
	sd     = flag.Duration("sd", 5*time.Second, "scanning duration, 0 for indefinitely")
)

func main() {
	flag.Parse()

	d, err := dev.NewDevice(*device)
	if err != nil {
		log.Fatalf("can't new device : %s", err)
	}
	ble.SetDefaultDevice(d)

	// Default to search device with name of Gopher (or specified by user).
	filter := func(a ble.Advertisement) bool {
		return strings.ToUpper(a.LocalName()) == strings.ToUpper(*name)
	}

	// If addr is specified, search for addr instead.
	if len(*addr) != 0 {
		filter = func(a ble.Advertisement) bool {
			return strings.ToUpper(a.Address().String()) == strings.ToUpper(*addr)
		}
	}

	// Scan for specified durantion, or until interrupted by user.
	fmt.Printf("Scanning for %s...\n", *sd)
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), *sd))
	cln, err := ble.Connect(ctx, filter)
	if err != nil {
		log.Fatalf("can't connect : %s", err)
	}

	go func() {
		cln.Disconnected()
		fmt.Printf("[ %s ] is disconnected \n", cln.Address())
	}()

	fmt.Printf("Discovering profile...\n")
	p, err := cln.DiscoverProfile(true)
	if err != nil {
		log.Fatalf("can't discover profile: %s", err)
	}

	// Start the exploration.
	explore(cln, p)

	// Disconnect the connection. (On OS X, this might take a while.)
	fmt.Printf("Disconnecting [ %s ]... (this might take up to few seconds on OS X)\n", cln.Address())
	cln.CancelConnection()
}

func explore(cln ble.Client, p *ble.Profile) error {
	for _, s := range p.Services {
		fmt.Printf("    Service: %s %s, Handle (0x%02X)\n", s.UUID, ble.Name(s.UUID), s.Handle)

		for _, c := range s.Characteristics {
			fmt.Printf("      Characteristic: %s %s, Property: 0x%02X (%s), Handle(0x%02X), VHandle(0x%02X)\n",
				c.UUID, ble.Name(c.UUID), c.Property, propString(c.Property), c.Handle, c.ValueHandle)
			if (c.Property & ble.CharRead) != 0 {
				b, err := cln.ReadCharacteristic(c)
				if err != nil {
					fmt.Printf("Failed to read characteristic: %s\n", err)
					continue
				}
				fmt.Printf("        Value         %x | %q\n", b, b)
			}

			for _, d := range c.Descriptors {
				fmt.Printf("        Descriptor: %s %s, Handle(0x%02x)\n", d.UUID, ble.Name(d.UUID), d.Handle)
				b, err := cln.ReadDescriptor(d)
				if err != nil {
					fmt.Printf("Failed to read descriptor: %s\n", err)
					continue
				}
				fmt.Printf("        Value         %x | %q\n", b, b)
			}

			if *sub != 0 {
				// Don't bother to subscribe the Service Changed characteristics.
				if c.UUID.Equal(ble.ServiceChangedUUID) {
					continue
				}

				// Don't touch the Apple-specific Service/Characteristic.
				// Service: D0611E78BBB44591A5F8487910AE4366
				// Characteristic: 8667556C9A374C9184ED54EE27D90049, Property: 0x18 (WN),
				//   Descriptor: 2902, Client Characteristic Configuration
				//   Value         0000 | "\x00\x00"
				if c.UUID.Equal(ble.MustParse("8667556C9A374C9184ED54EE27D90049")) {
					continue
				}

				if (c.Property & ble.CharNotify) != 0 {
					fmt.Printf("\n-- Subscribe to notification for %s --\n", *sub)
					h := func(req []byte) { fmt.Printf("Notified: %q [ % X ]\n", string(req), req) }
					if err := cln.Subscribe(c, false, h); err != nil {
						log.Fatalf("subscribe failed: %s", err)
					}
					time.Sleep(*sub)
					if err := cln.Unsubscribe(c, false); err != nil {
						log.Fatalf("unsubscribe failed: %s", err)
					}
					fmt.Printf("-- Unsubscribe to notification --\n")
				}
				if (c.Property & ble.CharIndicate) != 0 {
					fmt.Printf("\n-- Subscribe to indication of %s --\n", *sub)
					h := func(req []byte) { fmt.Printf("Indicated: %q [ % X ]\n", string(req), req) }
					if err := cln.Subscribe(c, true, h); err != nil {
						log.Fatalf("subscribe failed: %s", err)
					}
					time.Sleep(*sub)
					if err := cln.Unsubscribe(c, true); err != nil {
						log.Fatalf("unsubscribe failed: %s", err)
					}
					fmt.Printf("-- Unsubscribe to indication --\n")
				}
			}
		}
		fmt.Printf("\n")
	}
	return nil
}

func propString(p ble.Property) string {
	var s string
	for k, v := range map[ble.Property]string{
		ble.CharBroadcast:   "B",
		ble.CharRead:        "R",
		ble.CharWriteNR:     "w",
		ble.CharWrite:       "W",
		ble.CharNotify:      "N",
		ble.CharIndicate:    "I",
		ble.CharSignedWrite: "S",
		ble.CharExtended:    "E",
	} {
		if p&k != 0 {
			s += v
		}
	}
	return s
}

func chkErr(err error) {
	switch errors.Cause(err) {
	case nil:
	case context.DeadlineExceeded:
		fmt.Printf("done\n")
	case context.Canceled:
		fmt.Printf("canceled\n")
	default:
		log.Fatalf(err.Error())
	}
}
<commit_msg>example: update explorer to wait disconnection complete.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/currantlabs/ble"
	"github.com/currantlabs/ble/examples/lib/dev"
)

var (
	device = flag.String("device", "default", "implementation of ble")
	name   = flag.String("name", "Gopher", "name of remote peripheral")
	addr   = flag.String("addr", "", "address of remote peripheral (MAC on Linux, UUID on OS X)")
	sub    = flag.Duration("sub", 0, "subscribe to notification and indication for a specified period")
	sd     = flag.Duration("sd", 5*time.Second, "scanning duration, 0 for indefinitely")
)

func main() {
	flag.Parse()

	d, err := dev.NewDevice(*device)
	if err != nil {
		log.Fatalf("can't new device : %s", err)
	}
	ble.SetDefaultDevice(d)

	// Default to search device with name of Gopher (or specified by user).
	filter := func(a ble.Advertisement) bool {
		return strings.ToUpper(a.LocalName()) == strings.ToUpper(*name)
	}

	// If addr is specified, search for addr instead.
	if len(*addr) != 0 {
		filter = func(a ble.Advertisement) bool {
			return strings.ToUpper(a.Address().String()) == strings.ToUpper(*addr)
		}
	}

	// Scan for specified durantion, or until interrupted by user.
	fmt.Printf("Scanning for %s...\n", *sd)
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), *sd))
	cln, err := ble.Connect(ctx, filter)
	if err != nil {
		log.Fatalf("can't connect : %s", err)
	}

	// Make sure we had the chance to print out the message.
	done := make(chan struct{})
	// Normally, the connection is disconnected by us after our exploration.
	// However, it can be asynchronously disconnected by the remote peripheral.
	// So we wait(detect) the disconnection in the go routine.
	go func() {
		<-cln.Disconnected()
		fmt.Printf("[ %s ] is disconnected \n", cln.Address())
		close(done)
	}()

	fmt.Printf("Discovering profile...\n")
	p, err := cln.DiscoverProfile(true)
	if err != nil {
		log.Fatalf("can't discover profile: %s", err)
	}

	// Start the exploration.
	explore(cln, p)

	// Disconnect the connection. (On OS X, this might take a while.)
	fmt.Printf("Disconnecting [ %s ]... (this might take up to few seconds on OS X)\n", cln.Address())
	cln.CancelConnection()

	<-done
}

func explore(cln ble.Client, p *ble.Profile) error {
	for _, s := range p.Services {
		fmt.Printf("    Service: %s %s, Handle (0x%02X)\n", s.UUID, ble.Name(s.UUID), s.Handle)

		for _, c := range s.Characteristics {
			fmt.Printf("      Characteristic: %s %s, Property: 0x%02X (%s), Handle(0x%02X), VHandle(0x%02X)\n",
				c.UUID, ble.Name(c.UUID), c.Property, propString(c.Property), c.Handle, c.ValueHandle)
			if (c.Property & ble.CharRead) != 0 {
				b, err := cln.ReadCharacteristic(c)
				if err != nil {
					fmt.Printf("Failed to read characteristic: %s\n", err)
					continue
				}
				fmt.Printf("        Value         %x | %q\n", b, b)
			}

			for _, d := range c.Descriptors {
				fmt.Printf("        Descriptor: %s %s, Handle(0x%02x)\n", d.UUID, ble.Name(d.UUID), d.Handle)
				b, err := cln.ReadDescriptor(d)
				if err != nil {
					fmt.Printf("Failed to read descriptor: %s\n", err)
					continue
				}
				fmt.Printf("        Value         %x | %q\n", b, b)
			}

			if *sub != 0 {
				// Don't bother to subscribe the Service Changed characteristics.
				if c.UUID.Equal(ble.ServiceChangedUUID) {
					continue
				}

				// Don't touch the Apple-specific Service/Characteristic.
				// Service: D0611E78BBB44591A5F8487910AE4366
				// Characteristic: 8667556C9A374C9184ED54EE27D90049, Property: 0x18 (WN),
				//   Descriptor: 2902, Client Characteristic Configuration
				//   Value         0000 | "\x00\x00"
				if c.UUID.Equal(ble.MustParse("8667556C9A374C9184ED54EE27D90049")) {
					continue
				}

				if (c.Property & ble.CharNotify) != 0 {
					fmt.Printf("\n-- Subscribe to notification for %s --\n", *sub)
					h := func(req []byte) { fmt.Printf("Notified: %q [ % X ]\n", string(req), req) }
					if err := cln.Subscribe(c, false, h); err != nil {
						log.Fatalf("subscribe failed: %s", err)
					}
					time.Sleep(*sub)
					if err := cln.Unsubscribe(c, false); err != nil {
						log.Fatalf("unsubscribe failed: %s", err)
					}
					fmt.Printf("-- Unsubscribe to notification --\n")
				}
				if (c.Property & ble.CharIndicate) != 0 {
					fmt.Printf("\n-- Subscribe to indication of %s --\n", *sub)
					h := func(req []byte) { fmt.Printf("Indicated: %q [ % X ]\n", string(req), req) }
					if err := cln.Subscribe(c, true, h); err != nil {
						log.Fatalf("subscribe failed: %s", err)
					}
					time.Sleep(*sub)
					if err := cln.Unsubscribe(c, true); err != nil {
						log.Fatalf("unsubscribe failed: %s", err)
					}
					fmt.Printf("-- Unsubscribe to indication --\n")
				}
			}
		}
		fmt.Printf("\n")
	}
	return nil
}

func propString(p ble.Property) string {
	var s string
	for k, v := range map[ble.Property]string{
		ble.CharBroadcast:   "B",
		ble.CharRead:        "R",
		ble.CharWriteNR:     "w",
		ble.CharWrite:       "W",
		ble.CharNotify:      "N",
		ble.CharIndicate:    "I",
		ble.CharSignedWrite: "S",
		ble.CharExtended:    "E",
	} {
		if p&k != 0 {
			s += v
		}
	}
	return s
}

func chkErr(err error) {
	switch errors.Cause(err) {
	case nil:
	case context.DeadlineExceeded:
		fmt.Printf("done\n")
	case context.Canceled:
		fmt.Printf("canceled\n")
	default:
		log.Fatalf(err.Error())
	}
}
<|endoftext|>


<commit_before>package gass

import (
	"testing"
)

func testFile(t *testing.T, file string) {
	if _, err := CompileFile(file, true); err != nil {
		t.Error(err)
	}
}

func TestParser(t *testing.T) {
	testFile("test/parser.gass", "test/parser.css")
	testFile("test/variables.gass", "test/variables.css")
	testFile("test/functions.gass", "test/functions.css")
}

/*
func processExpect(path string, info os.FileInfo, err error) error {
	if filepath.Ext(path) == ".gass" && !strings.HasPrefix(path, "_") {
		in, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		out := CompileString(string(in))
		exp, err := ioutil.ReadFile(path[:len(path)-5] + ".expect")
		if err != nil {
			return err
		}
		ioutil.WriteFile(path[:len(path)-5]+".log", []byte(out), 0)
		if out != string(exp) {
			return errors.New("mismatch:" + path)
		}
	}
	return nil
}

func TestExpect(t *testing.T) {
	if err := filepath.Walk("test/expect", processExpect); err != nil {
		t.Error(err)
	}
}
*/
<commit_msg>fix gass_test error<commit_after>package gass

import (
	"testing"
)

func testFile(t *testing.T, file string) {
	if _, err := CompileFile(file, true); err != nil {
		t.Error(err)
	}
}

func TestParser(t *testing.T) {
	testFile(t, "test/parser.gass")
	testFile(t, "test/variables.gass")
	testFile(t, "test/functions.gass")
}

/*
func processExpect(path string, info os.FileInfo, err error) error {
	if filepath.Ext(path) == ".gass" && !strings.HasPrefix(path, "_") {
		in, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		out := CompileString(string(in))
		exp, err := ioutil.ReadFile(path[:len(path)-5] + ".expect")
		if err != nil {
			return err
		}
		ioutil.WriteFile(path[:len(path)-5]+".log", []byte(out), 0)
		if out != string(exp) {
			return errors.New("mismatch:" + path)
		}
	}
	return nil
}

func TestExpect(t *testing.T) {
	if err := filepath.Walk("test/expect", processExpect); err != nil {
		t.Error(err)
	}
}
*/
<|endoftext|>


<commit_before><commit_msg>fix (api): sql: no rows in result set on project.Load (#288)<commit_after><|endoftext|>


<commit_before>package mpb

import (
	"container/heap"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v4/cwriter"
)

const (
	// default RefreshRate
	prr = 120 * time.Millisecond
	// default width
	pwidth = 80
)

// Progress represents the container that renders Progress bars
type Progress struct {
	uwg          *sync.WaitGroup
	cwg          *sync.WaitGroup
	bwg          *sync.WaitGroup
	operateState chan func(*pState)
	done         chan struct{}
}

type pState struct {
	bHeap           *priorityQueue
	shutdownPending []*Bar
	heapUpdated     bool
	idCounter       int
	width           int
	rr              time.Duration
	pMatrix         map[int][]chan int
	aMatrix         map[int][]chan int
	forceRefreshCh  chan time.Time
	output          io.Writer

	// following are provided/overrided by user
	ctx              context.Context
	uwg              *sync.WaitGroup
	manualRefreshCh  <-chan time.Time
	shutdownNotifier chan struct{}
	waitBars         map[*Bar]*Bar
	debugOut         io.Writer
}

// New creates new Progress instance, which orchestrates bars rendering
// process. Accepts mpb.ContainerOption funcs for customization.
func New(options ...ContainerOption) *Progress {
	pq := make(priorityQueue, 0)
	heap.Init(&pq)

	s := &pState{
		ctx:            context.Background(),
		bHeap:          &pq,
		width:          pwidth,
		rr:             prr,
		waitBars:       make(map[*Bar]*Bar),
		debugOut:       ioutil.Discard,
		forceRefreshCh: make(chan time.Time),
		output:         os.Stdout,
	}

	for _, opt := range options {
		if opt != nil {
			opt(s)
		}
	}

	p := &Progress{
		uwg:          s.uwg,
		cwg:          new(sync.WaitGroup),
		bwg:          new(sync.WaitGroup),
		operateState: make(chan func(*pState)),
		done:         make(chan struct{}),
	}
	p.cwg.Add(1)
	go p.serve(s, cwriter.New(s.output))
	return p
}

// AddBar creates a new progress bar and adds to the container.
func (p *Progress) AddBar(total int64, options ...BarOption) *Bar {
	return p.Add(total, newDefaultBarFiller(), options...)
}

// AddSpinner creates a new spinner bar and adds to the container.
func (p *Progress) AddSpinner(total int64, alignment SpinnerAlignment, options ...BarOption) *Bar {
	filler := &spinnerFiller{
		frames:    defaultSpinnerStyle,
		alignment: alignment,
	}
	return p.Add(total, filler, options...)
}

// Add creates a bar which renders itself by provided filler.
func (p *Progress) Add(total int64, filler Filler, options ...BarOption) *Bar {
	p.bwg.Add(1)
	result := make(chan *Bar)
	select {
	case p.operateState <- func(s *pState) {
		b := newBar(s.ctx, p.bwg, filler, s.idCounter, s.width, total, options...)
		if b.runningBar != nil {
			s.waitBars[b.runningBar] = b
		} else {
			heap.Push(s.bHeap, b)
			s.heapUpdated = true
		}
		s.idCounter++
		result <- b
	}:
		return <-result
	case <-p.done:
		p.bwg.Done()
		return nil
	}
}

// Abort is only effective while bar progress is running, it means
// remove bar now without waiting for its completion. If bar is already
// completed, there is nothing to abort. If you need to remove bar
// after completion, use BarRemoveOnComplete BarOption.
func (p *Progress) Abort(b *Bar, remove bool) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		if remove {
			s.heapUpdated = heap.Remove(s.bHeap, b.index) != nil
		}
		s.shutdownPending = append(s.shutdownPending, b)
	}:
	case <-p.done:
	}
}

// UpdateBarPriority provides a way to change bar's order position.
// Zero is highest priority, i.e. bar will be on top.
func (p *Progress) UpdateBarPriority(b *Bar, priority int) {
	select {
	case p.operateState <- func(s *pState) { s.bHeap.update(b, priority) }:
	case <-p.done:
	}
}

// BarCount returns bars count
func (p *Progress) BarCount() int {
	result := make(chan int, 1)
	select {
	case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }:
		return <-result
	case <-p.done:
		return 0
	}
}

// Wait waits far all bars to complete and finally shutdowns container.
// After this method has been called, there is no way to reuse *Progress
// instance.
func (p *Progress) Wait() {
	if p.uwg != nil {
		// wait for user wg
		p.uwg.Wait()
	}

	// wait for bars to quit, if any
	p.bwg.Wait()

	close(p.done)

	// wait for container to quit
	p.cwg.Wait()
}

func (p *Progress) serve(s *pState, cw *cwriter.Writer) {
	defer p.cwg.Done()

	manualOrTickCh, cleanUp := s.manualOrTick()
	defer cleanUp()

	refreshCh := fanInRefreshSrc(p.done, s.forceRefreshCh, manualOrTickCh)

	for {
		select {
		case op := <-p.operateState:
			op(s)
		case _, ok := <-refreshCh:
			if !ok {
				if s.shutdownNotifier != nil {
					close(s.shutdownNotifier)
				}
				return
			}
			if err := s.render(p.done, cw); err != nil {
				fmt.Fprintf(s.debugOut, "[mpb] %s %v\n", time.Now(), err)
			}
		}
	}
}

func (s *pState) render(done <-chan struct{}, cw *cwriter.Writer) error {
	if s.heapUpdated {
		s.updateSyncMatrix()
		s.heapUpdated = false
	}
	syncWidth(s.pMatrix)
	syncWidth(s.aMatrix)

	tw, err := cw.GetWidth()
	if err != nil {
		tw = s.width
	}
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := (*s.bHeap)[i]
		go bar.render(s.debugOut, tw)
	}

	return s.flush(done, cw)
}

func (s *pState) flush(done <-chan struct{}, cw *cwriter.Writer) error {
	var lineCount int
	for s.bHeap.Len() > 0 {
		bar := heap.Pop(s.bHeap).(*Bar)
		frame := <-bar.bFrameCh
		defer func() {
			if frame.toShutdown {
				// force next refresh asap, without waiting for ticker
				go func() {
					select {
					case s.forceRefreshCh <- time.Now():
					case <-done:
						return
					}
				}()
				// shutdown at next flush, in other words decrement underlying WaitGroup
				// only after the bar with completed state has been flushed. this
				// ensures no bar ends up with less than 100% rendered.
				s.shutdownPending = append(s.shutdownPending, bar)
				if replacementBar, ok := s.waitBars[bar]; ok {
					heap.Push(s.bHeap, replacementBar)
					s.heapUpdated = true
					delete(s.waitBars, bar)
				}
				if frame.removeOnComplete {
					s.heapUpdated = true
					return
				}
			}
			heap.Push(s.bHeap, bar)
		}()
		cw.ReadFrom(frame.rd)
		lineCount += frame.extendedLines + 1
	}

	for i := len(s.shutdownPending) - 1; i >= 0; i-- {
		close(s.shutdownPending[i].shutdown)
		s.shutdownPending = s.shutdownPending[:i]
	}

	return cw.Flush(lineCount)
}

func (s *pState) manualOrTick() (<-chan time.Time, func()) {
	if s.manualRefreshCh != nil {
		return s.manualRefreshCh, func() {}
	}
	ticker := time.NewTicker(s.rr)
	return ticker.C, ticker.Stop
}

func (s *pState) updateSyncMatrix() {
	s.pMatrix = make(map[int][]chan int)
	s.aMatrix = make(map[int][]chan int)
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := (*s.bHeap)[i]
		table := bar.wSyncTable()
		pRow, aRow := table[0], table[1]

		for i, ch := range pRow {
			s.pMatrix[i] = append(s.pMatrix[i], ch)
		}

		for i, ch := range aRow {
			s.aMatrix[i] = append(s.aMatrix[i], ch)
		}
	}
}

func syncWidth(matrix map[int][]chan int) {
	for _, column := range matrix {
		column := column
		go func() {
			var maxWidth int
			for _, ch := range column {
				w := <-ch
				if w > maxWidth {
					maxWidth = w
				}
			}
			for _, ch := range column {
				ch <- maxWidth
			}
		}()
	}
}

func fanInRefreshSrc(done <-chan struct{}, channels ...<-chan time.Time) <-chan time.Time {
	var wg sync.WaitGroup
	multiplexedStream := make(chan time.Time)

	multiplex := func(c <-chan time.Time) {
		defer wg.Done()
		for {
			select {
			case v := <-c:
				multiplexedStream <- v
			case <-done:
				return
			}
		}
	}

	wg.Add(len(channels))
	for _, c := range channels {
		go multiplex(c)
	}

	go func() {
		wg.Wait()
		close(multiplexedStream)
	}()

	return multiplexedStream
}
<commit_msg>case bar.done<commit_after>package mpb

import (
	"container/heap"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v4/cwriter"
)

const (
	// default RefreshRate
	prr = 120 * time.Millisecond
	// default width
	pwidth = 80
)

// Progress represents the container that renders Progress bars
type Progress struct {
	uwg          *sync.WaitGroup
	cwg          *sync.WaitGroup
	bwg          *sync.WaitGroup
	operateState chan func(*pState)
	done         chan struct{}
}

type pState struct {
	bHeap           *priorityQueue
	shutdownPending []*Bar
	heapUpdated     bool
	idCounter       int
	width           int
	rr              time.Duration
	pMatrix         map[int][]chan int
	aMatrix         map[int][]chan int
	forceRefreshCh  chan time.Time
	output          io.Writer

	// following are provided/overrided by user
	ctx              context.Context
	uwg              *sync.WaitGroup
	manualRefreshCh  <-chan time.Time
	shutdownNotifier chan struct{}
	waitBars         map[*Bar]*Bar
	debugOut         io.Writer
}

// New creates new Progress instance, which orchestrates bars rendering
// process. Accepts mpb.ContainerOption funcs for customization.
func New(options ...ContainerOption) *Progress {
	pq := make(priorityQueue, 0)
	heap.Init(&pq)

	s := &pState{
		ctx:            context.Background(),
		bHeap:          &pq,
		width:          pwidth,
		rr:             prr,
		waitBars:       make(map[*Bar]*Bar),
		debugOut:       ioutil.Discard,
		forceRefreshCh: make(chan time.Time),
		output:         os.Stdout,
	}

	for _, opt := range options {
		if opt != nil {
			opt(s)
		}
	}

	p := &Progress{
		uwg:          s.uwg,
		cwg:          new(sync.WaitGroup),
		bwg:          new(sync.WaitGroup),
		operateState: make(chan func(*pState)),
		done:         make(chan struct{}),
	}
	p.cwg.Add(1)
	go p.serve(s, cwriter.New(s.output))
	return p
}

// AddBar creates a new progress bar and adds to the container.
func (p *Progress) AddBar(total int64, options ...BarOption) *Bar {
	return p.Add(total, newDefaultBarFiller(), options...)
}

// AddSpinner creates a new spinner bar and adds to the container.
func (p *Progress) AddSpinner(total int64, alignment SpinnerAlignment, options ...BarOption) *Bar {
	filler := &spinnerFiller{
		frames:    defaultSpinnerStyle,
		alignment: alignment,
	}
	return p.Add(total, filler, options...)
}

// Add creates a bar which renders itself by provided filler.
func (p *Progress) Add(total int64, filler Filler, options ...BarOption) *Bar {
	p.bwg.Add(1)
	result := make(chan *Bar)
	select {
	case p.operateState <- func(s *pState) {
		b := newBar(s.ctx, p.bwg, filler, s.idCounter, s.width, total, options...)
		if b.runningBar != nil {
			s.waitBars[b.runningBar] = b
		} else {
			heap.Push(s.bHeap, b)
			s.heapUpdated = true
		}
		s.idCounter++
		result <- b
	}:
		return <-result
	case <-p.done:
		p.bwg.Done()
		return nil
	}
}

// Abort is only effective while bar progress is running, it means
// remove bar now without waiting for its completion. If bar is already
// completed, there is nothing to abort. If you need to remove bar
// after completion, use BarRemoveOnComplete BarOption.
func (p *Progress) Abort(b *Bar, remove bool) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		if remove {
			s.heapUpdated = heap.Remove(s.bHeap, b.index) != nil
		}
		s.shutdownPending = append(s.shutdownPending, b)
	}:
	case <-p.done:
	}
}

// UpdateBarPriority provides a way to change bar's order position.
// Zero is highest priority, i.e. bar will be on top.
func (p *Progress) UpdateBarPriority(b *Bar, priority int) {
	select {
	case p.operateState <- func(s *pState) { s.bHeap.update(b, priority) }:
	case <-p.done:
	}
}

// BarCount returns bars count
func (p *Progress) BarCount() int {
	result := make(chan int, 1)
	select {
	case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }:
		return <-result
	case <-p.done:
		return 0
	}
}

// Wait waits far all bars to complete and finally shutdowns container.
// After this method has been called, there is no way to reuse *Progress
// instance.
func (p *Progress) Wait() {
	if p.uwg != nil {
		// wait for user wg
		p.uwg.Wait()
	}

	// wait for bars to quit, if any
	p.bwg.Wait()

	close(p.done)

	// wait for container to quit
	p.cwg.Wait()
}

func (p *Progress) serve(s *pState, cw *cwriter.Writer) {
	defer p.cwg.Done()

	manualOrTickCh, cleanUp := s.manualOrTick()
	defer cleanUp()

	refreshCh := fanInRefreshSrc(p.done, s.forceRefreshCh, manualOrTickCh)

	for {
		select {
		case op := <-p.operateState:
			op(s)
		case _, ok := <-refreshCh:
			if !ok {
				if s.shutdownNotifier != nil {
					close(s.shutdownNotifier)
				}
				return
			}
			if err := s.render(cw); err != nil {
				fmt.Fprintf(s.debugOut, "[mpb] %s %v\n", time.Now(), err)
			}
		}
	}
}

func (s *pState) render(cw *cwriter.Writer) error {
	if s.heapUpdated {
		s.updateSyncMatrix()
		s.heapUpdated = false
	}
	syncWidth(s.pMatrix)
	syncWidth(s.aMatrix)

	tw, err := cw.GetWidth()
	if err != nil {
		tw = s.width
	}
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := (*s.bHeap)[i]
		go bar.render(s.debugOut, tw)
	}

	return s.flush(cw)
}

func (s *pState) flush(cw *cwriter.Writer) error {
	var lineCount int
	for s.bHeap.Len() > 0 {
		bar := heap.Pop(s.bHeap).(*Bar)
		frame := <-bar.bFrameCh
		defer func() {
			if frame.toShutdown {
				go func() {
					// force next refresh, so it will be triggered either by ticker or by
					// this goroutine, whichever comes first
					select {
					case s.forceRefreshCh <- time.Now():
					case <-bar.done:
					}
				}()
				// shutdown at next flush, in other words decrement underlying WaitGroup
				// only after the bar with completed state has been flushed. this
				// ensures no bar ends up with less than 100% rendered.
				s.shutdownPending = append(s.shutdownPending, bar)
				if replacementBar, ok := s.waitBars[bar]; ok {
					heap.Push(s.bHeap, replacementBar)
					s.heapUpdated = true
					delete(s.waitBars, bar)
				}
				if frame.removeOnComplete {
					s.heapUpdated = true
					return
				}
			}
			heap.Push(s.bHeap, bar)
		}()
		cw.ReadFrom(frame.rd)
		lineCount += frame.extendedLines + 1
	}

	for i := len(s.shutdownPending) - 1; i >= 0; i-- {
		close(s.shutdownPending[i].shutdown)
		s.shutdownPending = s.shutdownPending[:i]
	}

	return cw.Flush(lineCount)
}

func (s *pState) manualOrTick() (<-chan time.Time, func()) {
	if s.manualRefreshCh != nil {
		return s.manualRefreshCh, func() {}
	}
	ticker := time.NewTicker(s.rr)
	return ticker.C, ticker.Stop
}

func (s *pState) updateSyncMatrix() {
	s.pMatrix = make(map[int][]chan int)
	s.aMatrix = make(map[int][]chan int)
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := (*s.bHeap)[i]
		table := bar.wSyncTable()
		pRow, aRow := table[0], table[1]

		for i, ch := range pRow {
			s.pMatrix[i] = append(s.pMatrix[i], ch)
		}

		for i, ch := range aRow {
			s.aMatrix[i] = append(s.aMatrix[i], ch)
		}
	}
}

func syncWidth(matrix map[int][]chan int) {
	for _, column := range matrix {
		column := column
		go func() {
			var maxWidth int
			for _, ch := range column {
				w := <-ch
				if w > maxWidth {
					maxWidth = w
				}
			}
			for _, ch := range column {
				ch <- maxWidth
			}
		}()
	}
}

func fanInRefreshSrc(done <-chan struct{}, channels ...<-chan time.Time) <-chan time.Time {
	var wg sync.WaitGroup
	multiplexedStream := make(chan time.Time)

	multiplex := func(c <-chan time.Time) {
		defer wg.Done()
		for {
			select {
			case v := <-c:
				multiplexedStream <- v
			case <-done:
				return
			}
		}
	}

	wg.Add(len(channels))
	for _, c := range channels {
		go multiplex(c)
	}

	go func() {
		wg.Wait()
		close(multiplexedStream)
	}()

	return multiplexedStream
}
<|endoftext|>