<commit_before>/*

	JOTPOT Server
	Version 25E

	Copyright (c) 2016-2017 Jacob O'Toole

	Permission is hereby granted, free of charge, to any person obtaining a copy
	of this software and associated documentation files (the "Software"), to deal
	in the Software without restriction, including without limitation the rights
	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	copies of the Software, and to permit persons to whom the Software is
	furnished to do so, subject to the following conditions:

	The above copyright notice and this permission notice shall be included in all
	copies or substantial portions of the Software.

	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	SOFTWARE.

*/

package main

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"net/http"
)

func panicIfErr(err error) {
	if err != nil {
		panic(err)
	}
}

type handler struct{}

func (m *handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	fmt.Println("Got request")

	//Dial server
	fmt.Println("Dialing server...")
	forward, err := net.Dial("tcp", ":80")
	panicIfErr(err)

	//Write headers etc. to server
	fmt.Println("Writing header to server...")
	err = req.Write(forward)
	panicIfErr(err)

	//Parse as response
	fmt.Println("Parsing server response...")
	forwardResp, err := http.ReadResponse(bufio.NewReader(forward), nil)
	panicIfErr(err)

	//Write headers etc. to client
	fmt.Println("Copying headers...")
	for key, val := range forwardResp.Header {
		//resp.Header().Set(key, val)
		resp.Header().Set(key, val[0])
		if len(val) > 1 {
			for _, val := range val {
				resp.Header().Add(key, val)
			}
		}
	}

	fmt.Println("Writing header to client...")
	resp.WriteHeader(forwardResp.StatusCode)
	panicIfErr(err)

	//Buf to store data in and n already set up
	fmt.Println("Setting stuff up...")
	buf := make([]byte, 1024)
	var n int
	var bClosed bool
	var fClosed bool

	//Body
	for {
		//Client -> Server
		fmt.Println("Client -> Server")
		n, err = req.Body.Read(buf)
		if err == nil {
			forward.Write(buf[:n])
		} else if err == io.EOF {
			bClosed = true
		}

		//Server -> Client
		fmt.Println("Server -> Client")
		n, err = forwardResp.Body.Read(buf)
		if err == nil {
			resp.Write(buf[:n])
		} else if err == io.EOF {
			fClosed = true
		}

		if bClosed && fClosed {
			fmt.Println("We are at the end!")
			forward.Close()
			return
		}
	}
}

func main() {
	http.ListenAndServe(":8081", &handler{})
}
<commit_msg>Added basic directors to the go load balencer and added load balancer headers for the rest of the server<commit_after>/*

	JOTPOT Server
	Version 25E

	Copyright (c) 2016-2017 Jacob O'Toole

	Permission is hereby granted, free of charge, to any person obtaining a copy
	of this software and associated documentation files (the "Software"), to deal
	in the Software without restriction, including without limitation the rights
	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	copies of the Software, and to permit persons to whom the Software is
	furnished to do so, subject to the following conditions:

	The above copyright notice and this permission notice shall be included in all
	copies or substantial portions of the Software.

	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	SOFTWARE.

*/

package main

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
)

//Panic if err is not nil
func panicIfErr(err error) {
	if err != nil {
		panic(err)
	}
}

type form struct {
	regexp     *regexp.Regexp
	serveraddr string
}
type uncompiledform struct {
	regexp     string
	serveraddr string
}
type director struct {
	DefaultAddr string
	DefaultHost string
	Directions  map[string][]form
}
type uncompileddirector struct {
	DefaultAddr string
	DefaultHost string
	Directions  map[string][]uncompiledform
}

type handler struct {
	isHTTPS  bool
	director *director
}

//Returns the server address based on the directions from the director
func (t director) WhereIs(u *url.URL) string {
	//Get relivent directions
	checks, ok := t.Directions[u.Hostname()]

	//Fallback to default host/default address
	if !ok {
		if t.DefaultHost == "" {
			return t.DefaultAddr
		}
		checks, ok = t.Directions[t.DefaultHost]
		if !ok {
			return t.DefaultAddr
		}
	}

	//Test for reg exp matches
	for _, exp := range checks {
		if exp.regexp.MatchString(u.Path) {
			return exp.serveraddr
		}
	}

	//Fallback to default address
	return t.DefaultAddr
}

//Compile uncompiled director
func (t uncompileddirector) Compile() (out director, err error) {
	var compiled *regexp.Regexp

	//Copy basic properties
	out.DefaultAddr = t.DefaultAddr
	out.DefaultHost = t.DefaultHost

	for host, tf := range t.Directions {
		//If it is a nil map, we cannot just add the key
		if out.Directions != nil {
			out.Directions[host] = []form{}
		} else {
			out.Directions = map[string][]form{host: []form{}}
		}

		//Compile all the rules
		for _, exp := range tf {
			compiled, err = regexp.Compile(exp.regexp)
			if err != nil {
				//Return with err if there was an err
				return
			}
			out.Directions[host] = append(out.Directions[host], form{compiled, exp.serveraddr})
		}
	}
	return
}

func (m *handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	fmt.Println("Got request")

	//What server should we dial
	connectTo := m.director.WhereIs(req.URL)
	fmt.Println(connectTo)

	//Dial server
	forward, err := net.Dial("tcp", ":80")
	panicIfErr(err)

	//Add load balancer headers
	req.Header["jp-source-ip"] = []string{req.RemoteAddr}
	if m.isHTTPS {
		req.Header["jp-source-secure"] = []string{"https"}
	} else {
		req.Header["jp-source-secure"] = []string{"http"}
	}

	//Write headers etc. to server
	err = req.Write(forward)
	panicIfErr(err)

	//Parse as response
	forwardResp, err := http.ReadResponse(bufio.NewReader(forward), nil)
	panicIfErr(err)

	//Write headers etc. to client
	for key, val := range forwardResp.Header {
		//resp.Header().Set(key, val)
		resp.Header().Set(key, val[0])
		if len(val) > 1 {
			for _, val := range val {
				resp.Header().Add(key, val)
			}
		}
	}

	resp.WriteHeader(forwardResp.StatusCode)
	panicIfErr(err)

	//Buf to store data in and n already set up
	buf := make([]byte, 1024)
	var n int
	var bClosed bool
	var fClosed bool

	//Body
	for {
		//Client -> Server
		fmt.Println("Client -> Server")
		n, err = req.Body.Read(buf)
		if err == nil {
			forward.Write(buf[:n])
		} else if err == io.EOF {
			bClosed = true
		}

		//Server -> Client
		fmt.Println("Server -> Client")
		n, err = forwardResp.Body.Read(buf)
		if err == nil {
			resp.Write(buf[:n])
		} else if err == io.EOF {
			fClosed = true
		}

		if bClosed && fClosed {
			fmt.Println("We are at the end!")
			forward.Close()
			return
		}
	}
}

func main() {
	//To be used later
	toCompile := uncompileddirector{
		//Default address and host
		"LIMBO",
		"LIMBO",
		//Simple test director
		map[string][]uncompiledform{
			"www.jotpot.co.uk": []uncompiledform{
				uncompiledform{".*", "192.168.1.11:80"},
			},
		},
	}

	//Compile it
	mainDirector, err := toCompile.Compile()
	panicIfErr(err)

	//Set up server
	http.ListenAndServe(":8081", &handler{false, &mainDirector})
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	scanAborted    = errors.New("scan aborted")
	scanInProgress = errors.New("scan already in progress")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	size    int64
	modTime time.Time
}

type scan struct {
	redis           *redisobj
	walkSourceFiles []*filedata
	walkRedisConn   redis.Conn

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *redisobj, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.pool.Get()
	defer conn.Close()

	s.conn = conn

	lockKey := fmt.Sprintf("SCANNING_%s", identifier)

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	lock, err := redis.Bool(conn.Do("SETNX", lockKey, 1))
	if err != nil {
		return err
	}
	if lock {
		// Lock aquired.
		defer conn.Do("DEL", lockKey)
		// Make the key expire automatically in case our process gets killed
		conn.Do("EXPIRE", lockKey, 600)
	} else {
		return scanInProgress
	}

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		s.setLastSync(conn, identifier, false)
		log.Error("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debug("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			SendPublish(conn, MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Info("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	SendPublish(s.conn, MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	Publish(conn, MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(path string, f os.FileInfo, err error) error {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(s.walkRedisConn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1"))
	if err != nil && err != redis.ErrNil {
		return err
	}

	if err != redis.ErrNil && len(properties) >= 3 {
		size, _ := strconv.ParseInt(properties[0], 10, 64)
		modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
		sha1 := properties[2]
		if size != d.size || modTime != d.modTime {
			h, err := hashFile(GetConfig().Repository + d.path)
			if err != nil {
				log.Warning("%s: hashing failed: %s", d.path, err.Error())
			} else {
				log.Info("%s: %s", d.path, h)
				d.sha1 = h
			}
		}
		if d.sha1 == "" {
			d.sha1 = sha1
		}
	}

	s.walkSourceFiles = append(s.walkSourceFiles, d)
	return nil
}

func ScanSource(r *redisobj, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	s.walkRedisConn = s.redis.pool.Get()
	defer s.walkRedisConn.Close()
	if err != nil {
		return fmt.Errorf("redis %s", err.Error())
	}
	defer s.walkRedisConn.Close()

	s.walkSourceFiles = make([]*filedata, 0, 1000)
	defer func() {
		// Reset the slice so it can be garbage collected
		s.walkSourceFiles = nil
	}()

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, s.walkSource)
	if isStopped(stop) {
		return scanAborted
	}
	if err != nil {
		return fmt.Errorf("[source] Error during walking: %s", err.Error())
	}
	log.Info("[source] Indexing the files...")

	s.walkRedisConn.Send("MULTI")

	// Remove any left over
	s.walkRedisConn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(s.walkRedisConn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	s.walkRedisConn.Send("MULTI")
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1)

		// Publish update
		SendPublish(s.walkRedisConn, FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			s.walkRedisConn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			SendPublish(s.walkRedisConn, FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	s.walkRedisConn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Info("[source] Scanned %d files", count)

	return nil
}
<commit_msg>Set the last scan time before the actual scanning to avoid multiple scan intents<commit_after>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	scanAborted    = errors.New("scan aborted")
	scanInProgress = errors.New("scan already in progress")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	size    int64
	modTime time.Time
}

type scan struct {
	redis           *redisobj
	walkSourceFiles []*filedata
	walkRedisConn   redis.Conn

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *redisobj, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.pool.Get()
	defer conn.Close()

	s.conn = conn

	lockKey := fmt.Sprintf("SCANNING_%s", identifier)

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	lock, err := redis.Bool(conn.Do("SETNX", lockKey, 1))
	if err != nil {
		return err
	}
	if lock {
		// Lock aquired.
		defer conn.Do("DEL", lockKey)
		// Make the key expire automatically in case our process gets killed
		conn.Do("EXPIRE", lockKey, 600)
	} else {
		return scanInProgress
	}

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Error("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debug("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			SendPublish(conn, MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Info("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	SendPublish(s.conn, MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	Publish(conn, MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(path string, f os.FileInfo, err error) error {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(s.walkRedisConn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1"))
	if err != nil && err != redis.ErrNil {
		return err
	}

	if err != redis.ErrNil && len(properties) >= 3 {
		size, _ := strconv.ParseInt(properties[0], 10, 64)
		modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
		sha1 := properties[2]
		if size != d.size || modTime != d.modTime {
			h, err := hashFile(GetConfig().Repository + d.path)
			if err != nil {
				log.Warning("%s: hashing failed: %s", d.path, err.Error())
			} else {
				log.Info("%s: %s", d.path, h)
				d.sha1 = h
			}
		}
		if d.sha1 == "" {
			d.sha1 = sha1
		}
	}

	s.walkSourceFiles = append(s.walkSourceFiles, d)
	return nil
}

func ScanSource(r *redisobj, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	s.walkRedisConn = s.redis.pool.Get()
	defer s.walkRedisConn.Close()
	if err != nil {
		return fmt.Errorf("redis %s", err.Error())
	}
	defer s.walkRedisConn.Close()

	s.walkSourceFiles = make([]*filedata, 0, 1000)
	defer func() {
		// Reset the slice so it can be garbage collected
		s.walkSourceFiles = nil
	}()

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, s.walkSource)
	if isStopped(stop) {
		return scanAborted
	}
	if err != nil {
		return fmt.Errorf("[source] Error during walking: %s", err.Error())
	}
	log.Info("[source] Indexing the files...")

	s.walkRedisConn.Send("MULTI")

	// Remove any left over
	s.walkRedisConn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(s.walkRedisConn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	s.walkRedisConn.Send("MULTI")
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1)

		// Publish update
		SendPublish(s.walkRedisConn, FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			s.walkRedisConn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			SendPublish(s.walkRedisConn, FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	s.walkRedisConn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Info("[source] Scanned %d files", count)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	quic "github.com/phuslu/quic-go"
	"github.com/phuslu/quic-go/h2quic"
	// quic "github.com/lucas-clemente/quic-go"
	// "github.com/lucas-clemente/quic-go/h2quic"
)

type ScanRecord struct {
	IP         string
	MatchHosts []string
	PingRTT    time.Duration
	SSLRTT     time.Duration

	httpVerifyTimeout time.Duration
}
type ScanRecordArray []*ScanRecord

type ScanOptions struct {
	Config *GScanConfig

	recordMutex sync.Mutex
	hostsMutex  sync.Mutex
	inputHosts  HostIPTable
	records     ScanRecordArray

	scanCounter int32
}

func (options *ScanOptions) AddRecord(rec *ScanRecord) {
	options.recordMutex.Lock()
	if nil == options.records {
		options.records = make(ScanRecordArray, 0)
	}
	options.records = append(options.records, rec)
	options.recordMutex.Unlock()
	log.Printf("Found a record: IP=%s, SSLRTT=%fs\n", rec.IP, rec.SSLRTT.Seconds())
}

func (options *ScanOptions) IncScanCounter() {
	atomic.AddInt32(&(options.scanCounter), 1)
	if options.scanCounter%1000 == 0 {
		log.Printf("Scanned %d IPs, Found %d records\n", options.scanCounter, options.RecordSize())
	}
}

func (options *ScanOptions) RecordSize() int {
	options.recordMutex.Lock()
	defer options.recordMutex.Unlock()
	return len(options.records)
}

func (options *ScanOptions) SSLMatchHosts(conn *tls.Conn) []string {
	hosts := make([]string, 0)
	options.hostsMutex.Lock()
	for _, host := range options.inputHosts {
		testhost := host.Host
		if strings.Contains(testhost, ".appspot.com") {
			testhost = "appengine.google.com"
		} else if strings.Contains(testhost, "ggpht.com") {
			testhost = "googleusercontent.com"
		} else if strings.Contains(testhost, ".books.google.com") {
			testhost = "books.google.com"
		} else if strings.Contains(testhost, ".googleusercontent.com") {
			testhost = "googleusercontent.com"
		}
		if conn.VerifyHostname(testhost) == nil {
			hosts = append(hosts, host.Host)
		}
	}
	options.hostsMutex.Unlock()
	dest := make([]string, len(hosts))
	perm := rand.Perm(len(hosts))
	for i, v := range perm {
		dest[v] = hosts[i]
	}
	hosts = dest
	return hosts
}

func (options *ScanOptions) HaveHostInRecords(host string) bool {
	options.hostsMutex.Lock()
	defer options.hostsMutex.Unlock()
	_, exists := options.inputHosts[host]
	return !exists
}

func (options *ScanOptions) RemoveFromInputHosts(hosts []string) {
	options.hostsMutex.Lock()
	for _, host := range hosts {
		delete(options.inputHosts, host)
	}
	options.hostsMutex.Unlock()
}

func matchHostnames(pattern, host string) bool {
	if len(pattern) == 0 || len(host) == 0 {
		return false
	}
	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}
	return true
}

var (
	tlsCfg = &tls.Config{
		InsecureSkipVerify: true,
	}
)

func testip_once(ip string, options *ScanOptions, record *ScanRecord) bool {
	start := time.Now()

	pingRTT := (options.Config.ScanMinPingRTT + options.Config.ScanMaxPingRTT) / 2
	if options.Config.VerifyPing {
		err := Ping(ip, options.Config.ScanMaxPingRTT)
		if err != nil {
			return false
		}
		end := time.Now()
		if nil == err {
			if options.Config.ScanMinPingRTT > 0 && end.Sub(start) < options.Config.ScanMinPingRTT {
				return false
			}
			pingRTT = end.Sub(start)
		}
	}
	record.PingRTT = record.PingRTT + pingRTT

	addr := net.JoinHostPort(ip, "443")

	start = time.Now()
	success := make(chan bool, 5)

	go func() {
		<-time.After(options.Config.ScanMaxSSLRTT + 500*time.Millisecond)
		success <- false
	}()

	var quicSessn quic.Session
	defer func() {
		if quicSessn != nil {
			quicSessn.Close(nil)
		}
	}()

	tr := &h2quic.RoundTripper{DisableCompression: true}
	defer tr.Close()

	quicCfg := &quic.Config{
		HandshakeTimeout: options.Config.ScanMaxSSLRTT - 500*time.Millisecond,
		// IdleTimeout:      options.Config.ScanMaxSSLRTT,
		KeepAlive: false,
	}

	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return false
	}
	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return false
	}

	udpConn.SetDeadline(time.Now().Add(options.Config.ScanMaxSSLRTT))
	defer udpConn.Close()

	go func(chan bool) {
		var err error
		// quicSessn, err = quic.DialAddr(addr, tlsCfg, quicCfg)
		quicSessn, err = quic.Dial(udpConn, udpAddr, addr, tlsCfg, quicCfg)
		if err != nil {
			success <- false
			return
		}
		tr.DialAddr = func(hostname string, tlsConfig *tls.Config, config *quic.Config) (quic.Session, error) {
			return quicSessn, err
		}
		hclient := &http.Client{
			Transport: tr,
		}

		for _, verifyHost := range options.Config.ScanGoogleIP.HTTPVerifyHosts {
			req, _ := http.NewRequest(http.MethodHead, "https://"+verifyHost, nil)
			req.Close = true
			resp, err := hclient.Do(req)
			// 无论是否有err，最好都close一下
			if resp != nil && resp.Body != nil {
				io.Copy(ioutil.Discard, resp.Body)
				resp.Body.Close()
			}
			if nil != err || resp.StatusCode >= 400 {
				success <- false
				return
			}
		}
		success <- true
	}(success)

	if <-success == false {
		return false
	}

	sslRTT := time.Since(start)
	record.SSLRTT = record.SSLRTT + sslRTT
	return true
}

func testip(ip string, options *ScanOptions) *ScanRecord {
	record := new(ScanRecord)
	record.IP = ip
	for i := 0; i < options.Config.ScanCountPerIP; i++ {
		if !testip_once(ip, options, record) {
			return nil
		}
	}
	record.PingRTT = record.PingRTT / time.Duration(options.Config.ScanCountPerIP)
	record.SSLRTT = record.SSLRTT / time.Duration(options.Config.ScanCountPerIP)
	return record
}

func testip_worker(ch chan string, options *ScanOptions, wg *sync.WaitGroup) {
	for ip := range ch {
		record := testip(ip, options)
		if nil != record {
			if !options.Config.scanIP {
				options.RemoveFromInputHosts(record.MatchHosts)
			}
			options.AddRecord(record)
		}
		options.IncScanCounter()
	}
	wg.Done()
}
<commit_msg>添加证书验证<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"crypto/tls"
	"encoding/base64"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	quic "github.com/phuslu/quic-go"
	"github.com/phuslu/quic-go/h2quic"
	// quic "github.com/lucas-clemente/quic-go"
	// "github.com/lucas-clemente/quic-go/h2quic"
)

type ScanRecord struct {
	IP         string
	MatchHosts []string
	PingRTT    time.Duration
	SSLRTT     time.Duration

	httpVerifyTimeout time.Duration
}
type ScanRecordArray []*ScanRecord

type ScanOptions struct {
	Config *GScanConfig

	recordMutex sync.Mutex
	hostsMutex  sync.Mutex
	inputHosts  HostIPTable
	records     ScanRecordArray

	scanCounter int32
}

func (options *ScanOptions) AddRecord(rec *ScanRecord) {
	options.recordMutex.Lock()
	if nil == options.records {
		options.records = make(ScanRecordArray, 0)
	}
	options.records = append(options.records, rec)
	options.recordMutex.Unlock()
	log.Printf("Found a record: IP=%s, SSLRTT=%fs\n", rec.IP, rec.SSLRTT.Seconds())
}

func (options *ScanOptions) IncScanCounter() {
	atomic.AddInt32(&(options.scanCounter), 1)
	if options.scanCounter%1000 == 0 {
		log.Printf("Scanned %d IPs, Found %d records\n", options.scanCounter, options.RecordSize())
	}
}

func (options *ScanOptions) RecordSize() int {
	options.recordMutex.Lock()
	defer options.recordMutex.Unlock()
	return len(options.records)
}

func (options *ScanOptions) SSLMatchHosts(conn *tls.Conn) []string {
	hosts := make([]string, 0)
	options.hostsMutex.Lock()
	for _, host := range options.inputHosts {
		testhost := host.Host
		if strings.Contains(testhost, ".appspot.com") {
			testhost = "appengine.google.com"
		} else if strings.Contains(testhost, "ggpht.com") {
			testhost = "googleusercontent.com"
		} else if strings.Contains(testhost, ".books.google.com") {
			testhost = "books.google.com"
		} else if strings.Contains(testhost, ".googleusercontent.com") {
			testhost = "googleusercontent.com"
		}
		if conn.VerifyHostname(testhost) == nil {
			hosts = append(hosts, host.Host)
		}
	}
	options.hostsMutex.Unlock()
	dest := make([]string, len(hosts))
	perm := rand.Perm(len(hosts))
	for i, v := range perm {
		dest[v] = hosts[i]
	}
	hosts = dest
	return hosts
}

func (options *ScanOptions) HaveHostInRecords(host string) bool {
	options.hostsMutex.Lock()
	defer options.hostsMutex.Unlock()
	_, exists := options.inputHosts[host]
	return !exists
}

func (options *ScanOptions) RemoveFromInputHosts(hosts []string) {
	options.hostsMutex.Lock()
	for _, host := range hosts {
		delete(options.inputHosts, host)
	}
	options.hostsMutex.Unlock()
}

func matchHostnames(pattern, host string) bool {
	if len(pattern) == 0 || len(host) == 0 {
		return false
	}
	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}
	return true
}

var (
	tlsCfg = &tls.Config{
		InsecureSkipVerify: true,
	}
	g2pkp, _ = base64.StdEncoding.DecodeString("7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y=")
)

func testip_once(ip string, options *ScanOptions, record *ScanRecord) bool {
	start := time.Now()

	pingRTT := (options.Config.ScanMinPingRTT + options.Config.ScanMaxPingRTT) / 2
	if options.Config.VerifyPing {
		err := Ping(ip, options.Config.ScanMaxPingRTT)
		if err != nil {
			return false
		}
		end := time.Now()
		if nil == err {
			if options.Config.ScanMinPingRTT > 0 && end.Sub(start) < options.Config.ScanMinPingRTT {
				return false
			}
			pingRTT = end.Sub(start)
		}
	}
	record.PingRTT = record.PingRTT + pingRTT

	addr := net.JoinHostPort(ip, "443")

	start = time.Now()
	success := make(chan bool, 5)

	go func() {
		<-time.After(options.Config.ScanMaxSSLRTT + 500*time.Millisecond)
		success <- false
	}()

	var quicSessn quic.Session
	defer func() {
		if quicSessn != nil {
			quicSessn.Close(nil)
		}
	}()

	tr := &h2quic.RoundTripper{DisableCompression: true}
	defer tr.Close()

	quicCfg := &quic.Config{
		HandshakeTimeout: options.Config.ScanMaxSSLRTT - 500*time.Millisecond,
		// IdleTimeout:      options.Config.ScanMaxSSLRTT,
		KeepAlive: false,
	}

	udpAddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return false
	}
	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return false
	}

	udpConn.SetDeadline(time.Now().Add(options.Config.ScanMaxSSLRTT))
	defer udpConn.Close()

	go func(chan bool) {
		var err error
		// quicSessn, err = quic.DialAddr(addr, tlsCfg, quicCfg)
		quicSessn, err = quic.Dial(udpConn, udpAddr, addr, tlsCfg, quicCfg)
		if err != nil {
			success <- false
			return
		}
		// 证书验证
		cs := quicSessn.ConnectionState()
		if cs == nil {
			success <- false
			return
		}
		pcs := cs.PeerCertificates
		if len(pcs) < 2 {
			success <- false
			return
		}
		pkp := sha256.Sum256(pcs[1].RawSubjectPublicKeyInfo)
		if !bytes.Equal(g2pkp, pkp[:]) {
			success <- false
			return
		}

		tr.DialAddr = func(hostname string, tlsConfig *tls.Config, config *quic.Config) (quic.Session, error) {
			return quicSessn, err
		}
		hclient := &http.Client{
			Transport: tr,
		}

		for _, verifyHost := range options.Config.ScanGoogleIP.HTTPVerifyHosts {
			req, _ := http.NewRequest(http.MethodHead, "https://"+verifyHost, nil)
			req.Close = true
			resp, err := hclient.Do(req)
			// 无论是否有err，最好都close一下
			if resp != nil && resp.Body != nil {
				io.Copy(ioutil.Discard, resp.Body)
				resp.Body.Close()
			}
			if nil != err || resp.StatusCode >= 400 {
				success <- false
				return
			}
		}
		success <- true
	}(success)

	if <-success == false {
		return false
	}

	sslRTT := time.Since(start)
	record.SSLRTT = record.SSLRTT + sslRTT
	return true
}

func testip(ip string, options *ScanOptions) *ScanRecord {
	record := new(ScanRecord)
	record.IP = ip
	for i := 0; i < options.Config.ScanCountPerIP; i++ {
		if !testip_once(ip, options, record) {
			return nil
		}
	}
	record.PingRTT = record.PingRTT / time.Duration(options.Config.ScanCountPerIP)
	record.SSLRTT = record.SSLRTT / time.Duration(options.Config.ScanCountPerIP)
	return record
}

func testip_worker(ch chan string, options *ScanOptions, wg *sync.WaitGroup) {
	for ip := range ch {
		record := testip(ip, options)
		if nil != record {
			if !options.Config.scanIP {
				options.RemoveFromInputHosts(record.MatchHosts)
			}
			options.AddRecord(record)
		}
		options.IncScanCounter()
	}
	wg.Done()
}
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file provides a REPL.

package golisp

import (
	"container/list"
	"fmt"
)

func Repl() {
	IsInteractive = true
	fmt.Printf("Welcome to GoLisp\n")
	fmt.Printf("Copyright 2014 SteelSeries\n")
	fmt.Printf("Evaluate '(quit)' to exit.\n\n")
	prompt := "> "
	LoadHistoryFromFile(".golisp_history")
	lastInput := ""
	for true {
		defer func() {
			if x := recover(); x != nil {
				fmt.Printf("Don't Panic! %v\n", x)
			}
		}()
		DebugCurrentFrame = nil
		DebugSingleStep = false
		DebugEvalInDebugRepl = false
		Global.CurrentCode = list.New()
		input := *ReadLine(&prompt)
		if input != "" {
			if input != lastInput {
				AddHistory(input)
			}
			lastInput = input
			code, err := Parse(input)
			if err != nil {
				fmt.Printf("Error: %s\n", err)
			} else {
				d, err := Eval(code, Global)
				if err != nil {
					fmt.Printf("Error in evaluation: %s\n", err)
					if DebugOnError {
						DebugRepl(DebugErrorEnv)
					}
				} else {
					fmt.Printf("==> %s\n", String(d))
				}
			}
		}
	}
}
<commit_msg>Handle C-d to clean up and exit (calls the implementation of quit)<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file provides a REPL.

package golisp

import (
	"container/list"
	"fmt"
)

func Repl() {
	IsInteractive = true
	fmt.Printf("Welcome to GoLisp\n")
	fmt.Printf("Copyright 2014 SteelSeries\n")
	fmt.Printf("Evaluate '(quit)' to exit.\n\n")
	prompt := "> "
	LoadHistoryFromFile(".golisp_history")
	lastInput := ""
	for true {
		defer func() {
			if x := recover(); x != nil {
				fmt.Printf("Don't Panic! %v\n", x)
			}
		}()
		DebugCurrentFrame = nil
		DebugSingleStep = false
		DebugEvalInDebugRepl = false
		Global.CurrentCode = list.New()
		inputp := ReadLine(&prompt)
		fmt.Printf("inputp: %v\n", inputp)
		if inputp == nil {
			QuitImpl(nil, nil)
		} else {
			input := *inputp
			fmt.Printf("input: <%s>\n", inputp)
			if input != "" {
				if input != lastInput {
					AddHistory(input)
				}
				lastInput = input
				code, err := Parse(input)
				if err != nil {
					fmt.Printf("Error: %s\n", err)
				} else {
					d, err := Eval(code, Global)
					if err != nil {
						fmt.Printf("Error in evaluation: %s\n", err)
						if DebugOnError {
							DebugRepl(DebugErrorEnv)
						}
					} else {
						fmt.Printf("==> %s\n", String(d))
					}
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/libgit2/git2go"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

const (
	DefaultCloneBasePath = "/tmp/risu/repository/"
)

var ren = render.New()

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	currentTime := time.Now()
	build := schema.Build{
		ID:           uuid.NewUUID(),
		SourceRepo:   opts.SourceRepo,
		SourceBranch: opts.SourceBranch,
		Name:         opts.Name,
		Dockerfile:   opts.Dockerfile,
		Status:       "building",
		CreatedAt:    currentTime,
		UpdatedAt:    currentTime,
	}

	reg := registry.NewRegistry("localfs", "")
	err = reg.Set(build)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusCreated, build)

	// git clone
	err = gitClone(build)
	if err != nil {
		log.Fatal(err)
	}
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reg := registry.NewRegistry("localfs", "")
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	reg := registry.NewRegistry("localfs", "")
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
		return
	}
	ren.JSON(w, http.StatusOK, build)
}

// Clone run "git clone <repository_URL>" and "git checkout branch"
func gitClone(build schema.Build) error {
	basePath := DefaultCloneBasePath
	if _, err := os.Stat(basePath); err != nil {
		os.MkdirAll(basePath, 0755)
	}

	// htpps://<token>@github.com/<SourceRepo>.git
	cloneURL := "https://" + os.Getenv("GITHUB_ACCESS_TOKEN") + "@github.com/" + build.SourceRepo + ".git"

	// debug
	fmt.Println(cloneURL)

	clonePath := basePath + build.SourceRepo

	// debug
	fmt.Println(clonePath)

	_, err := git.Clone(cloneURL, clonePath, &git.CloneOptions{})
	if err != nil {
		return err
	}
	return nil
}

func setUpServer() *negroni.Negroni {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	return n
}

func main() {
	if os.Getenv("GITHUB_ACCESS_TOKEN") == "" {
		log.Fatal("Please provide 'GITHUB_ACCESS_TOKEN' through environment")
		os.Exit(1)
	}
	n := setUpServer()
	n.Run(":8080")
}
<commit_msg>Change option in Clone<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/libgit2/git2go"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

const (
	DefaultCloneBasePath = "/tmp/risu/repository/"
)

var ren = render.New()

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}

	if opts.SourceBranch == "" {
		opts.SourceBranch = "master"
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	currentTime := time.Now()
	build := schema.Build{
		ID:           uuid.NewUUID(),
		SourceRepo:   opts.SourceRepo,
		SourceBranch: opts.SourceBranch,
		Name:         opts.Name,
		Dockerfile:   opts.Dockerfile,
		Status:       "building",
		CreatedAt:    currentTime,
		UpdatedAt:    currentTime,
	}

	reg := registry.NewRegistry("localfs", "")
	err = reg.Set(build)
	if err != nil {
		log.Fatal(err)
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusCreated, build)

	// git clone
	err = gitClone(build)
	if err != nil {
		log.Fatal(err)
	}
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reg := registry.NewRegistry("localfs", "")
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
		return
	}
	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	reg := registry.NewRegistry("localfs", "")
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
		return
	}
	ren.JSON(w, http.StatusOK, build)
}

// Clone run "git clone <repository_URL>" and "git checkout branch"
func gitClone(build schema.Build) error {
	basePath := DefaultCloneBasePath
	if _, err := os.Stat(basePath); err != nil {
		os.MkdirAll(basePath, 0755)
	}

	// htpps://<token>@github.com/<SourceRepo>.git
	cloneURL := "https://" + os.Getenv("GITHUB_ACCESS_TOKEN") + "@github.com/" + build.SourceRepo + ".git"

	// debug
	fmt.Println(cloneURL)

	clonePath := basePath + build.SourceRepo

	// debug
	fmt.Println(clonePath)

	_, err := git.Clone(cloneURL, clonePath, &git.CloneOptions{CheckoutBranch: build.SourceBranch})
	if err != nil {
		return err
	}
	return nil
}

func setUpServer() *negroni.Negroni {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	return n
}

func main() {
	if os.Getenv("GITHUB_ACCESS_TOKEN") == "" {
		log.Fatal("Please provide 'GITHUB_ACCESS_TOKEN' through environment")
		os.Exit(1)
	}
	n := setUpServer()
	n.Run(":8080")
}
<|endoftext|>


<commit_before>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter.

	Autoreader processes a package of go files, searching for structs that
	have a comment immediately above their declaration that begins with
	"+autoreader". For each such struct, it creates a Reader() and
	PropertyReader() method that just use boardgame.DefaultReader and
	boardgame.DefaultReadSetter.

	If you want only a reader or only a readsetter for a given struct, include
	the keyword "reader" or "readsetter", like so: "+autoreader reader"

	You can configure which package to process and where to write output via
	command-line flags. By default it processes the current package and writes
	its output to auto_reader.go, overwriting whatever file was there before.
	See command-line options by passing -h.

	The defaults are set reasonably so that you can use go:generate very
	easily. See examplepkg/ for a very simple example.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"go/format"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"text/template"
)

var headerTemplate *template.Template
var structHeaderTemplate *template.Template
var readerTemplate *template.Template
var readSetterTemplate *template.Template

const magicDocLinePrefix = "+autoreader"

type appOptions struct {
	OutputFile       string
	PackageDirectory string
	PrintToConsole   bool
	Help             bool
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func init() {
	headerTemplate = template.Must(template.New("header").Parse(headerTemplateText))
	structHeaderTemplate = template.Must(template.New("structHeader").Parse(structHeaderTemplateText))
	readerTemplate = template.Must(template.New("reader").Parse(readerTemplateText))
	readSetterTemplate = template.Must(template.New("readsetter").Parse(readSetterTemplateText))
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", ".", "Which package to process")
	options.flagSet.BoolVar(&options.Help, "h", false, "If set, print help message and quit.")
	options.flagSet.BoolVar(&options.PrintToConsole, "print", false, "If true, will print result to console instead of writing to out.")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stdout, os.Stderr)
}

func process(options *appOptions, out io.ReadWriter, errOut io.ReadWriter) {

	if options.Help {
		options.flagSet.SetOutput(out)
		options.flagSet.PrintDefaults()
		return
	}

	output, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	if options.PrintToConsole {
		fmt.Fprintln(out, output)
	} else {
		ioutil.WriteFile(options.OutputFile, []byte(output), 0644)
	}

}

func processPackage(location string) (output string, err error) {
	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", errors.New("Couldn't parse sources: " + err.Error())
	}

	haveOutputHeader := false

	for _, theStruct := range sources.Structs {

		if !haveOutputHeader {
			output += headerForPackage(theStruct.PackageName)
			haveOutputHeader = true
		}

		outputReader, outputReadSetter := structConfig(theStruct.DocLines)

		if outputReader || outputReadSetter {
			output += headerForStruct(theStruct.Name)
		}

		if outputReader {
			output += readerForStruct(theStruct.Name)
		}
		if outputReadSetter {
			output += readSetterForStruct(theStruct.Name)
		}
	}

	formattedBytes, err := format.Source([]byte(output))

	if err != nil {
		return "", errors.New("Couldn't go fmt code: " + err.Error())
	}

	return string(formattedBytes), nil
}

func structConfig(docLines []string) (outputReader bool, outputReadSetter bool) {

	for _, docLine := range docLines {
		docLine = strings.ToLower(docLine)
		docLine = strings.TrimPrefix(docLine, "//")
		docLine = strings.TrimSpace(docLine)
		if !strings.HasPrefix(docLine, magicDocLinePrefix) {
			continue
		}
		docLine = strings.TrimPrefix(docLine, magicDocLinePrefix)
		docLine = strings.TrimSpace(docLine)

		switch docLine {
		case "":
			return true, true
		case "both":
			return true, true
		case "reader":
			return true, false
		case "readsetter":
			return false, true
		}

	}
	return false, false
}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}

func headerForPackage(packageName string) string {
	return templateOutput(headerTemplate, map[string]string{
		"packageName": packageName,
	})
}

func headerForStruct(structName string) string {
	return templateOutput(structHeaderTemplate, map[string]string{
		"structName": structName,
	})
}

func readerForStruct(structName string) string {

	return templateOutput(readerTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})

}

func readSetterForStruct(structName string) string {
	return templateOutput(readSetterTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})
}

const headerTemplateText = `/************************************
 *
 * This file contains auto-generated methods to help certain structs
 * implement boardgame.SubState and boardgame.MutableSubState. It was
 * generated by autoreader.
 *
 * DO NOT EDIT by hand.
 *
 ************************************/
package {{.packageName}}

import (
	"github.com/jkomoros/boardgame"
)

`

const structHeaderTemplateText = `// Implementation for {{.structName}}

 `

const readerTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) Reader() boardgame.PropertyReader {
	return boardgame.DefaultReader({{.FirstLetter}})
}

`

const readSetterTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) ReadSetter() boardgame.PropertyReadSetter {
	return boardgame.DefaultReadSetter({{.FirstLetter}})
}

`
<commit_msg>Split out import header. Prepping for #310.<commit_after>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter.

	Autoreader processes a package of go files, searching for structs that
	have a comment immediately above their declaration that begins with
	"+autoreader". For each such struct, it creates a Reader() and
	PropertyReader() method that just use boardgame.DefaultReader and
	boardgame.DefaultReadSetter.

	If you want only a reader or only a readsetter for a given struct, include
	the keyword "reader" or "readsetter", like so: "+autoreader reader"

	You can configure which package to process and where to write output via
	command-line flags. By default it processes the current package and writes
	its output to auto_reader.go, overwriting whatever file was there before.
	See command-line options by passing -h.

	The defaults are set reasonably so that you can use go:generate very
	easily. See examplepkg/ for a very simple example.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"go/format"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"text/template"
)

var headerTemplate *template.Template
var structHeaderTemplate *template.Template
var readerTemplate *template.Template
var readSetterTemplate *template.Template

const magicDocLinePrefix = "+autoreader"

type appOptions struct {
	OutputFile       string
	PackageDirectory string
	PrintToConsole   bool
	Help             bool
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func init() {
	headerTemplate = template.Must(template.New("header").Parse(headerTemplateText))
	structHeaderTemplate = template.Must(template.New("structHeader").Parse(structHeaderTemplateText))
	readerTemplate = template.Must(template.New("reader").Parse(readerTemplateText))
	readSetterTemplate = template.Must(template.New("readsetter").Parse(readSetterTemplateText))
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", ".", "Which package to process")
	options.flagSet.BoolVar(&options.Help, "h", false, "If set, print help message and quit.")
	options.flagSet.BoolVar(&options.PrintToConsole, "print", false, "If true, will print result to console instead of writing to out.")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stdout, os.Stderr)
}

func process(options *appOptions, out io.ReadWriter, errOut io.ReadWriter) {

	if options.Help {
		options.flagSet.SetOutput(out)
		options.flagSet.PrintDefaults()
		return
	}

	output, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	if options.PrintToConsole {
		fmt.Fprintln(out, output)
	} else {
		ioutil.WriteFile(options.OutputFile, []byte(output), 0644)
	}

}

func processPackage(location string) (output string, err error) {
	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", errors.New("Couldn't parse sources: " + err.Error())
	}

	haveOutputHeader := false

	for _, theStruct := range sources.Structs {

		if !haveOutputHeader {
			output += headerForPackage(theStruct.PackageName)
			haveOutputHeader = true
		}

		outputReader, outputReadSetter := structConfig(theStruct.DocLines)

		if outputReader || outputReadSetter {
			output += headerForStruct(theStruct.Name)
		}

		if outputReader {
			output += readerForStruct(theStruct.Name)
		}
		if outputReadSetter {
			output += readSetterForStruct(theStruct.Name)
		}
	}

	formattedBytes, err := format.Source([]byte(output))

	if err != nil {
		return "", errors.New("Couldn't go fmt code: " + err.Error())
	}

	return string(formattedBytes), nil
}

func structConfig(docLines []string) (outputReader bool, outputReadSetter bool) {

	for _, docLine := range docLines {
		docLine = strings.ToLower(docLine)
		docLine = strings.TrimPrefix(docLine, "//")
		docLine = strings.TrimSpace(docLine)
		if !strings.HasPrefix(docLine, magicDocLinePrefix) {
			continue
		}
		docLine = strings.TrimPrefix(docLine, magicDocLinePrefix)
		docLine = strings.TrimSpace(docLine)

		switch docLine {
		case "":
			return true, true
		case "both":
			return true, true
		case "reader":
			return true, false
		case "readsetter":
			return false, true
		}

	}
	return false, false
}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}

func headerForPackage(packageName string) string {
	return templateOutput(headerTemplate, map[string]string{
		"packageName": packageName,
	}) + reflectImportText
}

func headerForStruct(structName string) string {
	return templateOutput(structHeaderTemplate, map[string]string{
		"structName": structName,
	})
}

func readerForStruct(structName string) string {

	return templateOutput(readerTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})

}

func readSetterForStruct(structName string) string {
	return templateOutput(readSetterTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})
}

const headerTemplateText = `/************************************
 *
 * This file contains auto-generated methods to help certain structs
 * implement boardgame.SubState and boardgame.MutableSubState. It was
 * generated by autoreader.
 *
 * DO NOT EDIT by hand.
 *
 ************************************/
package {{.packageName}}
`

const reflectImportText = `import (
	"github.com/jkomoros/boardgame"
)

`

const structHeaderTemplateText = `// Implementation for {{.structName}}

 `

const readerTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) Reader() boardgame.PropertyReader {
	return boardgame.DefaultReader({{.FirstLetter}})
}

`

const readSetterTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) ReadSetter() boardgame.PropertyReadSetter {
	return boardgame.DefaultReadSetter({{.FirstLetter}})
}

`
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

var ren = render.New()

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	build := schema.Build{
		ID:             uuid.NewUUID(),
		SourceRepo:     opts.SourceRepo,
		SourceRevision: opts.SourceRevision,
		Name:           opts.Name,
		Dockerfile:     opts.Dockerfile,
		Status:         "building",
		CreatedAt:      time.Now(),
		UpdatedAt:      time.Now(),
	}

	reg := registry.NewRegistry("localfs", "")
	reg.Set(build)

	// debug code
	builddata, err := reg.Get(build.ID)
	fmt.Fprintln(w, builddata)
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reg := registry.NewRegistry("localfs", "")
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
	}

	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	reg := registry.NewRegistry("localfs", "")
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
	}
	ren.JSON(w, http.StatusOK, build)
}

func main() {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	n.Run(":8080")
}
<commit_msg>Return a single JSON<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

var ren = render.New()

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	build := schema.Build{
		ID:             uuid.NewUUID(),
		SourceRepo:     opts.SourceRepo,
		SourceRevision: opts.SourceRevision,
		Name:           opts.Name,
		Dockerfile:     opts.Dockerfile,
		Status:         "building",
		CreatedAt:      time.Now(),
		UpdatedAt:      time.Now(),
	}

	reg := registry.NewRegistry("localfs", "")
	reg.Set(build)

	// debug code
	builddata, err := reg.Get(build.ID)
	fmt.Fprintln(w, builddata)
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reg := registry.NewRegistry("localfs", "")
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
	} else {
		ren.JSON(w, http.StatusOK, builds)
	}
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	reg := registry.NewRegistry("localfs", "")
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
	} else {
		ren.JSON(w, http.StatusOK, build)
	}
}

func main() {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	n.Run(":8080")
}
<|endoftext|>


<commit_before>// Package directEmail support direct send email from selected interface include SOCKS5 proxy server
package directEmail

import (
	"bytes"
	"crypto/tls"
	"errors"
	"fmt"
	"golang.org/x/net/proxy"
	"net"
	"net/smtp"
	"net/url"
	"strings"
	"time"
)

// A Email contains options for send email.
type Email struct {
	// Ip contains local IP address which use for send email
	// if blank use default interface
	// if use socks5 proxy "socks://123.124.125.126:8080"
	// and with auth "socks://user:password@123.124.125.126:8080"
	Ip string
	// Host is host name
	// if blank use DNS resolv for field fill
	Host string
	// Port SMTP server port
	Port uint16
	// MapIp use for translate local IP to global if NAT
	// if use Socks server translate IP SOCKS server to real IP
	MapIp map[string]string
	// FromEmail sender email (Required)
	FromEmail string
	// FromName sender name
	FromName string
	// ToEmail recipient email (Required)
	ToEmail string
	// ToName recipient name
	ToName string
	// Subject email subject
	Subject string

	headers     []string
	textPlain   []byte
	textHTML    []byte
	attachments [][]byte
	raw         bytes.Buffer
	bodyLenght  int
}

const (
	debugIs     = false
	connTimeout = 5 * time.Minute // SMTP RFC
)

// New returns a new Email instance for create and send email
func New() Email {
	return Email{Port: 25}
}

// SendThroughServer send email from SMTP server
func (slf *Email) SendThroughServer(host string, port uint16, username, password string) error {
	slf.Port = port

	dialFunc, err := slf.dialFunction()
	debug("Dialer selected, now dial to server\n")

	conn, err := dialFunc("tcp", net.JoinHostPort(host, fmt.Sprintf("%d", slf.Port)))
	if err != nil {
		debug("Not connected\n")
		return err
	}
	defer conn.Close()
	debug("Connected\n")

	err = conn.SetDeadline(time.Now().Add(connTimeout + time.Millisecond*10))
	if err != nil {
		return err
	}

	client, err := smtp.NewClient(conn, host)
	if err != nil {
		return err
	}

	auth := smtp.PlainAuth(
		"",
		username,
		password,
		host,
	)
	slf.Host = "localhost"

	debug("Connected, now send email\n")

	slf.cleanEmail()
	return slf.sendWithTimeout(auth, host, client)
}

// Send email directly
func (slf *Email) Send() error {
	var err error

	server, err := slf.domainFromEmail(slf.ToEmail)
	if err != nil {
		return errors.New("553 Bad ToEmail")
	}

	dialFunc, err := slf.dialFunction()
	if err != nil {
		return fmt.Errorf("421 %v", err)
	}

	conn, err := slf.newClient(server, dialFunc)
	if err != nil {
		debug("Not connected\n")
		return fmt.Errorf("421 %v", err)
	}
	defer conn.Close()
	debug("Connected\n")

	slf.cleanEmail()
	return slf.sendWithTimeout(nil, "", conn)
}

func (slf *Email) cleanEmail() {
	slf.ToEmail = strings.TrimSpace(slf.ToEmail)
	slf.FromEmail = strings.TrimSpace(slf.FromEmail)
}

var testHookStartTLS func(*tls.Config)

// sendWithTimeout hack for bad server
func (slf *Email) sendWithTimeout(auth smtp.Auth, host string, client *smtp.Client) error {
	var res error
	err := make(chan error, 1)
	go func() {
		err <- slf.send(auth, host, client)
	}()

	select {
	case res = <-err:

	case <-time.After(connTimeout):
		res = fmt.Errorf("421 send timeout after %s", connTimeout.String())
	}

	return res
}

// send sending email message
func (slf *Email) send(auth smtp.Auth, host string, client *smtp.Client) error {
	var err error

	if err := client.Hello(strings.TrimRight(slf.Host, ".")); err != nil {
		return err
	}

	if auth != nil {
		if ok, _ := client.Extension("STARTTLS"); ok {
			config := &tls.Config{ServerName: host}
			if testHookStartTLS != nil {
				testHookStartTLS(config)
			}
			if err = client.StartTLS(config); err != nil {
				return err
			}
		}
		if auth != nil {
			if err = client.Auth(auth); err != nil {
				return err
			}
		}
	}

	if err := client.Mail(slf.FromEmail); err != nil {
		return err
	}

	if err := client.Rcpt(slf.ToEmail); err != nil {
		return err
	}

	w, err := client.Data()
	if err != nil {
		return err
	}

	_, err = fmt.Fprint(w, slf.raw.String())
	if err != nil {
		return err
	}

	err = w.Close()
	if err != nil {
		return err
	}

	return client.Quit()

}

type conn func(network, address string) (net.Conn, error)

func (slf *Email) dialFunction() (conn, error) {
	var dialFunc conn

	if slf.Ip == "" {
		iface := net.Dialer{}
		dialFunc = iface.Dial
		debug("Dial function is default network interface\n")
	} else {
		if strings.ToLower(slf.Ip[0:8]) == "socks://" {
			u, err := url.Parse(slf.Ip)
			if err != nil {
				return nil, fmt.Errorf("Error parse socks: %s", err.Error())
			}
			var iface proxy.Dialer
			if u.User != nil {
				auth := proxy.Auth{}
				auth.User = u.User.Username()
				auth.Password, _ = u.User.Password()
				iface, err = proxy.SOCKS5("tcp", u.Host, &auth, proxy.FromEnvironment())
				if err != nil {
					return dialFunc, err
				}
			} else {
				iface, err = proxy.SOCKS5("tcp", u.Host, nil, proxy.FromEnvironment())
				if err != nil {
					return dialFunc, err
				}
			}
			slf.Ip = u.Host
			dialFunc = iface.Dial
			debug("Dial function is socks proxy from ", slf.Ip[8:], "\n")
		} else {
			addr := &net.TCPAddr{
				IP: net.ParseIP(slf.Ip),
			}
			iface := net.Dialer{
				LocalAddr: addr,
			}
			dialFunc = iface.Dial
			debug("Dial function is ", addr.String(), " network interface\n")
		}
	}

	return dialFunc, nil
}

func (slf *Email) newClient(server string, dialFunc conn) (client *smtp.Client, err error) {
	var conn net.Conn

	records, err := net.LookupMX(server)
	if err != nil {
		return
	}
	debug("MX for domain:\n")
	for i := range records {
		debug(" - ", records[i].Pref, " ", records[i].Host, "\n")
	}

	for i := range records {
		server := strings.TrimRight(strings.TrimSpace(records[i].Host), ".")
		debug("Connect to server ", server, "\n")
		conn, err = dialFunc("tcp", net.JoinHostPort(server, fmt.Sprintf("%d", slf.Port)))
		if err != nil {
			debug("Not connected\n")
			continue
		}
		debug("Connected\n")
		client, err = smtp.NewClient(conn, server)
		if err == nil {
			break
		}
	}
	if err != nil {
		return
	}

	err = conn.SetDeadline(time.Now().Add(connTimeout + time.Millisecond*10))
	if err != nil {
		return
	}

	if slf.Ip == "" {
		slf.Ip = conn.LocalAddr().String()
	}

	if slf.Host == "" {
		var myGlobalIP string
		myIP, _, err := net.SplitHostPort(strings.TrimLeft(slf.Ip, "socks://"))
		myGlobalIP, ok := slf.MapIp[myIP]
		if !ok {
			myGlobalIP = myIP
		}
		names, err := net.LookupAddr(myGlobalIP)
		if err != nil && len(names) < 1 {
			return nil, err
		}
		debug("LookUp ", myGlobalIP, " this result ", names[0], "\n")
		slf.Host = names[0]
	}

	return
}
<commit_msg>change timeout order<commit_after>// Package directEmail support direct send email from selected interface include SOCKS5 proxy server
package directEmail

import (
	"bytes"
	"crypto/tls"
	"errors"
	"fmt"
	"golang.org/x/net/proxy"
	"net"
	"net/smtp"
	"net/url"
	"strings"
	"time"
)

// A Email contains options for send email.
type Email struct {
	// Ip contains local IP address which use for send email
	// if blank use default interface
	// if use socks5 proxy "socks://123.124.125.126:8080"
	// and with auth "socks://user:password@123.124.125.126:8080"
	Ip string
	// Host is host name
	// if blank use DNS resolv for field fill
	Host string
	// Port SMTP server port
	Port uint16
	// MapIp use for translate local IP to global if NAT
	// if use Socks server translate IP SOCKS server to real IP
	MapIp map[string]string
	// FromEmail sender email (Required)
	FromEmail string
	// FromName sender name
	FromName string
	// ToEmail recipient email (Required)
	ToEmail string
	// ToName recipient name
	ToName string
	// Subject email subject
	Subject string

	headers     []string
	textPlain   []byte
	textHTML    []byte
	attachments [][]byte
	raw         bytes.Buffer
	bodyLenght  int
}

const (
	debugIs     = false
	connTimeout = 5 * time.Minute // SMTP RFC
)

// New returns a new Email instance for create and send email
func New() Email {
	return Email{Port: 25}
}

// SendThroughServer send email from SMTP server
func (slf *Email) SendThroughServer(host string, port uint16, username, password string) error {
	slf.Port = port

	dialFunc, err := slf.dialFunction()
	debug("Dialer selected, now dial to server\n")

	conn, err := dialFunc("tcp", net.JoinHostPort(host, fmt.Sprintf("%d", slf.Port)))
	if err != nil {
		debug("Not connected\n")
		return err
	}
	defer conn.Close()
	debug("Connected\n")

	err = conn.SetDeadline(time.Now().Add(connTimeout))
	if err != nil {
		return err
	}

	client, err := smtp.NewClient(conn, host)
	if err != nil {
		return err
	}

	auth := smtp.PlainAuth(
		"",
		username,
		password,
		host,
	)
	slf.Host = "localhost"

	debug("Connected, now send email\n")

	slf.cleanEmail()
	return slf.sendWithTimeout(auth, host, client)
}

// Send email directly
func (slf *Email) Send() error {
	var err error

	server, err := slf.domainFromEmail(slf.ToEmail)
	if err != nil {
		return errors.New("553 Bad ToEmail")
	}

	dialFunc, err := slf.dialFunction()
	if err != nil {
		return fmt.Errorf("421 %v", err)
	}

	conn, err := slf.newClient(server, dialFunc)
	if err != nil {
		debug("Not connected\n")
		return fmt.Errorf("421 %v", err)
	}
	defer conn.Close()
	debug("Connected\n")

	slf.cleanEmail()
	return slf.sendWithTimeout(nil, "", conn)
}

func (slf *Email) cleanEmail() {
	slf.ToEmail = strings.TrimSpace(slf.ToEmail)
	slf.FromEmail = strings.TrimSpace(slf.FromEmail)
}

var testHookStartTLS func(*tls.Config)

// sendWithTimeout hack for bad server
func (slf *Email) sendWithTimeout(auth smtp.Auth, host string, client *smtp.Client) error {
	var res error
	err := make(chan error, 1)
	go func() {
		err <- slf.send(auth, host, client)
	}()

	select {
	case res = <-err:

	case <-time.After(connTimeout + time.Millisecond * 500):
		res = fmt.Errorf("421 send timeout after %s", connTimeout.String())
	}

	return res
}

// send sending email message
func (slf *Email) send(auth smtp.Auth, host string, client *smtp.Client) error {
	var err error

	if err := client.Hello(strings.TrimRight(slf.Host, ".")); err != nil {
		return err
	}

	if auth != nil {
		if ok, _ := client.Extension("STARTTLS"); ok {
			config := &tls.Config{ServerName: host}
			if testHookStartTLS != nil {
				testHookStartTLS(config)
			}
			if err = client.StartTLS(config); err != nil {
				return err
			}
		}
		if auth != nil {
			if err = client.Auth(auth); err != nil {
				return err
			}
		}
	}

	if err := client.Mail(slf.FromEmail); err != nil {
		return err
	}

	if err := client.Rcpt(slf.ToEmail); err != nil {
		return err
	}

	w, err := client.Data()
	if err != nil {
		return err
	}

	_, err = fmt.Fprint(w, slf.raw.String())
	if err != nil {
		return err
	}

	err = w.Close()
	if err != nil {
		return err
	}

	return client.Quit()

}

type conn func(network, address string) (net.Conn, error)

func (slf *Email) dialFunction() (conn, error) {
	var dialFunc conn

	if slf.Ip == "" {
		iface := net.Dialer{}
		dialFunc = iface.Dial
		debug("Dial function is default network interface\n")
	} else {
		if strings.ToLower(slf.Ip[0:8]) == "socks://" {
			u, err := url.Parse(slf.Ip)
			if err != nil {
				return nil, fmt.Errorf("Error parse socks: %s", err.Error())
			}
			var iface proxy.Dialer
			if u.User != nil {
				auth := proxy.Auth{}
				auth.User = u.User.Username()
				auth.Password, _ = u.User.Password()
				iface, err = proxy.SOCKS5("tcp", u.Host, &auth, proxy.FromEnvironment())
				if err != nil {
					return dialFunc, err
				}
			} else {
				iface, err = proxy.SOCKS5("tcp", u.Host, nil, proxy.FromEnvironment())
				if err != nil {
					return dialFunc, err
				}
			}
			slf.Ip = u.Host
			dialFunc = iface.Dial
			debug("Dial function is socks proxy from ", slf.Ip[8:], "\n")
		} else {
			addr := &net.TCPAddr{
				IP: net.ParseIP(slf.Ip),
			}
			iface := net.Dialer{
				LocalAddr: addr,
			}
			dialFunc = iface.Dial
			debug("Dial function is ", addr.String(), " network interface\n")
		}
	}

	return dialFunc, nil
}

func (slf *Email) newClient(server string, dialFunc conn) (client *smtp.Client, err error) {
	var conn net.Conn

	records, err := net.LookupMX(server)
	if err != nil {
		return
	}
	debug("MX for domain:\n")
	for i := range records {
		debug(" - ", records[i].Pref, " ", records[i].Host, "\n")
	}

	for i := range records {
		server := strings.TrimRight(strings.TrimSpace(records[i].Host), ".")
		debug("Connect to server ", server, "\n")
		conn, err = dialFunc("tcp", net.JoinHostPort(server, fmt.Sprintf("%d", slf.Port)))
		if err != nil {
			debug("Not connected\n")
			continue
		}
		debug("Connected\n")
		client, err = smtp.NewClient(conn, server)
		if err == nil {
			break
		}
	}
	if err != nil {
		return
	}

	err = conn.SetDeadline(time.Now().Add(connTimeout))
	if err != nil {
		return
	}

	if slf.Ip == "" {
		slf.Ip = conn.LocalAddr().String()
	}

	if slf.Host == "" {
		var myGlobalIP string
		myIP, _, err := net.SplitHostPort(strings.TrimLeft(slf.Ip, "socks://"))
		myGlobalIP, ok := slf.MapIp[myIP]
		if !ok {
			myGlobalIP = myIP
		}
		names, err := net.LookupAddr(myGlobalIP)
		if err != nil && len(names) < 1 {
			return nil, err
		}
		debug("LookUp ", myGlobalIP, " this result ", names[0], "\n")
		slf.Host = names[0]
	}

	return
}
<|endoftext|>


<commit_before>package cmd

import (
	"os"
	"os/signal"
	"syscall"
	"time"

	"context"

	"github.com/mitchellh/mapstructure"
	"github.com/pkg/errors"
	"github.com/problame/go-streamrpc"
	"github.com/zrepl/zrepl/cmd/endpoint"
	"github.com/zrepl/zrepl/replication"
)

type PullJob struct {
	Name     string
	Connect  streamrpc.Connecter
	Interval time.Duration
	Mapping  *DatasetMapFilter
	// constructed from mapping during parsing
	pruneFilter    *DatasetMapFilter
	SnapshotPrefix string
	Prune          PrunePolicy
	Debug          JobDebugSettings

	rep *replication.Replication
}

func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {

	var asMap struct {
		Connect           map[string]interface{}
		Interval          string
		Mapping           map[string]string
		InitialReplPolicy string `mapstructure:"initial_repl_policy"`
		Prune             map[string]interface{}
		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
		Debug             map[string]interface{}
	}

	if err = mapstructure.Decode(i, &asMap); err != nil {
		err = errors.Wrap(err, "mapstructure error")
		return nil, err
	}

	j = &PullJob{Name: name}

	j.Connect, err = parseConnect(asMap.Connect)
	if err != nil {
		err = errors.Wrap(err, "cannot parse 'connect'")
		return nil, err
	}

	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
		err = errors.Wrap(err, "cannot parse 'interval'")
		return nil, err
	}

	j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false)
	if err != nil {
		err = errors.Wrap(err, "cannot parse 'mapping'")
		return nil, err
	}

	if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil {
		err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job")
		return nil, err
	}

	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
		return
	}

	if j.Prune, err = parsePrunePolicy(asMap.Prune, false); err != nil {
		err = errors.Wrap(err, "cannot parse prune policy")
		return
	}

	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
		err = errors.Wrap(err, "cannot parse 'debug'")
		return
	}

	if j.Debug.Conn.ReadDump != "" || j.Debug.Conn.WriteDump != "" {
		logConnecter := logNetConnConnecter{
			Connecter: j.Connect,
			ReadDump:  j.Debug.Conn.ReadDump,
			WriteDump: j.Debug.Conn.WriteDump,
		}
		j.Connect = logConnecter
	}

	return
}

func (j *PullJob) JobName() string {
	return j.Name
}

func (j *PullJob) JobType() JobType { return JobTypePull }

func (j *PullJob) JobStart(ctx context.Context) {

	log := getLogger(ctx)
	defer log.Info("exiting")

	// j.task is idle here idle here
	usr1 := make(chan os.Signal)
	signal.Notify(usr1, syscall.SIGUSR1)
	defer signal.Stop(usr1)

	ticker := time.NewTicker(j.Interval)
	for {
		begin := time.Now()
		j.doRun(ctx)
		duration := time.Now().Sub(begin)
		if duration > j.Interval {
			log.
				WithField("actual_duration", duration).
				WithField("configured_interval", j.Interval).
				Warn("pull run took longer than configured interval")
		}
		select {
		case <-ctx.Done():
			log.WithError(ctx.Err()).Info("context")
			return
		case <-ticker.C:
		case <-usr1:
		}
	}
}

var STREAMRPC_CONFIG = &streamrpc.ConnConfig{ // FIXME oversight and configurability
	RxHeaderMaxLen:       4096,
	RxStructuredMaxLen:   4096 * 4096,
	RxStreamMaxChunkSize: 4096 * 4096,
	TxChunkSize:          4096 * 4096,
	RxTimeout: streamrpc.Timeout{
		Progress: 10 * time.Second,
	},
	TxTimeout: streamrpc.Timeout{
		Progress: 10 * time.Second,
	},
}

func (j *PullJob) doRun(ctx context.Context) {

	log := getLogger(ctx)
	// FIXME
	clientConf := &streamrpc.ClientConfig{
		ConnConfig: STREAMRPC_CONFIG,
	}

	client, err := streamrpc.NewClient(j.Connect, clientConf)
	defer client.Close()

	sender := endpoint.NewRemote(client)

	puller, err := endpoint.NewReceiver(
		j.Mapping,
		NewPrefixFilter(j.SnapshotPrefix),
	)
	if err != nil {
		log.WithError(err).Error("error creating receiver endpoint")
		return
	}

	{
		ctx := replication.WithLogger(ctx, replicationLogAdaptor{log.WithField(logSubsysField, "replication")})
		ctx = streamrpc.ContextWithLogger(ctx, streamrpcLogAdaptor{log.WithField(logSubsysField, "rpc")})
		ctx = endpoint.WithLogger(ctx, log.WithField(logSubsysField, "endpoint"))
		j.rep = replication.NewReplication()
		j.rep.Drive(ctx, sender, puller)
	}

	client.Close()

	{
		ctx := WithLogger(ctx, log.WithField(logSubsysField, "prune"))
		pruner, err := j.Pruner(PrunePolicySideDefault, false)
		if err != nil {
			log.WithError(err).Error("error creating pruner")
		} else {
			pruner.Run(ctx)
		}
	}
}

func (j *PullJob) Report() *replication.Report {
	return j.rep.Report()
}

func (j *PullJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
	p = Pruner{
		time.Now(),
		dryRun,
		j.pruneFilter,
		j.SnapshotPrefix,
		j.Prune,
	}
	return
}
<commit_msg>rename variable<commit_after>package cmd

import (
	"os"
	"os/signal"
	"syscall"
	"time"

	"context"

	"github.com/mitchellh/mapstructure"
	"github.com/pkg/errors"
	"github.com/problame/go-streamrpc"
	"github.com/zrepl/zrepl/cmd/endpoint"
	"github.com/zrepl/zrepl/replication"
)

type PullJob struct {
	Name     string
	Connect  streamrpc.Connecter
	Interval time.Duration
	Mapping  *DatasetMapFilter
	// constructed from mapping during parsing
	pruneFilter    *DatasetMapFilter
	SnapshotPrefix string
	Prune          PrunePolicy
	Debug          JobDebugSettings

	rep *replication.Replication
}

func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {

	var asMap struct {
		Connect           map[string]interface{}
		Interval          string
		Mapping           map[string]string
		InitialReplPolicy string `mapstructure:"initial_repl_policy"`
		Prune             map[string]interface{}
		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
		Debug             map[string]interface{}
	}

	if err = mapstructure.Decode(i, &asMap); err != nil {
		err = errors.Wrap(err, "mapstructure error")
		return nil, err
	}

	j = &PullJob{Name: name}

	j.Connect, err = parseConnect(asMap.Connect)
	if err != nil {
		err = errors.Wrap(err, "cannot parse 'connect'")
		return nil, err
	}

	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
		err = errors.Wrap(err, "cannot parse 'interval'")
		return nil, err
	}

	j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false)
	if err != nil {
		err = errors.Wrap(err, "cannot parse 'mapping'")
		return nil, err
	}

	if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil {
		err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job")
		return nil, err
	}

	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
		return
	}

	if j.Prune, err = parsePrunePolicy(asMap.Prune, false); err != nil {
		err = errors.Wrap(err, "cannot parse prune policy")
		return
	}

	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
		err = errors.Wrap(err, "cannot parse 'debug'")
		return
	}

	if j.Debug.Conn.ReadDump != "" || j.Debug.Conn.WriteDump != "" {
		logConnecter := logNetConnConnecter{
			Connecter: j.Connect,
			ReadDump:  j.Debug.Conn.ReadDump,
			WriteDump: j.Debug.Conn.WriteDump,
		}
		j.Connect = logConnecter
	}

	return
}

func (j *PullJob) JobName() string {
	return j.Name
}

func (j *PullJob) JobType() JobType { return JobTypePull }

func (j *PullJob) JobStart(ctx context.Context) {

	log := getLogger(ctx)
	defer log.Info("exiting")

	// j.task is idle here idle here
	usr1 := make(chan os.Signal)
	signal.Notify(usr1, syscall.SIGUSR1)
	defer signal.Stop(usr1)

	ticker := time.NewTicker(j.Interval)
	for {
		begin := time.Now()
		j.doRun(ctx)
		duration := time.Now().Sub(begin)
		if duration > j.Interval {
			log.
				WithField("actual_duration", duration).
				WithField("configured_interval", j.Interval).
				Warn("pull run took longer than configured interval")
		}
		select {
		case <-ctx.Done():
			log.WithError(ctx.Err()).Info("context")
			return
		case <-ticker.C:
		case <-usr1:
		}
	}
}

var STREAMRPC_CONFIG = &streamrpc.ConnConfig{ // FIXME oversight and configurability
	RxHeaderMaxLen:       4096,
	RxStructuredMaxLen:   4096 * 4096,
	RxStreamMaxChunkSize: 4096 * 4096,
	TxChunkSize:          4096 * 4096,
	RxTimeout: streamrpc.Timeout{
		Progress: 10 * time.Second,
	},
	TxTimeout: streamrpc.Timeout{
		Progress: 10 * time.Second,
	},
}

func (j *PullJob) doRun(ctx context.Context) {

	log := getLogger(ctx)
	// FIXME
	clientConf := &streamrpc.ClientConfig{
		ConnConfig: STREAMRPC_CONFIG,
	}

	client, err := streamrpc.NewClient(j.Connect, clientConf)
	defer client.Close()

	sender := endpoint.NewRemote(client)

	receiver, err := endpoint.NewReceiver(
		j.Mapping,
		NewPrefixFilter(j.SnapshotPrefix),
	)
	if err != nil {
		log.WithError(err).Error("error creating receiver endpoint")
		return
	}

	{
		ctx := replication.WithLogger(ctx, replicationLogAdaptor{log.WithField(logSubsysField, "replication")})
		ctx = streamrpc.ContextWithLogger(ctx, streamrpcLogAdaptor{log.WithField(logSubsysField, "rpc")})
		ctx = endpoint.WithLogger(ctx, log.WithField(logSubsysField, "endpoint"))
		j.rep = replication.NewReplication()
		j.rep.Drive(ctx, sender, receiver)
	}

	client.Close()

	{
		ctx := WithLogger(ctx, log.WithField(logSubsysField, "prune"))
		pruner, err := j.Pruner(PrunePolicySideDefault, false)
		if err != nil {
			log.WithError(err).Error("error creating pruner")
		} else {
			pruner.Run(ctx)
		}
	}
}

func (j *PullJob) Report() *replication.Report {
	return j.rep.Report()
}

func (j *PullJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
	p = Pruner{
		time.Now(),
		dryRun,
		j.pruneFilter,
		j.SnapshotPrefix,
		j.Prune,
	}
	return
}
<|endoftext|>