<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	gz "github.com/NYTimes/gziphandler"
	sqlite "github.com/gwenn/gosqlite"
	com "github.com/sqlitebrowser/dbhub.io/common"
)

var (
	// Our self signed Certificate Authority chain
	ourCAPool *x509.CertPool

	// Log file for incoming HTTPS requests
	reqLog *os.File

	// Address of our server, formatted for display
	server string

	// Our parsed HTML templates
	tmpl *template.Template
)

func main() {
	// Read server configuration
	var err error
	if err = com.ReadConfig(); err != nil {
		log.Fatalf("Configuration file problem\n\n%v", err)
	}

	// Open the request log for writing
	reqLog, err = os.OpenFile(com.Conf.Api.RequestLog, os.O_CREATE|os.O_APPEND|os.O_WRONLY|os.O_SYNC, 0750)
	if err != nil {
		log.Fatalf("Error when opening request log: %s\n", err)
	}
	defer reqLog.Close()
	log.Printf("Request log opened: %s\n", com.Conf.Api.RequestLog)

	// Parse our template files
	tmpl = template.Must(template.New("templates").Delims("[[", "]]").ParseGlob(
		filepath.Join(com.Conf.Web.BaseDir, "api", "templates", "*.html")))

	// Connect to Minio server
	err = com.ConnectMinio()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Connect to PostgreSQL server
	err = com.ConnectPostgreSQL()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Connect to the Memcached server
	err = com.ConnectCache()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Add the default user to the system
	err = com.AddDefaultUser()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Add the default licences to the system
	err = com.AddDefaultLicences()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Load our self signed CA chain
	ourCAPool = x509.NewCertPool()
	certFile, err := ioutil.ReadFile(com.Conf.DB4S.CAChain)
	if err != nil {
		fmt.Printf("Error opening Certificate Authority chain file: %v\n", err)
		return
	}
	ok := ourCAPool.AppendCertsFromPEM(certFile)
	if !ok {
		fmt.Println("Error appending certificate file")
		return
	}

	// Our pages
	http.Handle("/", gz.GzipHandler(handleWrapper(rootHandler)))
	http.Handle("/v1/branches", gz.GzipHandler(handleWrapper(branchesHandler)))
	http.Handle("/v1/columns", gz.GzipHandler(handleWrapper(columnsHandler)))
	http.Handle("/v1/commits", gz.GzipHandler(handleWrapper(commitsHandler)))
	http.Handle("/v1/databases", gz.GzipHandler(handleWrapper(databasesHandler)))
	http.Handle("/v1/delete", gz.GzipHandler(handleWrapper(deleteHandler)))
	http.Handle("/v1/diff", gz.GzipHandler(handleWrapper(diffHandler)))
	http.Handle("/v1/download", gz.GzipHandler(handleWrapper(downloadHandler)))
	http.Handle("/v1/indexes", gz.GzipHandler(handleWrapper(indexesHandler)))
	http.Handle("/v1/metadata", gz.GzipHandler(handleWrapper(metadataHandler)))
	http.Handle("/v1/query", gz.GzipHandler(handleWrapper(queryHandler)))
	http.Handle("/v1/releases", gz.GzipHandler(handleWrapper(releasesHandler)))
	http.Handle("/v1/tables", gz.GzipHandler(handleWrapper(tablesHandler)))
	http.Handle("/v1/tags", gz.GzipHandler(handleWrapper(tagsHandler)))
	http.Handle("/v1/upload", gz.GzipHandler(handleWrapper(uploadHandler)))
	http.Handle("/v1/views", gz.GzipHandler(handleWrapper(viewsHandler)))
	http.Handle("/v1/webpage", gz.GzipHandler(handleWrapper(webpageHandler)))

	// favicon.ico
	http.Handle("/favicon.ico", gz.GzipHandler(handleWrapper(func(w http.ResponseWriter, r *http.Request) {
		logReq(r, "-")
		http.ServeFile(w, r, filepath.Join(com.Conf.Web.BaseDir, "webui", "favicon.ico"))
	})))

	// Load our self signed CA Cert chain, check client certificates if given, and set TLS1.2 as minimum
	newTLSConfig := &tls.Config{
		ClientAuth:               tls.VerifyClientCertIfGiven,
		ClientCAs:                ourCAPool,
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true,
		RootCAs:                  ourCAPool,
	}
	srv := &http.Server{
		Addr:         com.Conf.Api.BindAddress,
		TLSConfig:    newTLSConfig,
		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0),
	}

	// Generate the formatted server string
	server = fmt.Sprintf("https://%s", com.Conf.Api.ServerName)

	// Start API server
	log.Printf("API server starting on %s\n", server)
	err = srv.ListenAndServeTLS(com.Conf.Api.Certificate, com.Conf.Api.CertificateKey)

	// Shut down nicely
	com.DisconnectPostgreSQL()

	if err != nil {
		log.Fatal(err)
	}
}

// checkAuth authenticates and logs the incoming request
func checkAuth(w http.ResponseWriter, r *http.Request) (loggedInUser string, err error) {
	// Extract the API key from the request
	apiKey := r.FormValue("apikey")

	// Check if API key was provided
	if apiKey != "" {
		// Look up the owner of the API key
		loggedInUser, err = com.GetAPIKeyUser(apiKey)
	} else {
		// No API key was provided. Check for a client certificate instead
		loggedInUser, err = extractUserFromClientCert(w, r)
	}

	// Check for any errors
	if err != nil || loggedInUser == "" {
		err = fmt.Errorf("Incorrect or unknown API key and certificate")
		return
	}

	// Log the incoming request
	logReq(r, loggedInUser)
	return
}

// collectInfo is an internal function which:
//   1. Authenticates incoming requests
//   2. Extracts the database owner, name, and commit ID from the request
//   3. Checks permissions
func collectInfo(w http.ResponseWriter, r *http.Request) (loggedInUser, dbOwner, dbName, commitID string, httpStatus int, err error) {
	// Authenticate the request
	loggedInUser, err = checkAuth(w, r)
	if err != nil {
		httpStatus = http.StatusUnauthorized
		return
	}

	// Extract the database owner name, database name, and (optional) commit ID for the database from the request
	dbOwner, dbName, commitID, err = com.GetFormODC(r)
	if err != nil {
		httpStatus = http.StatusInternalServerError
		return
	}
	dbFolder := "/"

	// Check if the user has access to the requested database
	// Check if the requested database exists
	exists, err := com.CheckDBExists(loggedInUser, dbOwner, dbFolder, dbName)
	if err != nil {
		httpStatus = http.StatusInternalServerError
		return
	}
	if !exists {
		httpStatus = http.StatusNotFound
		return
	}
	return
}

// collectInfoAndOpen is an internal function which:
//   1. Calls collectInfo() to authenticate the request + collect the user/database/commit/etc details
//   2. Fetches the database from Minio
//   3. Opens the database, returning the connection handle
// This function exists purely because this code is common to most of the handlers
func collectInfoAndOpen(w http.ResponseWriter, r *http.Request) (sdb *sqlite.Conn, httpStatus int, err error) {
	// Authenticate the request and collect details for the requested database
	loggedInUser, dbOwner, dbName, commitID, httpStatus, err := collectInfo(w, r)
	if err != nil {
		httpStatus = http.StatusInternalServerError
		return
	}
	dbFolder := "/"

	// Get Minio bucket
	bucket, id, _, err := com.MinioLocation(dbOwner, dbFolder, dbName, commitID, loggedInUser)
	if err != nil {
		httpStatus = http.StatusInternalServerError
		return
	}

	// Sanity check
	if id == "" {
		// The requested database wasn't found, or the user doesn't have permission to access it
		err = fmt.Errorf("Requested database not found")
		log.Printf("Requested database not found. Owner: '%s%s%s'", dbOwner, dbFolder, dbName)
		httpStatus = http.StatusNotFound
		return
	}

	// Retrieve database file from Minio, using locally cached version if it's already there
	var newDB string
	newDB, err = com.RetrieveDatabaseFile(bucket, id)
	if err != nil {
		httpStatus = http.StatusNotFound
		return
	}

	// Open the SQLite database in read only mode
	sdb, err = sqlite.Open(newDB, sqlite.OpenReadOnly)
	if err != nil {
		log.Printf("Couldn't open database in collectInfoAndOpen(): %s", err)
		httpStatus = http.StatusInternalServerError
		return
	}
	if err = sdb.EnableExtendedResultCodes(true); err != nil {
		log.Printf("Couldn't enable extended result codes in collectInfoAndOpen(): %v\n", err.Error())
		httpStatus = http.StatusInternalServerError
		return
	}
	return
}

func extractUserFromClientCert(w http.ResponseWriter, r *http.Request) (userAcc string, err error) {
	// Check if a client certificate was provided
	if len(r.TLS.PeerCertificates) == 0 {
		err = errors.New("No client certificate provided")
		return
	}

	// Extract the account name and associated server from the validated client certificate
	cn := r.TLS.PeerCertificates[0].Subject.CommonName
	if cn == "" {
		// Common name is empty
		err = errors.New("Common name is blank in client certificate")
		return
	}
	s := strings.Split(cn, "@")
	if len(s) < 2 {
		err = errors.New("Missing information in client certificate")
		return
	}
	userAcc = s[0]
	certServer := s[1]
	if userAcc == "" || certServer == "" {
		// Missing details in common name field
		err = errors.New("Missing information in client certificate")
		return
	}

	// Verify the running server matches the one in the certificate
	db4sServer := com.Conf.DB4S.Server
	if certServer != db4sServer {
		err = fmt.Errorf("Server name in certificate '%s' doesn't match DB4S server '%s'\n", certServer,
			db4sServer)
		return
	}

	// Everything is ok, so return
	return
}

// handleWrapper does nothing useful except interface between types
// TODO: Get rid of this, as it shouldn't be needed
func handleWrapper(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		// Call the original function
		fn(w, r)
	}
}

// jsonErr returns an error message wrapped in JSON, for (potentially) easier processing by an API caller
func jsonErr(w http.ResponseWriter, msg string, statusCode int) {
	je := com.JsonError{
		Error: msg,
	}
	jsonData, err := json.MarshalIndent(je, "", "  ")
	if err != nil {
		errMsg := fmt.Sprintf("A 2nd error occurred when JSON marshalling an error structure: %v\n", err)
		log.Print(errMsg)
		http.Error(w, errMsg, http.StatusInternalServerError)
		fmt.Fprintf(w, `{"error":"An error occurred when marshalling JSON inside jsonErr()"}`)
		return
	}
	w.WriteHeader(statusCode)
	fmt.Fprintf(w, string(jsonData))
}

// logReq writes an entry for the incoming request to the request log
func logReq(r *http.Request, loggedInUser string) {
	fmt.Fprintf(reqLog, "%v - %s [%s] \"%s %s %s\" \"-\" \"-\" \"%s\" \"%s\"\n", r.RemoteAddr,
		loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto,
		r.Referer(), r.Header.Get("User-Agent"))
}
<commit_msg>api: catch potential fall through from public cert users<commit_after>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	gz "github.com/NYTimes/gziphandler"
	sqlite "github.com/gwenn/gosqlite"
	com "github.com/sqlitebrowser/dbhub.io/common"
)

var (
	// Our self signed Certificate Authority chain
	ourCAPool *x509.CertPool

	// Log file for incoming HTTPS requests
	reqLog *os.File

	// Address of our server, formatted for display
	server string

	// Our parsed HTML templates
	tmpl *template.Template
)

func main() {
	// Read server configuration
	var err error
	if err = com.ReadConfig(); err != nil {
		log.Fatalf("Configuration file problem\n\n%v", err)
	}

	// Open the request log for writing
	reqLog, err = os.OpenFile(com.Conf.Api.RequestLog, os.O_CREATE|os.O_APPEND|os.O_WRONLY|os.O_SYNC, 0750)
	if err != nil {
		log.Fatalf("Error when opening request log: %s\n", err)
	}
	defer reqLog.Close()
	log.Printf("Request log opened: %s\n", com.Conf.Api.RequestLog)

	// Parse our template files
	tmpl = template.Must(template.New("templates").Delims("[[", "]]").ParseGlob(
		filepath.Join(com.Conf.Web.BaseDir, "api", "templates", "*.html")))

	// Connect to Minio server
	err = com.ConnectMinio()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Connect to PostgreSQL server
	err = com.ConnectPostgreSQL()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Connect to the Memcached server
	err = com.ConnectCache()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Add the default user to the system
	err = com.AddDefaultUser()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Add the default licences to the system
	err = com.AddDefaultLicences()
	if err != nil {
		log.Fatalf(err.Error())
	}

	// Load our self signed CA chain
	ourCAPool = x509.NewCertPool()
	certFile, err := ioutil.ReadFile(com.Conf.DB4S.CAChain)
	if err != nil {
		fmt.Printf("Error opening Certificate Authority chain file: %v\n", err)
		return
	}
	ok := ourCAPool.AppendCertsFromPEM(certFile)
	if !ok {
		fmt.Println("Error appending certificate file")
		return
	}

	// Our pages
	http.Handle("/", gz.GzipHandler(handleWrapper(rootHandler)))
	http.Handle("/v1/branches", gz.GzipHandler(handleWrapper(branchesHandler)))
	http.Handle("/v1/columns", gz.GzipHandler(handleWrapper(columnsHandler)))
	http.Handle("/v1/commits", gz.GzipHandler(handleWrapper(commitsHandler)))
	http.Handle("/v1/databases", gz.GzipHandler(handleWrapper(databasesHandler)))
	http.Handle("/v1/delete", gz.GzipHandler(handleWrapper(deleteHandler)))
	http.Handle("/v1/diff", gz.GzipHandler(handleWrapper(diffHandler)))
	http.Handle("/v1/download", gz.GzipHandler(handleWrapper(downloadHandler)))
	http.Handle("/v1/indexes", gz.GzipHandler(handleWrapper(indexesHandler)))
	http.Handle("/v1/metadata", gz.GzipHandler(handleWrapper(metadataHandler)))
	http.Handle("/v1/query", gz.GzipHandler(handleWrapper(queryHandler)))
	http.Handle("/v1/releases", gz.GzipHandler(handleWrapper(releasesHandler)))
	http.Handle("/v1/tables", gz.GzipHandler(handleWrapper(tablesHandler)))
	http.Handle("/v1/tags", gz.GzipHandler(handleWrapper(tagsHandler)))
	http.Handle("/v1/upload", gz.GzipHandler(handleWrapper(uploadHandler)))
	http.Handle("/v1/views", gz.GzipHandler(handleWrapper(viewsHandler)))
	http.Handle("/v1/webpage", gz.GzipHandler(handleWrapper(webpageHandler)))

	// favicon.ico
	http.Handle("/favicon.ico", gz.GzipHandler(handleWrapper(func(w http.ResponseWriter, r *http.Request) {
		logReq(r, "-")
		http.ServeFile(w, r, filepath.Join(com.Conf.Web.BaseDir, "webui", "favicon.ico"))
	})))

	// Load our self signed CA Cert chain, check client certificates if given, and set TLS1.2 as minimum
	newTLSConfig := &tls.Config{
		ClientAuth:               tls.VerifyClientCertIfGiven,
		ClientCAs:                ourCAPool,
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true,
		RootCAs:                  ourCAPool,
	}
	srv := &http.Server{
		Addr:         com.Conf.Api.BindAddress,
		TLSConfig:    newTLSConfig,
		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0),
	}

	// Generate the formatted server string
	server = fmt.Sprintf("https://%s", com.Conf.Api.ServerName)

	// Start API server
	log.Printf("API server starting on %s\n", server)
	err = srv.ListenAndServeTLS(com.Conf.Api.Certificate, com.Conf.Api.CertificateKey)

	// Shut down nicely
	com.DisconnectPostgreSQL()

	if err != nil {
		log.Fatal(err)
	}
}

// checkAuth authenticates and logs the incoming request
func checkAuth(w http.ResponseWriter, r *http.Request) (loggedInUser string, err error) {
	// Extract the API key from the request
	apiKey := r.FormValue("apikey")

	// Check if API key was provided
	if apiKey != "" {
		// Look up the owner of the API key
		loggedInUser, err = com.GetAPIKeyUser(apiKey)
	} else {
		// No API key was provided. Check for a client certificate instead
		loggedInUser, err = extractUserFromClientCert(w, r)
	}

	// Check for any errors
	if err != nil || loggedInUser == "" {
		err = fmt.Errorf("Incorrect or unknown API key and certificate")
		return
	}

	// Log the incoming request
	logReq(r, loggedInUser)
	return
}

// collectInfo is an internal function which:
//   1. Authenticates incoming requests
//   2. Extracts the database owner, name, and commit ID from the request
//   3. Checks permissions
func collectInfo(w http.ResponseWriter, r *http.Request) (loggedInUser, dbOwner, dbName, commitID string, httpStatus int, err error) {
	// Authenticate the request
	loggedInUser, err = checkAuth(w, r)
	if err != nil {
		httpStatus = http.StatusUnauthorized
		return
	}

	// Extract the database owner name, database name, and (optional) commit ID for the database from the request
	dbOwner, dbName, commitID, err = com.GetFormODC(r)
	if err != nil {
		httpStatus = http.StatusInternalServerError
		return
	}
	dbFolder := "/"

	// Check if the user has access to the requested database
	// Check if the requested database exists
	exists, err := com.CheckDBExists(loggedInUser, dbOwner, dbFolder, dbName)
	if err != nil {
		httpStatus = http.StatusInternalServerError
		return
	}
	if !exists {
		httpStatus = http.StatusNotFound
		err = fmt.Errorf("Database does not exist, or user isn't authorised to access it")
		return
	}
	return
}

// collectInfoAndOpen is an internal function which:
//   1. Calls collectInfo() to authenticate the request + collect the user/database/commit/etc details
//   2. Fetches the database from Minio
//   3. Opens the database, returning the connection handle
// This function exists purely because this code is common to most of the handlers
func collectInfoAndOpen(w http.ResponseWriter, r *http.Request) (sdb *sqlite.Conn, httpStatus int, err error) {
	// Authenticate the request and collect details for the requested database
	loggedInUser, dbOwner, dbName, commitID, httpStatus, err := collectInfo(w, r)
	if err != nil {
		httpStatus = http.StatusInternalServerError
		return
	}
	dbFolder := "/"

	// Get Minio bucket
	bucket, id, _, err := com.MinioLocation(dbOwner, dbFolder, dbName, commitID, loggedInUser)
	if err != nil {
		httpStatus = http.StatusInternalServerError
		return
	}

	// Sanity check
	if id == "" {
		// The requested database wasn't found, or the user doesn't have permission to access it
		err = fmt.Errorf("Requested database not found")
		log.Printf("Requested database not found. Owner: '%s%s%s'", dbOwner, dbFolder, dbName)
		httpStatus = http.StatusNotFound
		return
	}

	// Retrieve database file from Minio, using locally cached version if it's already there
	var newDB string
	newDB, err = com.RetrieveDatabaseFile(bucket, id)
	if err != nil {
		httpStatus = http.StatusNotFound
		return
	}

	// Open the SQLite database in read only mode
	sdb, err = sqlite.Open(newDB, sqlite.OpenReadOnly)
	if err != nil {
		log.Printf("Couldn't open database in collectInfoAndOpen(): %s", err)
		httpStatus = http.StatusInternalServerError
		return
	}
	if err = sdb.EnableExtendedResultCodes(true); err != nil {
		log.Printf("Couldn't enable extended result codes in collectInfoAndOpen(): %v\n", err.Error())
		httpStatus = http.StatusInternalServerError
		return
	}
	return
}

func extractUserFromClientCert(w http.ResponseWriter, r *http.Request) (userAcc string, err error) {
	// Check if a client certificate was provided
	if len(r.TLS.PeerCertificates) == 0 {
		err = errors.New("No client certificate provided")
		return
	}

	// Extract the account name and associated server from the validated client certificate
	cn := r.TLS.PeerCertificates[0].Subject.CommonName
	if cn == "" {
		// Common name is empty
		err = errors.New("Common name is blank in client certificate")
		return
	}
	s := strings.Split(cn, "@")
	if len(s) < 2 {
		err = errors.New("Missing information in client certificate")
		return
	}
	userAcc = s[0]
	certServer := s[1]
	if userAcc == "" || certServer == "" {
		// Missing details in common name field
		err = errors.New("Missing information in client certificate")
		return
	}

	// Verify the running server matches the one in the certificate
	db4sServer := com.Conf.DB4S.Server
	if certServer != db4sServer {
		err = fmt.Errorf("Server name in certificate '%s' doesn't match DB4S server '%s'\n", certServer,
			db4sServer)
		return
	}

	// Everything is ok, so return
	return
}

// handleWrapper does nothing useful except interface between types
// TODO: Get rid of this, as it shouldn't be needed
func handleWrapper(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		// Call the original function
		fn(w, r)
	}
}

// jsonErr returns an error message wrapped in JSON, for (potentially) easier processing by an API caller
func jsonErr(w http.ResponseWriter, msg string, statusCode int) {
	je := com.JsonError{
		Error: msg,
	}
	jsonData, err := json.MarshalIndent(je, "", "  ")
	if err != nil {
		errMsg := fmt.Sprintf("A 2nd error occurred when JSON marshalling an error structure: %v\n", err)
		log.Print(errMsg)
		http.Error(w, errMsg, http.StatusInternalServerError)
		fmt.Fprintf(w, `{"error":"An error occurred when marshalling JSON inside jsonErr()"}`)
		return
	}
	w.WriteHeader(statusCode)
	fmt.Fprintf(w, string(jsonData))
}

// logReq writes an entry for the incoming request to the request log
func logReq(r *http.Request, loggedInUser string) {
	fmt.Fprintf(reqLog, "%v - %s [%s] \"%s %s %s\" \"-\" \"-\" \"%s\" \"%s\"\n", r.RemoteAddr,
		loggedInUser, time.Now().Format(time.RFC3339Nano), r.Method, r.URL, r.Proto,
		r.Referer(), r.Header.Get("User-Agent"))
}
<|endoftext|>


<commit_before>package functions

import (
	"archive/zip"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"
)

var (
	Bold = func(content string) (template.HTML, error) {
		return template.HTML("" + content + ""), nil
	}
	Italic = func(content string) (template.HTML, error) {
		return template.HTML("" + content + ""), nil
	}
	Underline = func(content string) (template.HTML, error) {
		return template.HTML("" + content + ""), nil
	}
)

func Exists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if os.IsNotExist(err) {
		return false
	}
	return true
}

func RemoveFileIfExist(filename string) error {
	if !Exists(filename) {
		continue
	}
	err := os.Remove(filename)
	if err != nil {
		return err
	}
	return nil
}

func CreateDirIfNotExist(dir string) error {
	if !Exists(dir) {
		err := os.Mkdir(dir, 0744)
		if err != nil {
			return fmt.Errorf("Failed to create dir %v %v", dir, err)
		}
	}
	return nil
}

func MakeTimestamp() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}

func GetFilenameNoExtension(s string) string {
	n := strings.LastIndexByte(s, '.')
	if n >= 0 {
		return s[:n]
	}
	return s
}

func RemoveDuplicates(a []string) []string {
	result := []string{}
	seen := map[string]string{}
	for _, val := range a {
		if _, ok := seen[val]; !ok {
			result = append(result, val)
			seen[val] = val
		}
	}
	return result
}

func encodeBase64(b []byte) string {
	return base64.StdEncoding.EncodeToString(b)
}

func decodeBase64(s string) []byte {
	data, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		panic(err)
	}
	return data
}

func Encrypt(key, text []byte) string {
	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}
	ciphertext := make([]byte, aes.BlockSize+len(text))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}
	cfb := cipher.NewCFBEncrypter(block, iv)
	cfb.XORKeyStream(ciphertext[aes.BlockSize:], text)
	return encodeBase64(ciphertext)
}

func Decrypt(key []byte, b64 string) string {
	text := decodeBase64(b64)
	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}
	if len(text) < aes.BlockSize {
		panic("ciphertext too short")
	}
	iv := text[:aes.BlockSize]
	text = text[aes.BlockSize:]
	cfb := cipher.NewCFBDecrypter(block, iv)
	cfb.XORKeyStream(text, text)
	return string(text)
}

func CopyFile(src, dst string) (err error) {
	sfi, err := os.Stat(src)
	if err != nil {
		return
	}
	if !sfi.Mode().IsRegular() {
		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
	}
	dfi, err := os.Stat(dst)
	if err != nil {
		if !os.IsNotExist(err) {
			return
		}
	} else {
		if !(dfi.Mode().IsRegular()) {
			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
		}
		if os.SameFile(sfi, dfi) {
			return
		}
	}
	if err = os.Link(src, dst); err == nil {
		return
	}
	err = copyFileContents(src, dst)
	return
}

func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}

func Zipit(source, target string) error {
	zipfile, err := os.Create(target)
	if err != nil {
		return err
	}
	defer zipfile.Close()

	archive := zip.NewWriter(zipfile)
	defer archive.Close()

	info, err := os.Stat(source)
	if err != nil {
		return nil
	}

	var baseDir string
	if info.IsDir() {
		baseDir = filepath.Base(source)
	}

	filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}

		if baseDir != "" {
			header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
		}

		if info.IsDir() {
			header.Name += "/"
		} else {
			header.Method = zip.Deflate
		}

		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(writer, file)
		return err
	})

	return err
}

func Unzip(archive, target string) error {
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(target, 0755); err != nil {
		return err
	}

	for _, file := range reader.File {
		path := filepath.Join(target, file.Name)
		if file.FileInfo().IsDir() {
			os.MkdirAll(path, file.Mode())
			continue
		}

		fileReader, err := file.Open()
		if err != nil {
			return err
		}
		defer fileReader.Close()

		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
		if err != nil {
			return err
		}
		defer targetFile.Close()

		if _, err := io.Copy(targetFile, fileReader); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Fixed continue error<commit_after>package functions

import (
	"archive/zip"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"
)

var (
	Bold = func(content string) (template.HTML, error) {
		return template.HTML("" + content + ""), nil
	}
	Italic = func(content string) (template.HTML, error) {
		return template.HTML("" + content + ""), nil
	}
	Underline = func(content string) (template.HTML, error) {
		return template.HTML("" + content + ""), nil
	}
)

func Exists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if os.IsNotExist(err) {
		return false
	}
	return true
}

func RemoveFileIfExist(filename string) error {
	if Exists(filename) {
		err := os.Remove(filename)
		if err != nil {
			return err
		}
	}
	return nil
}

func CreateDirIfNotExist(dir string) error {
	if !Exists(dir) {
		err := os.Mkdir(dir, 0744)
		if err != nil {
			return fmt.Errorf("Failed to create dir %v %v", dir, err)
		}
	}
	return nil
}

func MakeTimestamp() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}

func GetFilenameNoExtension(s string) string {
	n := strings.LastIndexByte(s, '.')
	if n >= 0 {
		return s[:n]
	}
	return s
}

func RemoveDuplicates(a []string) []string {
	result := []string{}
	seen := map[string]string{}
	for _, val := range a {
		if _, ok := seen[val]; !ok {
			result = append(result, val)
			seen[val] = val
		}
	}
	return result
}

func encodeBase64(b []byte) string {
	return base64.StdEncoding.EncodeToString(b)
}

func decodeBase64(s string) []byte {
	data, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		panic(err)
	}
	return data
}

func Encrypt(key, text []byte) string {
	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}
	ciphertext := make([]byte, aes.BlockSize+len(text))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}
	cfb := cipher.NewCFBEncrypter(block, iv)
	cfb.XORKeyStream(ciphertext[aes.BlockSize:], text)
	return encodeBase64(ciphertext)
}

func Decrypt(key []byte, b64 string) string {
	text := decodeBase64(b64)
	block, err := aes.NewCipher(key)
	if err != nil {
		panic(err)
	}
	if len(text) < aes.BlockSize {
		panic("ciphertext too short")
	}
	iv := text[:aes.BlockSize]
	text = text[aes.BlockSize:]
	cfb := cipher.NewCFBDecrypter(block, iv)
	cfb.XORKeyStream(text, text)
	return string(text)
}

func CopyFile(src, dst string) (err error) {
	sfi, err := os.Stat(src)
	if err != nil {
		return
	}
	if !sfi.Mode().IsRegular() {
		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
	}
	dfi, err := os.Stat(dst)
	if err != nil {
		if !os.IsNotExist(err) {
			return
		}
	} else {
		if !(dfi.Mode().IsRegular()) {
			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
		}
		if os.SameFile(sfi, dfi) {
			return
		}
	}
	if err = os.Link(src, dst); err == nil {
		return
	}
	err = copyFileContents(src, dst)
	return
}

func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}

func Zipit(source, target string) error {
	zipfile, err := os.Create(target)
	if err != nil {
		return err
	}
	defer zipfile.Close()

	archive := zip.NewWriter(zipfile)
	defer archive.Close()

	info, err := os.Stat(source)
	if err != nil {
		return nil
	}

	var baseDir string
	if info.IsDir() {
		baseDir = filepath.Base(source)
	}

	filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}

		if baseDir != "" {
			header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
		}

		if info.IsDir() {
			header.Name += "/"
		} else {
			header.Method = zip.Deflate
		}

		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(writer, file)
		return err
	})

	return err
}

func Unzip(archive, target string) error {
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(target, 0755); err != nil {
		return err
	}

	for _, file := range reader.File {
		path := filepath.Join(target, file.Name)
		if file.FileInfo().IsDir() {
			os.MkdirAll(path, file.Mode())
			continue
		}

		fileReader, err := file.Open()
		if err != nil {
			return err
		}
		defer fileReader.Close()

		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
		if err != nil {
			return err
		}
		defer targetFile.Close()

		if _, err := io.Copy(targetFile, fileReader); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>remove test.go<commit_after><|endoftext|>


<commit_before>// A set of tests for AGI in Go
//
// Copyright (C) 2013 - 2014, Lefteris Zafiris <zaf.000@gmail.com>
// This program is free software, distributed under the terms of
// the BSD 3-Clause License. See the LICENSE file
// at the top of the source tree.
//
// Based on agi-test.agi from asterisk source tree.
// Can be used both as standalone AGI app or a FastAGI server
// if called with the flag '-spawn_fagi'

package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"net"

	"github.com/zaf/agi"
)

var listen = flag.Bool("spawn_fagi", false, "Spawn as a FastAGI server")

func main() {
	flag.Parse()
	if *listen {
		//If called as a FastAGI server
		ln, err := net.Listen("tcp", ":4573")
		if err != nil {
			log.Fatal(err)
		}
		defer ln.Close()
		for {
			conn, err := ln.Accept()
			if err != nil {
				log.Println(err)
				continue
			}
			go spawnAgi(conn)
		}
	} else {
		//If called as standalone AGI app
		spawnAgi(nil)
	}
}

func spawnAgi(c net.Conn) {
	var myAgi *agi.Session
	var err error
	if c != nil {
		//Create a new FastAGI session
		rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
		myAgi, err = agi.Init(rw)
		defer func() {
			c.Close()
			myAgi.Destroy()
		}()
	} else {
		//Create a new AGI session
		myAgi, err = agi.Init(nil)
		defer myAgi.Destroy()
	}
	if err != nil {
		log.Printf("Error Parsing AGI environment: %v\n", err)
		return
	}
	testAgi(myAgi)
	return
}

func testAgi(sess *agi.Session) {
	//Perform some tests
	var tests, pass, fail int
	var err error
	var r agi.Reply

	sess.Verbose("Testing answer...")
	r, err = sess.Answer()
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing channelstatus...")
	r, err = sess.ChannelStatus()
	if err != nil || r.Res != 6 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing databaseput...")
	r, err = sess.DatabasePut("test", "my_key", "true")
	if err != nil || r.Res != 1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing databaseget...")
	r, err = sess.DatabaseGet("test", "my_key")
	if err != nil || r.Res != 1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing databasedel...")
	r, err = sess.DatabaseDel("test", "my_key")
	if err != nil || r.Res != 1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing databasedeltree...")
	r, err = sess.DatabaseDelTree("test")
	if err != nil || r.Res != 1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing streamfile...")
	r, err = sess.StreamFile("beep", "")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing sendtext...")
	r, err = sess.SendText("Hello World")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing sendimage...")
	r, err = sess.SendImage("asterisk-image")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing saynumber...")
	r, err = sess.SayNumber(192837465, "")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing wait for digit...")
	r, err = sess.WaitForDigit(3000)
	if err != nil || r.Res == -1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing redord...")
	r, err = sess.RecordFile("/tmp/testagi", "alaw", "1234567890*#", 3000)
	if err != nil || r.Res == -1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing record playback...")
	r, err = sess.StreamFile("/tmp/testagi", "")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing set variable...")
	r, err = sess.SetVariable("testagi", "foo")
	if err != nil || r.Res != 1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing get variable...")
	r, err = sess.GetVariable("testagi")
	if err != nil || r.Res != 1 || r.Dat != "foo" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing get full variable...")
	r, err = sess.GetFullVariable("${testagi}")
	if err != nil || r.Res != 1 || r.Dat != "foo" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing exec...")
	r, err = sess.Exec("Wait", "3")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("================== Complete ======================")
	sess.Verbose(fmt.Sprintf("%d tests completed, %d passed, %d failed", tests, pass, fail))
	sess.Verbose("==================================================")
	return
}
<commit_msg>Two small fixes in agi-test example: A typo and a test case.<commit_after>// A set of tests for AGI in Go
//
// Copyright (C) 2013 - 2014, Lefteris Zafiris <zaf.000@gmail.com>
// This program is free software, distributed under the terms of
// the BSD 3-Clause License. See the LICENSE file
// at the top of the source tree.
//
// Based on agi-test.agi from asterisk source tree.
// Can be used both as standalone AGI app or a FastAGI server
// if called with the flag '-spawn_fagi'

package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"net"

	"github.com/zaf/agi"
)

var listen = flag.Bool("spawn_fagi", false, "Spawn as a FastAGI server")

func main() {
	flag.Parse()
	if *listen {
		//If called as a FastAGI server
		ln, err := net.Listen("tcp", ":4573")
		if err != nil {
			log.Fatal(err)
		}
		defer ln.Close()
		for {
			conn, err := ln.Accept()
			if err != nil {
				log.Println(err)
				continue
			}
			go spawnAgi(conn)
		}
	} else {
		//If called as standalone AGI app
		spawnAgi(nil)
	}
}

func spawnAgi(c net.Conn) {
	var myAgi *agi.Session
	var err error
	if c != nil {
		//Create a new FastAGI session
		rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
		myAgi, err = agi.Init(rw)
		defer func() {
			c.Close()
			myAgi.Destroy()
		}()
	} else {
		//Create a new AGI session
		myAgi, err = agi.Init(nil)
		defer myAgi.Destroy()
	}
	if err != nil {
		log.Printf("Error Parsing AGI environment: %v\n", err)
		return
	}
	testAgi(myAgi)
	return
}

func testAgi(sess *agi.Session) {
	//Perform some tests
	var tests, pass, fail int
	var err error
	var r agi.Reply

	sess.Verbose("Testing answer...")
	r, err = sess.Answer()
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing channelstatus...")
	r, err = sess.ChannelStatus()
	if err != nil || r.Res != 6 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing databaseput...")
	r, err = sess.DatabasePut("test", "my_key", "true")
	if err != nil || r.Res != 1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing databaseget...")
	r, err = sess.DatabaseGet("test", "my_key")
	if err != nil || r.Res != 1 || r.Dat != "true" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing databasedel...")
	r, err = sess.DatabaseDel("test", "my_key")
	if err != nil || r.Res != 1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing databasedeltree...")
	r, err = sess.DatabaseDelTree("test")
	if err != nil || r.Res != 1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing streamfile...")
	r, err = sess.StreamFile("beep", "")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing sendtext...")
	r, err = sess.SendText("Hello World")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing sendimage...")
	r, err = sess.SendImage("asterisk-image")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing saynumber...")
	r, err = sess.SayNumber(192837465, "")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing wait for digit...")
	r, err = sess.WaitForDigit(3000)
	if err != nil || r.Res == -1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing record...")
	r, err = sess.RecordFile("/tmp/testagi", "alaw", "1234567890*#", 3000)
	if err != nil || r.Res == -1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing record playback...")
	r, err = sess.StreamFile("/tmp/testagi", "")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing set variable...")
	r, err = sess.SetVariable("testagi", "foo")
	if err != nil || r.Res != 1 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing get variable...")
	r, err = sess.GetVariable("testagi")
	if err != nil || r.Res != 1 || r.Dat != "foo" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing get full variable...")
	r, err = sess.GetFullVariable("${testagi}")
	if err != nil || r.Res != 1 || r.Dat != "foo" {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("Testing exec...")
	r, err = sess.Exec("Wait", "3")
	if err != nil || r.Res != 0 {
		sess.Verbose("Failed.")
		fail++
	} else {
		pass++
	}
	tests++

	sess.Verbose("================== Complete ======================")
	sess.Verbose(fmt.Sprintf("%d tests completed, %d passed, %d failed", tests, pass, fail))
	sess.Verbose("==================================================")
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/dynport/urknall/utils"
)

// The "FileCommand" is used to write files to the host being provisioned. The go templating mechanism (see
// http://golang.org/pkg/text/template) is applied on the file's content using the package. Thereby it is possible to
// have dynamic content (based on the package's configuration) for the file content and at the same time store it in
// an asset (which is generated at compile time). Please note that the underlying actions will panic if either no path
// or content are given.
type FileCommand struct {
	Path        string      // Path to the file to create.
	Content     string      // Content of the file to create.
	Owner       string      // Owner of the file to create (root per default).
	Permissions os.FileMode // Permissions of the file created (only changed from system default if set).
}

func (cmd *FileCommand) Render(i interface{}) {
	cmd.Path = utils.MustRenderTemplate(cmd.Path, i)
	cmd.Content = utils.MustRenderTemplate(cmd.Content, i)
}

func (cmd *FileCommand) Validate() error {
	if cmd.Path == "" {
		return fmt.Errorf("no path given")
	}

	if cmd.Content == "" {
		return fmt.Errorf("no content given for file %q", cmd.Path)
	}

	return nil
}

// Helper method to create a file at the given path with the given content, and with owner and permissions set
// accordingly. The "Owner" and "Permissions" options are optional in the sense that they are ignored if set to go's
// default value.
func WriteFile(path string, content string, owner string, permissions os.FileMode) *FileCommand {
	return &FileCommand{Path: path, Content: content, Owner: owner, Permissions: permissions}
}

var b64 = base64.StdEncoding

func (fc *FileCommand) Shell() string {
	buf := &bytes.Buffer{}

	// Zip the content.
	zipper := gzip.NewWriter(buf)
	zipper.Write([]byte(fc.Content))
	zipper.Flush()
	zipper.Close()

	// Encode the zipped content in Base64.
	encoded := b64.EncodeToString(buf.Bytes())

	// Compute sha256 hash of the encoded and zipped content.
	hash := sha256.New()
	hash.Write([]byte(fc.Content))

	// Create temporary filename (hash as filename).
	tmpPath := fmt.Sprintf("/tmp/wunderscale.%x", hash.Sum(nil))

	// Get directory part of target file.
	dir := filepath.Dir(fc.Path)

	// Create command, that will decode and unzip the content and write to the temporary file.
	cmd := ""
	cmd += fmt.Sprintf("mkdir -p %s", dir)
	cmd += fmt.Sprintf(" && echo %s | base64 -d | gunzip > %s", encoded, tmpPath)
	if fc.Owner != "" { // If owner given, change accordingly.
		cmd += fmt.Sprintf(" && chown %s %s", fc.Owner, tmpPath)
	}
	if fc.Permissions > 0 { // If mode given, change accordingly.
		cmd += fmt.Sprintf(" && chmod %o %s", fc.Permissions, tmpPath)
	}
	// Move the temporary file to the requested location.
	cmd += fmt.Sprintf(" && mv %s %s", tmpPath, fc.Path)
	return cmd
}

func (fc *FileCommand) Logging() string {
	sList := []string{"[FILE   ]"}

	if fc.Owner != "" && fc.Owner != "root" {
		sList = append(sList, fmt.Sprintf("[CHOWN:%s]", fc.Owner))
	}

	if fc.Permissions != 0 {
		sList = append(sList, fmt.Sprintf("[CHMOD:%.4o]", fc.Permissions))
	}

	sList = append(sList, " "+fc.Path)

	cLen := len(fc.Content)
	if cLen > 50 {
		cLen = 50
	}
	//sList = append(sList, fmt.Sprintf(" << %s", strings.Replace(string(fc.Content[0:cLen]), "\n", "⁋", -1)))
	return strings.Join(sList, "")
}
<commit_msg>added SendFile command to basic commands<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/dynport/urknall/utils"
)

// The "FileCommand" is used to write files to the host being provisioned. The go templating mechanism (see
// http://golang.org/pkg/text/template) is applied on the file's content using the package. Thereby it is possible to
// have dynamic content (based on the package's configuration) for the file content and at the same time store it in
// an asset (which is generated at compile time). Please note that the underlying actions will panic if either no path
// or content are given.
type FileCommand struct {
	Path        string      // Path to the file to create.
	Content     string      // Content of the file to create.
	Owner       string      // Owner of the file to create (root per default).
	Permissions os.FileMode // Permissions of the file created (only changed from system default if set).
}

func (cmd *FileCommand) Render(i interface{}) {
	cmd.Path = utils.MustRenderTemplate(cmd.Path, i)
	cmd.Content = utils.MustRenderTemplate(cmd.Content, i)
}

func (cmd *FileCommand) Validate() error {
	if cmd.Path == "" {
		return fmt.Errorf("no path given")
	}

	if cmd.Content == "" {
		return fmt.Errorf("no content given for file %q", cmd.Path)
	}

	return nil
}

// Helper method to create a file at the given path with the given content, and with owner and permissions set
// accordingly. The "Owner" and "Permissions" options are optional in the sense that they are ignored if set to go's
// default value.
func WriteFile(path string, content string, owner string, permissions os.FileMode) *FileCommand {
	return &FileCommand{Path: path, Content: content, Owner: owner, Permissions: permissions}
}

var b64 = base64.StdEncoding

func (fc *FileCommand) Shell() string {
	buf := &bytes.Buffer{}

	// Zip the content.
	zipper := gzip.NewWriter(buf)
	zipper.Write([]byte(fc.Content))
	zipper.Flush()
	zipper.Close()

	// Encode the zipped content in Base64.
	encoded := b64.EncodeToString(buf.Bytes())

	// Compute sha256 hash of the encoded and zipped content.
	hash := sha256.New()
	hash.Write([]byte(fc.Content))

	// Create temporary filename (hash as filename).
	tmpPath := fmt.Sprintf("/tmp/wunderscale.%x", hash.Sum(nil))

	// Get directory part of target file.
	dir := filepath.Dir(fc.Path)

	// Create command, that will decode and unzip the content and write to the temporary file.
	cmd := ""
	cmd += fmt.Sprintf("mkdir -p %s", dir)
	cmd += fmt.Sprintf(" && echo %s | base64 -d | gunzip > %s", encoded, tmpPath)
	if fc.Owner != "" { // If owner given, change accordingly.
		cmd += fmt.Sprintf(" && chown %s %s", fc.Owner, tmpPath)
	}
	if fc.Permissions > 0 { // If mode given, change accordingly.
		cmd += fmt.Sprintf(" && chmod %o %s", fc.Permissions, tmpPath)
	}
	// Move the temporary file to the requested location.
	cmd += fmt.Sprintf(" && mv %s %s", tmpPath, fc.Path)
	return cmd
}

func (fc *FileCommand) Logging() string {
	sList := []string{"[FILE   ]"}

	if fc.Owner != "" && fc.Owner != "root" {
		sList = append(sList, fmt.Sprintf("[CHOWN:%s]", fc.Owner))
	}

	if fc.Permissions != 0 {
		sList = append(sList, fmt.Sprintf("[CHMOD:%.4o]", fc.Permissions))
	}

	sList = append(sList, " "+fc.Path)

	cLen := len(fc.Content)
	if cLen > 50 {
		cLen = 50
	}
	//sList = append(sList, fmt.Sprintf(" << %s", strings.Replace(string(fc.Content[0:cLen]), "\n", "⁋", -1)))
	return strings.Join(sList, "")
}

type FileSendCommand struct {
	Source      string
	Target      string
	Owner       string
	Permissions os.FileMode
}

func SendFile(source, target, owner string, perm os.FileMode) *FileSendCommand {
	return &FileSendCommand{
		Source:      source,
		Target:      target,
		Owner:       owner,
		Permissions: perm,
	}
}

func (fsc *FileSendCommand) Render(i interface{}) {
	fsc.Source = utils.MustRenderTemplate(fsc.Source, i)
	fsc.Target = utils.MustRenderTemplate(fsc.Target, i)
}

func (fsc *FileSendCommand) Validate() error {
	if fsc.Source == "" {
		return fmt.Errorf("no source path given")
	}

	if _, e := os.Stat(fsc.Source); e != nil {
		return e
	}

	if fsc.Target == "" {
		return fmt.Errorf("no target path given for file %q", fsc.Source)
	}

	return nil
}

func (fsc *FileSendCommand) sourceHash() string {
	fh, e := os.Open(fsc.Source)
	if e != nil {
		panic(e)
	}
	defer fh.Close()

	hash := sha1.New()
	if _, e = io.Copy(hash, fh); e != nil {
		panic(e)
	}

	return hex.EncodeToString(hash.Sum(nil))
}

func (fsc *FileSendCommand) Shell() string {
	sList := []string{
		fmt.Sprintf("echo %q", fsc.sourceHash()), // nope use content hash
		fmt.Sprintf("cat - > %s", fsc.Target),
	}

	if fsc.Owner != "root" {
		sList = append(sList, fmt.Sprintf("chown %s %s", fsc.Owner, fsc.Target))
	}
	sList = append(sList, fmt.Sprintf("chmod %s %s", fsc.Permissions, fsc.Target))
	return strings.Join(sList, " && ")
}

func (fsc *FileSendCommand) Input() io.ReadCloser {
	fh, e := os.Open(fsc.Source)
	if e != nil {
		panic(e)
	}
	return fh
}

func (fsc *FileSendCommand) Logging() string {
	sList := []string{"[FILE   ]"}

	if fsc.Owner != "" && fsc.Owner != "root" {
		sList = append(sList, fmt.Sprintf("[CHOWN:%s]", fsc.Owner))
	}

	if fsc.Permissions != 0 {
		sList = append(sList, fmt.Sprintf("[CHMOD:%.4o]", fsc.Permissions))
	}

	sList = append(sList, fmt.Sprintf(" Writing local file %s to %s", fsc.Source, fsc.Target))

	return strings.Join(sList, "")
}
<|endoftext|>


<commit_before>// +build openbsd

package host

import (
	"bytes"
	"context"
	"encoding/binary"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"time"
	"unsafe"

	"github.com/shirou/gopsutil/internal/common"
	"github.com/shirou/gopsutil/process"
)

const (
	UTNameSize = 32 /* see MAXLOGNAME in <sys/param.h> */
	UTLineSize = 8
	UTHostSize = 16
)

func Info() (*InfoStat, error) {
	return InfoWithContext(context.Background())
}

func InfoWithContext(ctx context.Context) (*InfoStat, error) {
	ret := &InfoStat{
		OS:             runtime.GOOS,
		PlatformFamily: "openbsd",
	}

	hostname, err := os.Hostname()
	if err == nil {
		ret.Hostname = hostname
	}

	platform, family, version, err := PlatformInformation()
	if err == nil {
		ret.Platform = platform
		ret.PlatformFamily = family
		ret.PlatformVersion = version
	}
	system, role, err := Virtualization()
	if err == nil {
		ret.VirtualizationSystem = system
		ret.VirtualizationRole = role
	}

	procs, err := process.Pids()
	if err == nil {
		ret.Procs = uint64(len(procs))
	}

	boot, err := BootTime()
	if err == nil {
		ret.BootTime = boot
		ret.Uptime = uptime(boot)
	}

	return ret, nil
}

func BootTime() (uint64, error) {
	return BootTimeWithContext(context.Background())
}

func BootTimeWithContext(ctx context.Context) (uint64, error) {
	val, err := common.DoSysctrl("kern.boottime")
	if err != nil {
		return 0, err
	}

	boottime, err := strconv.ParseUint(val[0], 10, 64)
	if err != nil {
		return 0, err
	}

	return boottime, nil
}

func uptime(boot uint64) uint64 {
	return uint64(time.Now().Unix()) - boot
}

func Uptime() (uint64, error) {
	return UptimeWithContext(context.Background())
}

func UptimeWithContext(ctx context.Context) (uint64, error) {
	boot, err := BootTime()
	if err != nil {
		return 0, err
	}
	return uptime(boot), nil
}

func PlatformInformation() (string, string, string, error) {
	return PlatformInformationWithContext(context.Background())
}

func PlatformInformationWithContext(ctx context.Context) (string, string, string, error) {
	platform := ""
	family := ""
	version := ""
	uname, err := exec.LookPath("uname")
	if err != nil {
		return "", "", "", err
	}

	out, err := invoke.CommandWithContext(ctx, uname, "-s")
	if err == nil {
		platform = strings.ToLower(strings.TrimSpace(string(out)))
	}

	out, err = invoke.CommandWithContext(ctx, uname, "-r")
	if err == nil {
		version = strings.ToLower(strings.TrimSpace(string(out)))
	}

	return platform, family, version, nil
}

func Virtualization() (string, string, error) {
	return VirtualizationWithContext(context.Background())
}

func VirtualizationWithContext(ctx context.Context) (string, string, error) {
	return "", "", common.ErrNotImplementedError
}

func Users() ([]UserStat, error) {
	return UsersWithContext(context.Background())
}

func UsersWithContext(ctx context.Context) ([]UserStat, error) {
	var ret []UserStat
	utmpfile := "/var/run/utmp"
	file, err := os.Open(utmpfile)
	if err != nil {
		return ret, err
	}
	defer file.Close()

	buf, err := ioutil.ReadAll(file)
	if err != nil {
		return ret, err
	}

	u := Utmp{}
	entrySize := int(unsafe.Sizeof(u))
	count := len(buf) / entrySize

	for i := 0; i < count; i++ {
		b := buf[i*entrySize : i*entrySize+entrySize]
		var u Utmp
		br := bytes.NewReader(b)
		err := binary.Read(br, binary.LittleEndian, &u)
		if err != nil || u.Time == 0 {
			continue
		}
		user := UserStat{
			User:     common.IntToString(u.Name[:]),
			Terminal: common.IntToString(u.Line[:]),
			Host:     common.IntToString(u.Host[:]),
			Started:  int(u.Time),
		}

		ret = append(ret, user)
	}

	return ret, nil
}

func SensorsTemperatures() ([]TemperatureStat, error) {
	return SensorsTemperaturesWithContext(context.Background())
}

func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, error) {
	return []TemperatureStat{}, common.ErrNotImplementedError
}

func KernelVersion() (string, error) {
	return KernelVersionWithContext(context.Background())
}

func KernelVersionWithContext(ctx context.Context) (string, error) {
	_, _, version, err := PlatformInformation()
	return version, err
}
<commit_msg>[host][openbsd] Remove external calls to uname in PlatformInformation()<commit_after>// +build openbsd

package host

import (
	"bytes"
	"context"
	"encoding/binary"
	"io/ioutil"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
	"unsafe"

	"github.com/shirou/gopsutil/internal/common"
	"github.com/shirou/gopsutil/process"
	"golang.org/x/sys/unix"
)

const (
	UTNameSize = 32 /* see MAXLOGNAME in <sys/param.h> */
	UTLineSize = 8
	UTHostSize = 16
)

func Info() (*InfoStat, error) {
	return InfoWithContext(context.Background())
}

func InfoWithContext(ctx context.Context) (*InfoStat, error) {
	ret := &InfoStat{
		OS:             runtime.GOOS,
		PlatformFamily: "openbsd",
	}

	hostname, err := os.Hostname()
	if err == nil {
		ret.Hostname = hostname
	}

	platform, family, version, err := PlatformInformation()
	if err == nil {
		ret.Platform = platform
		ret.PlatformFamily = family
		ret.PlatformVersion = version
	}
	system, role, err := Virtualization()
	if err == nil {
		ret.VirtualizationSystem = system
		ret.VirtualizationRole = role
	}

	procs, err := process.Pids()
	if err == nil {
		ret.Procs = uint64(len(procs))
	}

	boot, err := BootTime()
	if err == nil {
		ret.BootTime = boot
		ret.Uptime = uptime(boot)
	}

	return ret, nil
}

func BootTime() (uint64, error) {
	return BootTimeWithContext(context.Background())
}

func BootTimeWithContext(ctx context.Context) (uint64, error) {
	val, err := common.DoSysctrl("kern.boottime")
	if err != nil {
		return 0, err
	}

	boottime, err := strconv.ParseUint(val[0], 10, 64)
	if err != nil {
		return 0, err
	}

	return boottime, nil
}

func uptime(boot uint64) uint64 {
	return uint64(time.Now().Unix()) - boot
}

func Uptime() (uint64, error) {
	return UptimeWithContext(context.Background())
}

func UptimeWithContext(ctx context.Context) (uint64, error) {
	boot, err := BootTime()
	if err != nil {
		return 0, err
	}
	return uptime(boot), nil
}

func PlatformInformation() (string, string, string, error) {
	return PlatformInformationWithContext(context.Background())
}

func PlatformInformationWithContext(ctx context.Context) (string, string, string, error) {
	platform := ""
	family := ""
	version := ""

	p, err := unix.Sysctl("kern.ostype")
	if err == nil {
		platform = strings.ToLower(p)
	}
	v, err := unix.Sysctl("kern.osrelease")
	if err == nil {
		version = strings.ToLower(v)
	}

	return platform, family, version, nil
}

func Virtualization() (string, string, error) {
	return VirtualizationWithContext(context.Background())
}

func VirtualizationWithContext(ctx context.Context) (string, string, error) {
	return "", "", common.ErrNotImplementedError
}

func Users() ([]UserStat, error) {
	return UsersWithContext(context.Background())
}

func UsersWithContext(ctx context.Context) ([]UserStat, error) {
	var ret []UserStat
	utmpfile := "/var/run/utmp"
	file, err := os.Open(utmpfile)
	if err != nil {
		return ret, err
	}
	defer file.Close()

	buf, err := ioutil.ReadAll(file)
	if err != nil {
		return ret, err
	}

	u := Utmp{}
	entrySize := int(unsafe.Sizeof(u))
	count := len(buf) / entrySize

	for i := 0; i < count; i++ {
		b := buf[i*entrySize : i*entrySize+entrySize]
		var u Utmp
		br := bytes.NewReader(b)
		err := binary.Read(br, binary.LittleEndian, &u)
		if err != nil || u.Time == 0 {
			continue
		}
		user := UserStat{
			User:     common.IntToString(u.Name[:]),
			Terminal: common.IntToString(u.Line[:]),
			Host:     common.IntToString(u.Host[:]),
			Started:  int(u.Time),
		}

		ret = append(ret, user)
	}

	return ret, nil
}

func SensorsTemperatures() ([]TemperatureStat, error) {
	return SensorsTemperaturesWithContext(context.Background())
}

func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, error) {
	return []TemperatureStat{}, common.ErrNotImplementedError
}

func KernelVersion() (string, error) {
	return KernelVersionWithContext(context.Background())
}

func KernelVersionWithContext(ctx context.Context) (string, error) {
	_, _, version, err := PlatformInformation()
	return version, err
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/satori/go.uuid"

	"go.pachyderm.com/pachyderm/src/pkg/container"
	"go.pachyderm.com/pachyderm/src/pps"
	"go.pachyderm.com/pachyderm/src/pps/persist"
	"go.pachyderm.com/pachyderm/src/pps/watch"
	"go.pedge.io/google-protobuf"
	"go.pedge.io/protolog"
	"golang.org/x/net/context"
)

var (
	emptyInstance = &google_protobuf.Empty{}
)

type apiServer struct {
	persistAPIClient persist.APIClient
	watchAPIClient   watch.APIClient
	containerClient  container.Client
}

func newAPIServer(
	persistAPIClient persist.APIClient,
	watchAPIClient watch.APIClient,
	containerClient container.Client,
) *apiServer {
	return &apiServer{persistAPIClient, watchAPIClient, containerClient}
}

func (a *apiServer) CreateJob(ctx context.Context, request *pps.CreateJobRequest) (response *pps.Job, err error) {
	persistJob, err := a.persistAPIClient.CreateJob(ctx, jobToPersist(request.Job))
	if err != nil {
		return nil, err
	}
	return persistToJob(persistJob), nil
}

func (a *apiServer) GetJob(ctx context.Context, request *pps.GetJobRequest) (response *pps.Job, err error) {
	persistJob, err := a.persistAPIClient.GetJobByID(ctx, &google_protobuf.StringValue{Value: request.JobId})
	if err != nil {
		return nil, err
	}
	return persistToJob(persistJob), nil
}

func (a *apiServer) GetJobsByPipelineName(ctx context.Context, request *pps.GetJobsByPipelineNameRequest) (response *pps.Jobs, err error) {
	persistPipelines, err := a.persistAPIClient.GetPipelinesByName(ctx, &google_protobuf.StringValue{Value: request.PipelineName})
	if err != nil {
		return nil, err
	}
	var jobs []*pps.Job
	for _, persistPipeline := range persistPipelines.Pipeline {
		persistJobs, err := a.persistAPIClient.GetJobsByPipelineID(ctx, &google_protobuf.StringValue{Value: persistPipeline.Id})
		if err != nil {
			return nil, err
		}
		iJobs := persistToJobs(persistJobs)
		jobs = append(jobs, iJobs.Job...)
	}
	return &pps.Jobs{
		Job: jobs,
	}, nil
}

func (a *apiServer) StartJob(ctx context.Context, request *pps.StartJobRequest) (response *google_protobuf.Empty, err error) {
	persistJob, err := a.persistAPIClient.GetJobByID(ctx, &google_protobuf.StringValue{Value: request.JobId})
	if err != nil {
		return nil, err
	}
	if err := a.startPersistJob(persistJob); err != nil {
		return nil, err
	}
	return emptyInstance, nil
}

func (a *apiServer) GetJobStatus(ctx context.Context, request *pps.GetJobStatusRequest) (response *pps.JobStatus, err error) {
	persistJobStatuses, err := a.persistAPIClient.GetJobStatusesByJobID(ctx, &google_protobuf.StringValue{Value: request.JobId})
	if err != nil {
		return nil, err
	}
	if len(persistJobStatuses.JobStatus) == 0 {
		return nil, fmt.Errorf("pachyderm.pps.server: no job statuses for %s", request.JobId)
	}
	return persistToJobStatus(persistJobStatuses.JobStatus[0]), nil
}

func (a *apiServer) GetJobLogs(request *pps.GetJobLogsRequest, responseServer pps.API_GetJobLogsServer) (err error) {
	persistJobLogs, err := a.persistAPIClient.GetJobLogsByJobID(context.Background(), &google_protobuf.StringValue{Value: request.JobId})
	if err != nil {
		return err
	}
	for _, persistJobLog := range persistJobLogs.JobLog {
		if persistJobLog.OutputStream == request.OutputStream {
			if err := responseServer.Send(&google_protobuf.BytesValue{Value: persistJobLog.Value}); err != nil {
				return err
			}
		}
	}
	return nil
}

func (a *apiServer) CreatePipeline(ctx context.Context, request *pps.CreatePipelineRequest) (response *pps.Pipeline, err error) {
	persistPipeline, err := a.persistAPIClient.CreatePipeline(ctx, pipelineToPersist(request.Pipeline))
	if err != nil {
		return nil, err
	}
	return persistToPipeline(persistPipeline), nil
}

func (a *apiServer) GetPipeline(ctx context.Context, request *pps.GetPipelineRequest) (response *pps.Pipeline, err error) {
	persistPipelines, err := a.persistAPIClient.GetPipelinesByName(ctx, &google_protobuf.StringValue{Value: request.PipelineName})
	if err != nil {
		return nil, err
	}
	if len(persistPipelines.Pipeline) == 0 {
		return nil, fmt.Errorf("pachyderm.pps.server: no piplines for name %s", request.PipelineName)
	}
	return persistToPipeline(persistPipelines.Pipeline[0]), nil
}

func (a *apiServer) GetAllPipelines(ctx context.Context, request *google_protobuf.Empty) (response *pps.Pipelines, err error) {
	persistPipelines, err := a.persistAPIClient.GetAllPipelines(ctx, request)
	if err != nil {
		return nil, err
	}
	pipelineMap := make(map[string]*pps.Pipeline)
	for _, persistPipeline := range persistPipelines.Pipeline {
		// pipelines are ordered newest to oldest, so if we have already
		// seen a pipeline with the same name, it is newer
		if _, ok := pipelineMap[persistPipeline.Name]; !ok {
			pipelineMap[persistPipeline.Name] = persistToPipeline(persistPipeline)
		}
	}
	pipelines := make([]*pps.Pipeline, len(pipelineMap))
	i := 0
	for _, pipeline := range pipelineMap {
		pipelines[i] = pipeline
		i++
	}
	return &pps.Pipelines{
		Pipeline: pipelines,
	}, nil
}

func (a *apiServer) startPersistJob(persistJob *persist.Job) error {
	if _, err := a.persistAPIClient.CreateJobStatus(
		context.Background(),
		&persist.JobStatus{
			JobId: persistJob.Id,
			Type:  pps.JobStatusType_JOB_STATUS_TYPE_STARTED,
		},
	); err != nil {
		return err
	}
	// TODO(pedge): throttling? worker pool?
	go func() {
		if err := a.runJob(persistJob); err != nil {
			protolog.Errorln(err.Error())
			// TODO(pedge): how to handle the error?
			if _, err = a.persistAPIClient.CreateJobStatus(
				context.Background(),
				&persist.JobStatus{
					JobId:   persistJob.Id,
					Type:    pps.JobStatusType_JOB_STATUS_TYPE_ERROR,
					Message: err.Error(),
				},
			); err != nil {
				protolog.Errorln(err.Error())
			}
		} else {
			// TODO(pedge): how to handle the error?
			if _, err = a.persistAPIClient.CreateJobStatus(
				context.Background(),
				&persist.JobStatus{
					JobId: persistJob.Id,
					Type:  pps.JobStatusType_JOB_STATUS_TYPE_SUCCESS,
				},
			); err != nil {
				protolog.Errorln(err.Error())
			}
		}
	}()
	return nil
}

func (a *apiServer) runJob(persistJob *persist.Job) error {
	switch {
	case persistJob.GetTransform() != nil:
		return a.reallyRunJob(strings.Replace(uuid.NewV4().String(), "-", "", -1), persistJob.Id, persistJob.GetTransform(), persistJob.JobInput, persistJob.JobOutput, 1)
	case persistJob.GetPipelineId() != "":
		persistPipeline, err := a.persistAPIClient.GetPipelineByID(
			context.Background(),
			&google_protobuf.StringValue{Value: persistJob.GetPipelineId()},
		)
		if err != nil {
			return err
		}
		if persistPipeline.Transform == nil {
			return fmt.Errorf("pachyderm.pps.server: transform not set on pipeline %v", persistPipeline)
		}
		return a.reallyRunJob(persistPipeline.Name, persistJob.Id, persistPipeline.Transform, persistJob.JobInput, persistJob.JobOutput, 1)
	default:
		return fmt.Errorf("pachyderm.pps.server: neither transform or pipeline id set on job %v", persistJob)
	}
}

func (a *apiServer) reallyRunJob(
	name string,
	jobID string,
	transform *pps.Transform,
	jobInputs []*pps.JobInput,
	jobOutputs []*pps.JobOutput,
	numContainers int,
) error {
	image, err := a.buildOrPull(name, transform)
	if err != nil {
		return err
	}
	var containerIDs []string
	defer a.removeContainers(containerIDs)
	for i := 0; i < numContainers; i++ {
		containerID, err := a.containerClient.Create(
			image,
			container.CreateOptions{
				Binds:      append(getInputBinds(jobInputs), getOutputBinds(jobOutputs)...),
				HasCommand: len(transform.Cmd) > 0,
			},
		)
		if err != nil {
			return err
		}
		containerIDs = append(containerIDs, containerID)
	}
	for _, containerID := range containerIDs {
		if err := a.containerClient.Start(
			containerID,
			container.StartOptions{
				Commands: transform.Cmd,
			},
		); err != nil {
			return err
		}
	}
	errC := make(chan error, len(containerIDs))
	for _, containerID := range containerIDs {
		go a.writeContainerLogs(containerID, jobID, errC)
	}
	for _, containerID := range containerIDs {
		if err := a.containerClient.Wait(containerID, container.WaitOptions{}); err != nil {
			return err
		}
	}
	err = nil
	for _ = range containerIDs {
		if logsErr := <-errC; logsErr != nil && err == nil {
			err = logsErr
		}
	}
	return err
}

// return image name
func (a *apiServer) buildOrPull(name string, transform *pps.Transform) (string, error) {
	image := transform.Image
	if transform.Build != "" {
		image = fmt.Sprintf("ppspipelines/%s", name)
		if err := a.containerClient.Build(
			image,
			transform.Build,
			// TODO(pedge): this will not work, the path to a dockerfile is not real
			container.BuildOptions{
				Dockerfile:   transform.Dockerfile,
				OutputStream: ioutil.Discard,
			},
		); err != nil {
			return "", err
		}
	} else if err := a.containerClient.Pull(
		transform.Image,
		container.PullOptions{},
	); err != nil {
		return "", err
	}
	return image, nil
}

func (a *apiServer) removeContainers(containerIDs []string) {
	for _, containerID := range containerIDs {
		_ = a.containerClient.Kill(containerID, container.KillOptions{})
		//_ = a.containerClient.Remove(containerID, container.RemoveOptions{})
	}
}

func (a *apiServer) writeContainerLogs(containerID string, jobID string, errC chan error) {
	errC <- a.containerClient.Logs(
		containerID,
		container.LogsOptions{
			Stdout: newJobLogWriter(
				jobID,
				pps.OutputStream_OUTPUT_STREAM_STDOUT,
				a.persistAPIClient,
			),
			Stderr: newJobLogWriter(
				jobID,
				pps.OutputStream_OUTPUT_STREAM_STDERR,
				a.persistAPIClient,
			),
		},
	)
}

func getInputBinds(jobInputs []*pps.JobInput) []string {
	var binds []string
	for _, jobInput := range jobInputs {
		if jobInput.GetHostDir() != "" {
			binds = append(binds, getBinds(jobInput.GetHostDir(), filepath.Join("/var/lib/pps/host", jobInput.GetHostDir()), "ro"))
		}
	}
	return binds
}

func getOutputBinds(jobOutputs []*pps.JobOutput) []string {
	var binds []string
	for _, jobOutput := range jobOutputs {
		if jobOutput.GetHostDir() != "" {
			binds = append(binds, getBinds(jobOutput.GetHostDir(), filepath.Join("/var/lib/pps/host", jobOutput.GetHostDir()), "rw"))
		}
	}
	return binds
}

func getBinds(from string, to string, postfix string) string {
	return fmt.Sprintf("%s:%s:%s", from, to, postfix)
}
<commit_msg>call watch api client from pps api server<commit_after>package server

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/satori/go.uuid"

	"go.pachyderm.com/pachyderm/src/pkg/container"
	"go.pachyderm.com/pachyderm/src/pps"
	"go.pachyderm.com/pachyderm/src/pps/persist"
	"go.pachyderm.com/pachyderm/src/pps/watch"
	"go.pedge.io/google-protobuf"
	"go.pedge.io/protolog"
	"golang.org/x/net/context"
)

var (
	emptyInstance = &google_protobuf.Empty{}
)

type apiServer struct {
	persistAPIClient persist.APIClient
	watchAPIClient   watch.APIClient
	containerClient  container.Client
}

func newAPIServer(
	persistAPIClient persist.APIClient,
	watchAPIClient watch.APIClient,
	containerClient container.Client,
) *apiServer {
	return &apiServer{persistAPIClient, watchAPIClient, containerClient}
}

func (a *apiServer) CreateJob(ctx context.Context, request *pps.CreateJobRequest) (response *pps.Job, err error) {
	persistJob, err := a.persistAPIClient.CreateJob(ctx, jobToPersist(request.Job))
	if err != nil {
		return nil, err
	}
	return persistToJob(persistJob), nil
}

func (a *apiServer) GetJob(ctx context.Context, request *pps.GetJobRequest) (response *pps.Job, err error) {
	persistJob, err := a.persistAPIClient.GetJobByID(ctx, &google_protobuf.StringValue{Value: request.JobId})
	if err != nil {
		return nil, err
	}
	return persistToJob(persistJob), nil
}

func (a *apiServer) GetJobsByPipelineName(ctx context.Context, request *pps.GetJobsByPipelineNameRequest) (response *pps.Jobs, err error) {
	persistPipelines, err := a.persistAPIClient.GetPipelinesByName(ctx, &google_protobuf.StringValue{Value: request.PipelineName})
	if err != nil {
		return nil, err
	}
	var jobs []*pps.Job
	for _, persistPipeline := range persistPipelines.Pipeline {
		persistJobs, err := a.persistAPIClient.GetJobsByPipelineID(ctx, &google_protobuf.StringValue{Value: persistPipeline.Id})
		if err != nil {
			return nil, err
		}
		iJobs := persistToJobs(persistJobs)
		jobs = append(jobs, iJobs.Job...)
	}
	return &pps.Jobs{
		Job: jobs,
	}, nil
}

func (a *apiServer) StartJob(ctx context.Context, request *pps.StartJobRequest) (response *google_protobuf.Empty, err error) {
	persistJob, err := a.persistAPIClient.GetJobByID(ctx, &google_protobuf.StringValue{Value: request.JobId})
	if err != nil {
		return nil, err
	}
	if err := a.startPersistJob(persistJob); err != nil {
		return nil, err
	}
	return emptyInstance, nil
}

func (a *apiServer) GetJobStatus(ctx context.Context, request *pps.GetJobStatusRequest) (response *pps.JobStatus, err error) {
	persistJobStatuses, err := a.persistAPIClient.GetJobStatusesByJobID(ctx, &google_protobuf.StringValue{Value: request.JobId})
	if err != nil {
		return nil, err
	}
	if len(persistJobStatuses.JobStatus) == 0 {
		return nil, fmt.Errorf("pachyderm.pps.server: no job statuses for %s", request.JobId)
	}
	return persistToJobStatus(persistJobStatuses.JobStatus[0]), nil
}

func (a *apiServer) GetJobLogs(request *pps.GetJobLogsRequest, responseServer pps.API_GetJobLogsServer) (err error) {
	persistJobLogs, err := a.persistAPIClient.GetJobLogsByJobID(context.Background(), &google_protobuf.StringValue{Value: request.JobId})
	if err != nil {
		return err
	}
	for _, persistJobLog := range persistJobLogs.JobLog {
		if persistJobLog.OutputStream == request.OutputStream {
			if err := responseServer.Send(&google_protobuf.BytesValue{Value: persistJobLog.Value}); err != nil {
				return err
			}
		}
	}
	return nil
}

func (a *apiServer) CreatePipeline(ctx context.Context, request *pps.CreatePipelineRequest) (response *pps.Pipeline, err error) {
	persistPipeline, err := a.persistAPIClient.CreatePipeline(ctx, pipelineToPersist(request.Pipeline))
	if err != nil {
		return nil, err
	}
	if _, err := a.watchAPIClient.RegisterChangeEvent(
		ctx,
		&watch.ChangeEvent{
			Type:         watch.ChangeEvent_CHANGE_EVENT_TYPE_CREATE,
			PipelineName: persistPipeline.Name,
		},
	); err != nil {
		// TODO(pedge): need to roll back the db create
		return nil, err
	}
	return persistToPipeline(persistPipeline), nil
}

func (a *apiServer) GetPipeline(ctx context.Context, request *pps.GetPipelineRequest) (response *pps.Pipeline, err error) {
	persistPipelines, err := a.persistAPIClient.GetPipelinesByName(ctx, &google_protobuf.StringValue{Value: request.PipelineName})
	if err != nil {
		return nil, err
	}
	if len(persistPipelines.Pipeline) == 0 {
		return nil, fmt.Errorf("pachyderm.pps.server: no piplines for name %s", request.PipelineName)
	}
	return persistToPipeline(persistPipelines.Pipeline[0]), nil
}

func (a *apiServer) GetAllPipelines(ctx context.Context, request *google_protobuf.Empty) (response *pps.Pipelines, err error) {
	persistPipelines, err := a.persistAPIClient.GetAllPipelines(ctx, request)
	if err != nil {
		return nil, err
	}
	pipelineMap := make(map[string]*pps.Pipeline)
	for _, persistPipeline := range persistPipelines.Pipeline {
		// pipelines are ordered newest to oldest, so if we have already
		// seen a pipeline with the same name, it is newer
		if _, ok := pipelineMap[persistPipeline.Name]; !ok {
			pipelineMap[persistPipeline.Name] = persistToPipeline(persistPipeline)
		}
	}
	pipelines := make([]*pps.Pipeline, len(pipelineMap))
	i := 0
	for _, pipeline := range pipelineMap {
		pipelines[i] = pipeline
		i++
	}
	return &pps.Pipelines{
		Pipeline: pipelines,
	}, nil
}

func (a *apiServer) startPersistJob(persistJob *persist.Job) error {
	if _, err := a.persistAPIClient.CreateJobStatus(
		context.Background(),
		&persist.JobStatus{
			JobId: persistJob.Id,
			Type:  pps.JobStatusType_JOB_STATUS_TYPE_STARTED,
		},
	); err != nil {
		return err
	}
	// TODO(pedge): throttling? worker pool?
	go func() {
		if err := a.runJob(persistJob); err != nil {
			protolog.Errorln(err.Error())
			// TODO(pedge): how to handle the error?
			if _, err = a.persistAPIClient.CreateJobStatus(
				context.Background(),
				&persist.JobStatus{
					JobId:   persistJob.Id,
					Type:    pps.JobStatusType_JOB_STATUS_TYPE_ERROR,
					Message: err.Error(),
				},
			); err != nil {
				protolog.Errorln(err.Error())
			}
		} else {
			// TODO(pedge): how to handle the error?
			if _, err = a.persistAPIClient.CreateJobStatus(
				context.Background(),
				&persist.JobStatus{
					JobId: persistJob.Id,
					Type:  pps.JobStatusType_JOB_STATUS_TYPE_SUCCESS,
				},
			); err != nil {
				protolog.Errorln(err.Error())
			}
		}
	}()
	return nil
}

func (a *apiServer) runJob(persistJob *persist.Job) error {
	switch {
	case persistJob.GetTransform() != nil:
		return a.reallyRunJob(strings.Replace(uuid.NewV4().String(), "-", "", -1), persistJob.Id, persistJob.GetTransform(), persistJob.JobInput, persistJob.JobOutput, 1)
	case persistJob.GetPipelineId() != "":
		persistPipeline, err := a.persistAPIClient.GetPipelineByID(
			context.Background(),
			&google_protobuf.StringValue{Value: persistJob.GetPipelineId()},
		)
		if err != nil {
			return err
		}
		if persistPipeline.Transform == nil {
			return fmt.Errorf("pachyderm.pps.server: transform not set on pipeline %v", persistPipeline)
		}
		return a.reallyRunJob(persistPipeline.Name, persistJob.Id, persistPipeline.Transform, persistJob.JobInput, persistJob.JobOutput, 1)
	default:
		return fmt.Errorf("pachyderm.pps.server: neither transform or pipeline id set on job %v", persistJob)
	}
}

func (a *apiServer) reallyRunJob(
	name string,
	jobID string,
	transform *pps.Transform,
	jobInputs []*pps.JobInput,
	jobOutputs []*pps.JobOutput,
	numContainers int,
) error {
	image, err := a.buildOrPull(name, transform)
	if err != nil {
		return err
	}
	var containerIDs []string
	defer a.removeContainers(containerIDs)
	for i := 0; i < numContainers; i++ {
		containerID, err := a.containerClient.Create(
			image,
			container.CreateOptions{
				Binds:      append(getInputBinds(jobInputs), getOutputBinds(jobOutputs)...),
				HasCommand: len(transform.Cmd) > 0,
			},
		)
		if err != nil {
			return err
		}
		containerIDs = append(containerIDs, containerID)
	}
	for _, containerID := range containerIDs {
		if err := a.containerClient.Start(
			containerID,
			container.StartOptions{
				Commands: transform.Cmd,
			},
		); err != nil {
			return err
		}
	}
	errC := make(chan error, len(containerIDs))
	for _, containerID := range containerIDs {
		go a.writeContainerLogs(containerID, jobID, errC)
	}
	for _, containerID := range containerIDs {
		if err := a.containerClient.Wait(containerID, container.WaitOptions{}); err != nil {
			return err
		}
	}
	err = nil
	for _ = range containerIDs {
		if logsErr := <-errC; logsErr != nil && err == nil {
			err = logsErr
		}
	}
	return err
}

// return image name
func (a *apiServer) buildOrPull(name string, transform *pps.Transform) (string, error) {
	image := transform.Image
	if transform.Build != "" {
		image = fmt.Sprintf("ppspipelines/%s", name)
		if err := a.containerClient.Build(
			image,
			transform.Build,
			// TODO(pedge): this will not work, the path to a dockerfile is not real
			container.BuildOptions{
				Dockerfile:   transform.Dockerfile,
				OutputStream: ioutil.Discard,
			},
		); err != nil {
			return "", err
		}
	} else if err := a.containerClient.Pull(
		transform.Image,
		container.PullOptions{},
	); err != nil {
		return "", err
	}
	return image, nil
}

func (a *apiServer) removeContainers(containerIDs []string) {
	for _, containerID := range containerIDs {
		_ = a.containerClient.Kill(containerID, container.KillOptions{})
		//_ = a.containerClient.Remove(containerID, container.RemoveOptions{})
	}
}

func (a *apiServer) writeContainerLogs(containerID string, jobID string, errC chan error) {
	errC <- a.containerClient.Logs(
		containerID,
		container.LogsOptions{
			Stdout: newJobLogWriter(
				jobID,
				pps.OutputStream_OUTPUT_STREAM_STDOUT,
				a.persistAPIClient,
			),
			Stderr: newJobLogWriter(
				jobID,
				pps.OutputStream_OUTPUT_STREAM_STDERR,
				a.persistAPIClient,
			),
		},
	)
}

func getInputBinds(jobInputs []*pps.JobInput) []string {
	var binds []string
	for _, jobInput := range jobInputs {
		if jobInput.GetHostDir() != "" {
			binds = append(binds, getBinds(jobInput.GetHostDir(), filepath.Join("/var/lib/pps/host", jobInput.GetHostDir()), "ro"))
		}
	}
	return binds
}

func getOutputBinds(jobOutputs []*pps.JobOutput) []string {
	var binds []string
	for _, jobOutput := range jobOutputs {
		if jobOutput.GetHostDir() != "" {
			binds = append(binds, getBinds(jobOutput.GetHostDir(), filepath.Join("/var/lib/pps/host", jobOutput.GetHostDir()), "rw"))
		}
	}
	return binds
}

func getBinds(from string, to string, postfix string) string {
	return fmt.Sprintf("%s:%s:%s", from, to, postfix)
}
<|endoftext|>


<commit_before>package lib

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/ahmetb/go-linq"
)

// PathList is a list of paths
type PathList struct {
	SourcePath string
	TargetPath string
}

// HashList is a list of HashData struct
type HashList struct {
	List         []HashData `json:"hash_list"`
	VerifyResult bool       `json:"verify_result,omitempty"`
}

// HashData is hash value and file name struct
type HashData struct {
	RelativePath string `json:"relative_path"`
	HashValue    string `json:"hash_value"`
	VerifyResult bool   `json:"verify_result,omitempty"`
	Reason       string `json:"reason_of_failed,omitempty"`
}

// GetHashList returns the hash list of source.
// For source, a json file or a directory is supported.
func GetHashList(source string) (HashList, error) {
	source = filepath.Clean(source)
	list := HashList{}
	info, err := os.Stat(source)
	if err != nil {
		return list, err
	}
	if info.IsDir() {
		return generateHashList(source)
	}
	return readHashList(source)
}

// GenerateHashList generates hash information of dir path.
func generateHashList(dir string) (HashList, error) {
	dir = filepath.Clean(dir)
	list := HashList{}
	walk := func(path string, info os.FileInfo, err error) error {
		// root path is not used.
		if strings.Compare(dir, path) == 0 {
			return nil
		}

		rel, err := filepath.Rel(dir, path)
		if err != nil {
			log.Println("Get relative path error.", err)
			return err
		}
		data := HashData{}
		data.RelativePath = rel

		if info.IsDir() {
			// directories do not have hash value
			data.HashValue = "-"
		} else {
			bytes, err := ioutil.ReadFile(path)
			if err != nil {
				log.Println("ReadFile error.", err)
				return err
			}
			hash := sha256.Sum256(bytes)
			data.HashValue = hex.EncodeToString(hash[:])
		}

		list.List = append(list.List, data)
		return nil
	}

	err := filepath.Walk(dir, walk)
	if err != nil {
		return list, err
	}

	return list, nil
}

// ReadHashList reads source json file and return a hash list.
func readHashList(source string) (HashList, error) {
	list := HashList{}
	source = filepath.Clean(source)
	data, err := ioutil.ReadFile(source)
	if err != nil {
		return list, err
	}

	err = json.Unmarshal(data, &list)
	if err != nil {
		return list, err
	}
	return list, nil
}

// VerifyHashList verifies hash list of source and target.
// Then, returns HashList that has VerifyResult.
func VerifyHashList(source, target HashList) HashList {
	result := verifyWithSource(source, target)
	result = verifyWithTarget(result, target)
	sort.Slice(result.List, func(i int, j int) bool {
		return result.List[i].RelativePath < result.List[j].RelativePath
	})
	result.VerifyResult = linq.From(result.List).All(func(arg1 interface{}) bool {
		return arg1.(HashData).VerifyResult == true
	})
	return result
}

func verifyWithSource(source, target HashList) HashList {
	result := HashList{VerifyResult: false}

	for _, item := range source.List {
		selected := linq.From(target.List).
			SingleWith(func(c interface{}) bool {
				return c.(HashData).RelativePath == item.RelativePath
			})

		if selected == nil {
			var message string
			if item.HashValue == "-" {
				message = "This directory does not exist"
			} else {
				message = "This file does not exist"
			}

			fail := HashData{
				RelativePath: item.RelativePath,
				HashValue:    item.HashValue,
				VerifyResult: false,
				Reason:       message,
			}
			result.List = append(result.List, fail)
			log.Printf(`Required item does not exist. "%s"`, item.RelativePath)
			continue
		}

		data := selected.(HashData)
		if data.HashValue == item.HashValue {
			data.VerifyResult = true
		} else {
			data.VerifyResult = false
			data.Reason = "Hash value does not match"
			log.Printf(`Hash value does not match. "%s"`, item.RelativePath)
		}
		result.List = append(result.List, data)
	}

	return result
}

// verifyWithTarget verifies the result of verification with the hash list
// of the target directory to check wheather the unnecessary items exist.
func verifyWithTarget(result, target HashList) HashList {
	for _, item := range target.List {
		more := linq.From(result.List).
			SingleWith(func(c interface{}) bool {
				return c.(HashData).RelativePath == item.RelativePath
			})

		if more == nil {
			item.VerifyResult = false
			item.Reason = "Unnecessary item exists."
			log.Printf(`Unnecessary item exists. "%s"`, item.RelativePath)
			result.List = append(result.List, item)
		}
	}
	return result
}
<commit_msg>Added sync.WaitGroup<commit_after>package lib

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/ahmetb/go-linq"
)

// PathList is a list of paths
type PathList struct {
	SourcePath string
	TargetPath string
}

// HashList is a list of HashData struct
type HashList struct {
	List         []HashData `json:"hash_list"`
	VerifyResult bool       `json:"verify_result,omitempty"`
}

// HashData is hash value and file name struct
type HashData struct {
	RelativePath string `json:"relative_path"`
	HashValue    string `json:"hash_value"`
	VerifyResult bool   `json:"verify_result,omitempty"`
	Reason       string `json:"reason_of_failed,omitempty"`
}

// GetHashList returns the hash list of source.
// For source, a json file or a directory is supported.
func GetHashList(source string) (HashList, error) {
	source = filepath.Clean(source)
	list := HashList{}
	info, err := os.Stat(source)
	if err != nil {
		return list, err
	}
	if info.IsDir() {
		return generateHashList(source)
	}
	return readHashList(source)
}

// GenerateHashList generates hash information of dir path.
func generateHashList(dir string) (HashList, error) {
	type result struct {
		Data  HashData
		Error error
	}

	dir = filepath.Clean(dir)
	list := HashList{}
	c := make(chan result)
	var wg sync.WaitGroup
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		// root path is not used.
		if strings.Compare(dir, path) == 0 {
			return nil
		}

		wg.Add(1)
		go func() {
			defer wg.Done()
			data := HashData{}
			rel, err := filepath.Rel(dir, path)
			if err != nil {
				log.Println("Get relative path error.", err)
				c <- result{Data: data, Error: err}
				return
			}

			data.RelativePath = rel

			if info.IsDir() {
				// directories do not have hash value
				data.HashValue = "-"
			} else {
				bytes, err := ioutil.ReadFile(path)
				if err != nil {
					log.Println("ReadFile error.", err)
					c <- result{Data: data, Error: err}
					return
				}
				hash := sha256.Sum256(bytes)
				data.HashValue = hex.EncodeToString(hash[:])
				c <- result{Data: data, Error: nil}
			}
		}()
		return nil
	})
	if err != nil {
		return list, err
	}

	go func() {
		wg.Wait()
		close(c)
	}()

	for r := range c {
		if r.Error != nil {
			log.Println(err)
		}
		list.List = append(list.List, r.Data)
	}

	return list, nil
}

// ReadHashList reads source json file and return a hash list.
func readHashList(source string) (HashList, error) {
	list := HashList{}
	source = filepath.Clean(source)
	data, err := ioutil.ReadFile(source)
	if err != nil {
		return list, err
	}

	err = json.Unmarshal(data, &list)
	if err != nil {
		return list, err
	}
	return list, nil
}

// VerifyHashList verifies hash list of source and target.
// Then, returns HashList that has VerifyResult.
func VerifyHashList(source, target HashList) HashList {
	result := verifyWithSource(source, target)
	result = verifyWithTarget(result, target)
	sort.Slice(result.List, func(i int, j int) bool {
		return result.List[i].RelativePath < result.List[j].RelativePath
	})
	result.VerifyResult = linq.From(result.List).All(func(arg1 interface{}) bool {
		return arg1.(HashData).VerifyResult == true
	})
	return result
}

func verifyWithSource(source, target HashList) HashList {
	result := HashList{VerifyResult: false}

	for _, item := range source.List {
		selected := linq.From(target.List).
			SingleWith(func(c interface{}) bool {
				return c.(HashData).RelativePath == item.RelativePath
			})

		if selected == nil {
			var message string
			if item.HashValue == "-" {
				message = "This directory does not exist"
			} else {
				message = "This file does not exist"
			}

			fail := HashData{
				RelativePath: item.RelativePath,
				HashValue:    item.HashValue,
				VerifyResult: false,
				Reason:       message,
			}
			result.List = append(result.List, fail)
			log.Printf(`Required item does not exist. "%s"`, item.RelativePath)
			continue
		}

		data := selected.(HashData)
		if data.HashValue == item.HashValue {
			data.VerifyResult = true
		} else {
			data.VerifyResult = false
			data.Reason = "Hash value does not match"
			log.Printf(`Hash value does not match. "%s"`, item.RelativePath)
		}
		result.List = append(result.List, data)
	}

	return result
}

// verifyWithTarget verifies the result of verification with the hash list
// of the target directory to check wheather the unnecessary items exist.
func verifyWithTarget(result, target HashList) HashList {
	for _, item := range target.List {
		more := linq.From(result.List).
			SingleWith(func(c interface{}) bool {
				return c.(HashData).RelativePath == item.RelativePath
			})

		if more == nil {
			item.VerifyResult = false
			item.Reason = "Unnecessary item exists."
			log.Printf(`Unnecessary item exists. "%s"`, item.RelativePath)
			result.List = append(result.List, item)
		}
	}
	return result
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/mitchellh/colorstring"

	"github.com/yuuki1/gokc/log"
	"github.com/yuuki1/gokc/parser"
)

func setDebugOutputLevel() {
	for _, f := range os.Args {
		if f == "-D" || f == "--debug" || f == "-debug" {
			log.IsDebug = true
		}
	}

	debugEnv := os.Getenv("GOKC_DEBUG")
	if debugEnv != "" {
		showDebug, err := strconv.ParseBool(debugEnv)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error parsing boolean value from GRABENI_DEBUG: %s\n", err)
			os.Exit(1)
		}
		log.IsDebug = showDebug
	}
}

func init() {
	setDebugOutputLevel()
}

func main() {
	var filepath string
	flag.StringVar(&filepath, "f", "", "keepalived.conf file path")
	flag.Parse()

	if filepath == "" {
		fmt.Println("filepath required")
		os.Exit(1)
	}

	if _, err := os.Stat(filepath); err != nil {
		log.Error(err)
	}

	file, err := os.Open(filepath)
	if err != nil {
		fmt.Printf("%s\n", err)
		os.Exit(1)
	}

	p := parser.NewParser(file)
	if err := p.Parse(); err != nil {
		var msgs []string
		for _, e := range p.Errors() {
			msg := colorstring.Color(fmt.Sprintf("[white]%s:%d:%d: [red]%s[reset]", filepath, e.Line, e.Column, e.Message))
			msgs = append(msgs, msg)
		}
		log.Errorf("%s", strings.Join(msgs, "\n"))
	}

	log.Infof("gokc: the configuration file %s syntax is ok", filepath)
}
<commit_msg>Use log.Error<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/mitchellh/colorstring"

	"github.com/yuuki1/gokc/log"
	"github.com/yuuki1/gokc/parser"
)

func setDebugOutputLevel() {
	for _, f := range os.Args {
		if f == "-D" || f == "--debug" || f == "-debug" {
			log.IsDebug = true
		}
	}

	debugEnv := os.Getenv("GOKC_DEBUG")
	if debugEnv != "" {
		showDebug, err := strconv.ParseBool(debugEnv)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error parsing boolean value from GRABENI_DEBUG: %s\n", err)
			os.Exit(1)
		}
		log.IsDebug = showDebug
	}
}

func init() {
	setDebugOutputLevel()
}

func main() {
	var filepath string
	flag.StringVar(&filepath, "f", "", "keepalived.conf file path")
	flag.Parse()

	if filepath == "" {
		log.Error("filepath required")
	}

	if _, err := os.Stat(filepath); err != nil {
		log.Error(err)
	}

	file, err := os.Open(filepath)
	if err != nil {
		log.Error(err)
	}

	p := parser.NewParser(file)
	if err := p.Parse(); err != nil {
		var msgs []string
		for _, e := range p.Errors() {
			msg := colorstring.Color(fmt.Sprintf("[white]%s:%d:%d: [red]%s[reset]", filepath, e.Line, e.Column, e.Message))
			msgs = append(msgs, msg)
		}
		log.Errorf("%s", strings.Join(msgs, "\n"))
	}

	log.Infof("gokc: the configuration file %s syntax is ok", filepath)
}
<|endoftext|>