<commit_before>
// Copyright (c) James Percent and Unlock contributors.
// All rights reserved.
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//    1. Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//
//    2. Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
//    3. Neither the name of Unlock nor the names of its contributors may be used
//       to endorse or promote products derived from this software without
//       specific prior written permission.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
    "npl/bu/edu/unzip"
    "net/http"
    "fmt"
    "io/ioutil"
    "flag"
    "os"
    "os/exec"
    "log"
    "path/filepath"
    "io"
)

func runCommand(command string, errorMsg string, failOnError bool) bool {
    result := true
    cmd := exec.Command("cmd", "/C", command) // note the mixing of ` and "; my editor `\` are not friends
    if err := cmd.Run(); err != nil {
        if failOnError == true {
            log.Fatalln(`FATAL: `+errorMsg+`; command = `+command, err)
        } else {
            log.Println(`Non-fatal `+errorMsg+`; command = `+command, err)
        }
        result = false
    }
    return result
}

func install(command string, packageName string, failOnError bool) {
    log.Print(`Installing `+packageName+`: `)
    log.Println("command = "+ command)
    result := runCommand(command, `Failed to install `+packageName, failOnError)
    if result {
        fmt.Println(`Success`)
    } else {
        fmt.Println(`Failure`)
    }
}

func chdirFailOnError(directory string, errorMsg string) {
    if err := os.Chdir(directory); err != nil {
        log.Fatalln(`install-win.chdirFailOnError: ERROR: Change directory to `+directory+` failed: `+errorMsg, err)
    }
}

func unzipExpand(fileName string) {
    u := &unzip.Unzip{fileName, ``, nil}
    if err := u.Expand(); err != nil {
        log.Fatalln(`Failed to expand `+fileName, err)
    }
}

func downloadAndWriteFile(url string, fileName string) string {
    fullPath := filepath.Join(getDownloadDirectory(), fileName)

	isFileExist,_ := checkFileExists(fullPath)

    if isFileExist == false {
        log.Println("Downloading file "+fileName+" from URL = "+url)
        resp, err := http.Get(url)
        if err != nil {
            log.Fatalln(err)
        }

        defer resp.Body.Close()
        body, err1 := ioutil.ReadAll(resp.Body)
        if err1 != nil {
            log.Fatalln(err)
        }

        if err = ioutil.WriteFile(fullPath, body, 0744); err != nil {
            log.Fatalln(err)
        }
    }

    return fullPath
}

func checkFileExists(path string) (bool, error) {
    _, err := os.Stat(path)
    if err == nil { return true, nil }
    if os.IsNotExist(err) { return false, nil }
    return false, err
}

func getDownloadDirectory() string {
    path := getWorkingDirectoryAbsolutePath()

    if *repoPath != `` {
		path = filepath.Join(*repoPath, `package`)
	}

    return path
}

func getWorkingDirectoryAbsolutePath() string {
    cwd, err := filepath.Abs(``)
	log.Println(`Current working directory = `, cwd)
    if err != nil {
        log.Fatalln(err)
    }
    return cwd
}

func installZippedPythonPackage(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    log.Println(`Downloading `+packageName+`... `)
    downloadedFile := downloadAndWriteFile(baseUrl+fileName, fileName)

	unzipExpand(downloadedFile)
    chdirFailOnError(packageDirectory, `Failed to install `+packageName)
    log.Println("CWD = "+getWorkingDirectoryAbsolutePath())
    command := pythonPath+` setup.py install`
    install(command, packageName, true)
    os.Chdir(`..`)
}

func installPython(baseUrl string, pythonPathEnvVar string, pythonInstallerName string, pythonBasePath string, pythonPackageName string) {
    //cwd := getWorkingDirectoryAbsolutePath()
    log.Println(`Downloading `+pythonPackageName+`...`)
    downloadedFile := downloadAndWriteFile(baseUrl+pythonInstallerName, pythonInstallerName)
    log.Println(`Installing `+pythonPackageName)
//    output, err := exec.Command("cmd", "/C", "msiexec /i ", cwd+"\\"+pythonInstallerName,`TARGETDIR=`+pythonBasePath,`/qb`, `ALLUSERS=0`).CombinedOutput()
    //output, err := exec.Command("cmd", "/C", cwd+"\\"+pythonInstallerName).CombinedOutput()
    output, err := exec.Command("cmd", "/C", downloadedFile).CombinedOutput()
    if len(output) > 0 {
        log.Printf("%s\n", output)
    }

    if err != nil {
        log.Fatalln(`FATAL: failed to install python `, err)
    }

    if err := os.Setenv(`PYTHONPATH`, pythonPathEnvVar); err != nil {
        log.Println(`Could not properly set the PYTHONPATH env variable; on some systems this can cause problems during virtual env creation`)
    }
    log.Println(`PYTHON PATH = `+os.Getenv(`PYTHONPATH`))
}

func installEasyInstall(baseUrl string, pythonPath string) {
    downloadAndWriteFile(baseUrl+`distribute_setup-py`, `distribute_setup.py`)
    install(pythonPath+` distribute_setup.py`, `easy_install`, true)
}

func installPip(easyInstallPath string) {
    install(easyInstallPath+` pip`, `pip`, true)
}

func installVirtualenv(pipPath string) {
    install(pipPath+` install virtualenv`, `virtualenv`, true)
}

func createVirtualenv(unlockDirectory string, virtualenvPath string, envName string) {
    errorMsg := `Failed to create virtual environment`
    var cwd = getWorkingDirectoryAbsolutePath()
    chdirFailOnError(unlockDirectory, errorMsg)
    command := virtualenvPath+` --system-site-packages `+envName //python27`
    runCommand(command, errorMsg, true)
    os.Chdir(cwd)
}

func installPyglet12alpha(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory)
}

func installNumPy(baseUrl string, numpyPath string) {
    /*downloadAndWriteFile(baseUrl+numpyPath, numpyPath)//numpy-MKL-1.7.1.win32-py2.7.exe)
    var cwd = getWorkingDirectoryAbsolutePath()
    install(cwd+"\\"+numpyPath, `numpy`, true)*/

    downloadAndInstallBinPackage(baseUrl, numpyPath, `numpy`)
}

func downloadAndInstallBinPackage(baseUrl string, fileName string, packageName string) {
    downloadedFile := downloadAndWriteFile(baseUrl + fileName, fileName)
    install(downloadedFile, packageName, true)
}

func installPySerial26(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory);
}

func installAvbin(baseUrl string, avbin string) {
    /*downloadAndWriteFile(baseUrl+avbin, avbin)
    var cwd = getWorkingDirectoryAbsolutePath()
    install(cwd+"\\"+avbin, `avbin`, true)*/

    downloadAndInstallBinPackage(baseUrl, avbin, `avbin`)

    // XXX - last minute hack
    data, err1 := ioutil.ReadFile(`C:\Windows\System32\avbin.dll`)
    if err1 != nil {
        log.Fatalln(err1)
    }

    if err := ioutil.WriteFile(`C:\Windows\SysWOW64\avbin.dll`, data, 0744); err != nil {
        log.Println(err)
    }

}

func installScons(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory);
}

func installUnlock(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory)
}

var confFile = flag.String("conf", "", "Qualified file name of Unlock installation configuration file")
var devOption = flag.Bool("dev", false, "Setup development env")
var repoPath = flag.String("repo", "", "Path to project's git repo")

func createConf() UnlockInstallConf {
    if *confFile == `` {
        return UnlockInstallConf {`C:\Unlock`, `http://jpercent.org/unlock/`, `C:\Python33;C:\Python33\Lib;C:\Python33\DLLs`, `python-3.3.2.msi`,
            `Python-3.3.2`, `C:\Python33`, `C:\Python33\python.exe`, `numpy-MKL-1.7.1.win32-py3.3.exe`,
            `C:\Python33\Scripts\easy_install.exe`, `C:\Python33\Scripts\pip.exe`,
            `C:\Python33\Scripts\virtualenv.exe`, `python33`,
            `C:\Python33\Lib\site-packages\numpy`, `C:\Unlock\python33\Lib\site-packages`,
            `C:\Unlock\python33\Scripts\python.exe`, `C:\Unlock\python33\Scripts\pip.exe`,
            `pyglet-1.2alpha-p3.zip`, `pyglet-1.2alpha`, `pyglet-1.2alpha1`, `AVbin10-win32.exe`,
            `pyserial-2.6.zip`, `pyserial-2.6`, `pyserial-2.6`, `unlock-0.3.7-win32.zip`, `unlock`, `unlock-0.3.7`,
            `scons-2.3.0.zip`, `scons`, `scons-2.3.0`,
            `unlock.exe`, `vcredist_2010_x86.exe`, `pyaudio-0.2.7.py33.exe`, `pywin32-218.win32-py3.3.exe`}
    } else {
        return ParseConf(*confFile)
    }
}

func numpyHack(pythonPath string, from string, to string) {
    var copydir = "import shutil\n"
    copydir += "import os\n"
    copydir += `shutil.copytree('`+from+`','`+to+`')`+"\n"
    fmt.Println(copydir)
    command := pythonPath+` -c '`+copydir+`'`
    runCommand(command, "numpyHack Failed", false)
}

func installUnlockRunner(baseUrl string, unlockDirectory string, unlockexe string) {
    var cwd = getWorkingDirectoryAbsolutePath()
    chdirFailOnError(unlockDirectory, ` ERROR: Failed to install unlock.exe: couldn't change dir `)
    downloadAndWriteFile(baseUrl+unlockexe, unlockexe)
    os.Chdir(cwd)
}

func main() {

    flag.Parse()
    logf, err := os.OpenFile(`unlock-install.log`, os.O_WRONLY|os.O_APPEND|os.O_CREATE,0640)
    if err != nil {
        log.Fatalln(err)
    }

    log.SetOutput(io.MultiWriter(logf, os.Stdout))
    log.Printf("conf file = "+*confFile)

    var conf = createConf()

    installPython(conf.BaseUrl, conf.PythonPathEnvVar, conf.PythonInstallerName, conf.PythonBasePath, conf.PythonPackageName)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.VCRedistPackageName, `vcredist`)
	installNumPy(conf.BaseUrl, conf.NumpyPackageName)
    //installEasyInstall(conf.BaseUrl, conf.PythonPath)
    //installPip(conf.EasyInstallPath)
    //installVirtualenv(conf.PipPath)
    installAvbin(conf.BaseUrl, conf.Avbin)

    if err := os.MkdirAll(conf.UnlockDirectory, 0755); err != nil {
        log.Fatalln(`Failed to create `+conf.UnlockDirectory, err)
    }
    //createVirtualenv(conf.UnlockDirectory, conf.VirtualenvPath, conf.EnvName)
    // XXX - this is a hack for numpy.  on my machine the virtual env does the right thing, but on other machines it does not.
    //       I found this solution on stackoverflow; its not the best as it does register numpy with pip, but it does work for
    //       now.
    //numpyHack(conf.EnvPythonPath, conf.NumpyHack, conf.NumpyHack1)

    installPyglet12alpha(conf.PythonPath, conf.BaseUrl, conf.PygletZipName, conf.PygletPackageName, conf.PygletDirectory)
    installPySerial26(conf.PythonPath, conf.BaseUrl, conf.PyserialZipName, conf.PyserialPackageName, conf.PyserialDirectory)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.PyAudioPackageName, `pyaudio`)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.PyWinPackageName, `pywin`)

	// Skip install unlock software for development option
	if *devOption == false {
        installUnlock(conf.PythonPath, conf.BaseUrl, conf.UnlockZipName, conf.UnlockPackageName, conf.UnlockPackageDirectory)
        //installScons(conf.PythonPath, conf.BaseUrl, conf.SconsZipName, conf.SconsPackageName, conf.SconsPackageDirectory)
        //installUnlockRunner(conf.BaseUrl, conf.UnlockDirectory, conf.Unlockexe)
	}
}
<commit_msg>Working on file checksum.<commit_after>
// Copyright (c) James Percent and Unlock contributors.
// All rights reserved.
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//    1. Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//
//    2. Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
//    3. Neither the name of Unlock nor the names of its contributors may be used
//       to endorse or promote products derived from this software without
//       specific prior written permission.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
    "npl/bu/edu/unzip"
    "net/http"
    "fmt"
    "io/ioutil"
    "flag"
    "os"
    "os/exec"
    "log"
    "path/filepath"
    "io"
    "bytes"
    "crypto/sha1"
)

func runCommand(command string, errorMsg string, failOnError bool) bool {
    result := true
    cmd := exec.Command("cmd", "/C", command) // note the mixing of ` and "; my editor `\` are not friends
    if err := cmd.Run(); err != nil {
        if failOnError == true {
            log.Fatalln(`FATAL: `+errorMsg+`; command = `+command, err)
        } else {
            log.Println(`Non-fatal `+errorMsg+`; command = `+command, err)
        }
        result = false
    }
    return result
}

func install(command string, packageName string, failOnError bool) {
    log.Print(`Installing `+packageName+`: `)
    log.Println("command = "+ command)
    result := runCommand(command, `Failed to install `+packageName, failOnError)
    if result {
        fmt.Println(`Success`)
    } else {
        fmt.Println(`Failure`)
    }
}

func chdirFailOnError(directory string, errorMsg string) {
    if err := os.Chdir(directory); err != nil {
        log.Fatalln(`install-win.chdirFailOnError: ERROR: Change directory to `+directory+` failed: `+errorMsg, err)
    }
}

func unzipExpand(fileName string) {
    u := &unzip.Unzip{fileName, ``, nil}
    if err := u.Expand(); err != nil {
        log.Fatalln(`Failed to expand `+fileName, err)
    }
}

func downloadAndWriteFile(fileUrl string, fileName string) string {
    return downloadAndWriteFileWithChecksum(fileUrl, fileName, true)
}

func downloadAndWriteFileWithChecksum(fileUrl string, fileName string, shouldCheckSum bool) string {
    fullPath := filepath.Join(getDownloadDirectory(), fileName)

	//isFileExist,_ := checkFileExists(fullPath)
    isFileGood := false
    if (shouldCheckSum) {
        isFileGood = !checkSum(fullPath, fileUrl+".sha1")
    }

    if /*!isFileExist ||*/ !isFileGood {
        log.Println("Downloading file "+fileName+" from URL = "+fileUrl)
        resp, err := http.Get(fileUrl)
        if err != nil {
            log.Fatalln(err)
        }

        defer resp.Body.Close()
        body, err1 := ioutil.ReadAll(resp.Body)
        if err1 != nil {
            log.Fatalln(err)
        }

        if err = ioutil.WriteFile(fullPath, body, 0744); err != nil {
            log.Fatalln(err)
        }
    }

    return fullPath
}

func checkSum(filePath string, checksumFileUrl string) bool {
    computedHash := computeChecksum(filePath)
    downloadedHash := downloadChecksum(checksumFileUrl)

    return bytes.Compare(computedHash, downloadedHash) == 0
}

func computeChecksum(filePath string) []byte {
    content,err := ioutil.ReadFile(filePath)
    if err != nil { panic(err) }

    s1 := sha1.New()
    s1.Write([]byte(content))
    hashed := s1.Sum(nil)

    log.Println(`Computed checksum: `, hashed)

    return hashed
}

func downloadChecksum(checksumFileUrl string) []byte {
    checksumFile := downloadAndWriteFileWithChecksum(checksumFileUrl, `checksum`, false)

    content,err := ioutil.ReadFile(checksumFile)
    if err != nil { panic(err) }

    log.Println(`Downloaded checksum: `, content)

    return content
}

func checkFileExists(path string) (bool, error) {
    _, err := os.Stat(path)
    if err == nil { return true, nil }
    if os.IsNotExist(err) { return false, nil }
    return false, err
}

func getDownloadDirectory() string {
    path := getWorkingDirectoryAbsolutePath()

    if *repoPath != `` {
		path = filepath.Join(*repoPath, `package`)
	}

    return path
}

func getWorkingDirectoryAbsolutePath() string {
    cwd, err := filepath.Abs(``)
	log.Println(`Current working directory = `, cwd)
    if err != nil {
        log.Fatalln(err)
    }
    return cwd
}

func installZippedPythonPackage(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    log.Println(`Downloading `+packageName+`... `)
    downloadedFile := downloadAndWriteFile(baseUrl+fileName, fileName)

	unzipExpand(downloadedFile)
    chdirFailOnError(packageDirectory, `Failed to install `+packageName)
    log.Println("CWD = "+getWorkingDirectoryAbsolutePath())
    command := pythonPath+` setup.py install`
    install(command, packageName, true)
    os.Chdir(`..`)
}

func installPython(baseUrl string, pythonPathEnvVar string, pythonInstallerName string, pythonBasePath string, pythonPackageName string) {
    //cwd := getWorkingDirectoryAbsolutePath()
    log.Println(`Downloading `+pythonPackageName+`...`)
    downloadedFile := downloadAndWriteFile(baseUrl+pythonInstallerName, pythonInstallerName)
    log.Println(`Installing `+pythonPackageName)
//    output, err := exec.Command("cmd", "/C", "msiexec /i ", cwd+"\\"+pythonInstallerName,`TARGETDIR=`+pythonBasePath,`/qb`, `ALLUSERS=0`).CombinedOutput()
    //output, err := exec.Command("cmd", "/C", cwd+"\\"+pythonInstallerName).CombinedOutput()
    output, err := exec.Command("cmd", "/C", downloadedFile).CombinedOutput()
    if len(output) > 0 {
        log.Printf("%s\n", output)
    }

    if err != nil {
        log.Fatalln(`FATAL: failed to install python `, err)
    }

    if err := os.Setenv(`PYTHONPATH`, pythonPathEnvVar); err != nil {
        log.Println(`Could not properly set the PYTHONPATH env variable; on some systems this can cause problems during virtual env creation`)
    }
    log.Println(`PYTHON PATH = `+os.Getenv(`PYTHONPATH`))
}

func installEasyInstall(baseUrl string, pythonPath string) {
    downloadAndWriteFile(baseUrl+`distribute_setup-py`, `distribute_setup.py`)
    install(pythonPath+` distribute_setup.py`, `easy_install`, true)
}

func installPip(easyInstallPath string) {
    install(easyInstallPath+` pip`, `pip`, true)
}

func installVirtualenv(pipPath string) {
    install(pipPath+` install virtualenv`, `virtualenv`, true)
}

func createVirtualenv(unlockDirectory string, virtualenvPath string, envName string) {
    errorMsg := `Failed to create virtual environment`
    var cwd = getWorkingDirectoryAbsolutePath()
    chdirFailOnError(unlockDirectory, errorMsg)
    command := virtualenvPath+` --system-site-packages `+envName //python27`
    runCommand(command, errorMsg, true)
    os.Chdir(cwd)
}

func installPyglet12alpha(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory)
}

func installNumPy(baseUrl string, numpyPath string) {
    /*downloadAndWriteFile(baseUrl+numpyPath, numpyPath)//numpy-MKL-1.7.1.win32-py2.7.exe)
    var cwd = getWorkingDirectoryAbsolutePath()
    install(cwd+"\\"+numpyPath, `numpy`, true)*/

    downloadAndInstallBinPackage(baseUrl, numpyPath, `numpy`)
}

func downloadAndInstallBinPackage(baseUrl string, fileName string, packageName string) {
    downloadedFile := downloadAndWriteFile(baseUrl + fileName, fileName)
    install(downloadedFile, packageName, true)
}

func installPySerial26(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory);
}

func installAvbin(baseUrl string, avbin string) {
    /*downloadAndWriteFile(baseUrl+avbin, avbin)
    var cwd = getWorkingDirectoryAbsolutePath()
    install(cwd+"\\"+avbin, `avbin`, true)*/

    downloadAndInstallBinPackage(baseUrl, avbin, `avbin`)

    // XXX - last minute hack
    data, err1 := ioutil.ReadFile(`C:\Windows\System32\avbin.dll`)
    if err1 != nil {
        log.Fatalln(err1)
    }

    if err := ioutil.WriteFile(`C:\Windows\SysWOW64\avbin.dll`, data, 0744); err != nil {
        log.Println(err)
    }

}

func installScons(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory);
}

func installUnlock(pythonPath string, baseUrl string, fileName string, packageName string, packageDirectory string) {
    installZippedPythonPackage(pythonPath, baseUrl, fileName, packageName, packageDirectory)
}

var confFile = flag.String("conf", "", "Qualified file name of Unlock installation configuration file")
var devOption = flag.Bool("dev", false, "Setup development env")
var repoPath = flag.String("repo", "", "Path to project's git repo")

func createConf() UnlockInstallConf {
    if *confFile == `` {
        return UnlockInstallConf {`C:\Unlock`, `http://jpercent.org/unlock/`, `C:\Python33;C:\Python33\Lib;C:\Python33\DLLs`, `python-3.3.2.msi`,
            `Python-3.3.2`, `C:\Python33`, `C:\Python33\python.exe`, `numpy-MKL-1.7.1.win32-py3.3.exe`,
            `C:\Python33\Scripts\easy_install.exe`, `C:\Python33\Scripts\pip.exe`,
            `C:\Python33\Scripts\virtualenv.exe`, `python33`,
            `C:\Python33\Lib\site-packages\numpy`, `C:\Unlock\python33\Lib\site-packages`,
            `C:\Unlock\python33\Scripts\python.exe`, `C:\Unlock\python33\Scripts\pip.exe`,
            `pyglet-1.2alpha-p3.zip`, `pyglet-1.2alpha`, `pyglet-1.2alpha1`, `AVbin10-win32.exe`,
            `pyserial-2.6.zip`, `pyserial-2.6`, `pyserial-2.6`, `unlock-0.3.7-win32.zip`, `unlock`, `unlock-0.3.7`,
            `scons-2.3.0.zip`, `scons`, `scons-2.3.0`,
            `unlock.exe`, `vcredist_2010_x86.exe`, `pyaudio-0.2.7.py33.exe`, `pywin32-218.win32-py3.3.exe`}
    } else {
        return ParseConf(*confFile)
    }
}

func numpyHack(pythonPath string, from string, to string) {
    var copydir = "import shutil\n"
    copydir += "import os\n"
    copydir += `shutil.copytree('`+from+`','`+to+`')`+"\n"
    fmt.Println(copydir)
    command := pythonPath+` -c '`+copydir+`'`
    runCommand(command, "numpyHack Failed", false)
}

func installUnlockRunner(baseUrl string, unlockDirectory string, unlockexe string) {
    var cwd = getWorkingDirectoryAbsolutePath()
    chdirFailOnError(unlockDirectory, ` ERROR: Failed to install unlock.exe: couldn't change dir `)
    downloadAndWriteFile(baseUrl+unlockexe, unlockexe)
    os.Chdir(cwd)
}

func main() {

    flag.Parse()
    logf, err := os.OpenFile(`unlock-install.log`, os.O_WRONLY|os.O_APPEND|os.O_CREATE,0640)
    if err != nil {
        log.Fatalln(err)
    }

    log.SetOutput(io.MultiWriter(logf, os.Stdout))
    log.Printf("conf file = "+*confFile)

    var conf = createConf()

    installPython(conf.BaseUrl, conf.PythonPathEnvVar, conf.PythonInstallerName, conf.PythonBasePath, conf.PythonPackageName)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.VCRedistPackageName, `vcredist`)
	installNumPy(conf.BaseUrl, conf.NumpyPackageName)
    //installEasyInstall(conf.BaseUrl, conf.PythonPath)
    //installPip(conf.EasyInstallPath)
    //installVirtualenv(conf.PipPath)
    installAvbin(conf.BaseUrl, conf.Avbin)

    if err := os.MkdirAll(conf.UnlockDirectory, 0755); err != nil {
        log.Fatalln(`Failed to create `+conf.UnlockDirectory, err)
    }
    //createVirtualenv(conf.UnlockDirectory, conf.VirtualenvPath, conf.EnvName)
    // XXX - this is a hack for numpy.  on my machine the virtual env does the right thing, but on other machines it does not.
    //       I found this solution on stackoverflow; its not the best as it does register numpy with pip, but it does work for
    //       now.
    //numpyHack(conf.EnvPythonPath, conf.NumpyHack, conf.NumpyHack1)

    installPyglet12alpha(conf.PythonPath, conf.BaseUrl, conf.PygletZipName, conf.PygletPackageName, conf.PygletDirectory)
    installPySerial26(conf.PythonPath, conf.BaseUrl, conf.PyserialZipName, conf.PyserialPackageName, conf.PyserialDirectory)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.PyAudioPackageName, `pyaudio`)
    downloadAndInstallBinPackage(conf.BaseUrl, conf.PyWinPackageName, `pywin`)

	// Skip install unlock software for development option
	if *devOption == false {
        installUnlock(conf.PythonPath, conf.BaseUrl, conf.UnlockZipName, conf.UnlockPackageName, conf.UnlockPackageDirectory)
        //installScons(conf.PythonPath, conf.BaseUrl, conf.SconsZipName, conf.SconsPackageName, conf.SconsPackageDirectory)
        //installUnlockRunner(conf.BaseUrl, conf.UnlockDirectory, conf.Unlockexe)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"time"
	"strings"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceContainerReplicaController() *schema.Resource {
	return &schema.Resource{
		Create: resourceContainerReplicaControllerCreate,
		Read:   resourceContainerReplicaControllerRead,
		Delete: resourceContainerReplicaControllerDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"docker_image": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"container_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"zone": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"external_port": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"resource_version": &schema.Schema{
				Type:     schema.TypeList,
				ForceNew: true,
				Optional: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"optional_args": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem:	  schema.TypeString,
			},

			"env_args": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem:	  schema.TypeString,
			},

			"external_ip": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

		},
	}
}

func resourceContainerReplicaControllerCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	err := config.initKubectl(d.Get("container_name").(string), d.Get("zone").(string))
	if err != nil {
		return err
	}

	optional_args := cleanAdditionalArgs(d.Get("optional_args").(map[string]interface{}))
	env_args := cleanAdditionalArgs(d.Get("env_args").(map[string]interface{}))
	uid, err := CreateKubeRC(d.Get("name").(string), d.Get("docker_image").(string), d.Get("external_port").(string), optional_args, env_args)
	if err != nil {
		return err
	}

	err = resourceContainerReplicaControllerRead(d, meta)
	if err != nil {
		return err
	}

	d.SetId(uid)

	return nil
}

// if the error string has a 'code=404' in it, the owning cluster is gone.
//  remove the rc from the tfstate file
func checkMissingCluster(d *schema.ResourceData, err error) error {
	if strings.Contains(err.Error(), "code=404") {
		//  the owning cluster doesn't exist, the container can't
		d.SetId("")
		return nil
	}
	return err
}

func resourceContainerReplicaControllerRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	err := config.initKubectl(d.Get("container_name").(string), d.Get("zone").(string))
	if err != nil {
		return checkMissingCluster(d, err)
	}

    //  the endpoint kubectl hits is flaky.  put a loop on it.
	pod_count, external_ip, err := ReadKubeRC(d.Get("name").(string), d.Get("external_port").(string))
	if err != nil {
		is_error = true
		for i := 0; i < (10 * 6) && is_error; i++ {
			time.Sleep(10 * time.Second)
			pod_count, external_ip, err = ReadKubeRC(d.Get("name").(string), d.Get("external_port").(string))
			if err == nil {
				is_error = false
			}
		}
		if err != nil {
			return err
		}
	}

	if pod_count == 0 {
		//  something has gone awry, there should always be at least one pod
		log.Printf("There are no pods associated with this Replica Controller.  This is unexpected and probably wrong.  Please investigate")
	}

	if external_ip != "" {
		d.Set("external_ip", external_ip)
	}

	return nil
}

func resourceContainerReplicaControllerDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	err := config.initKubectl(d.Get("container_name").(string), d.Get("zone").(string))
	if err != nil {
		return checkMissingCluster(d, err)
	}

	err = DeleteKubeRC(d.Get("name").(string),d.Get("external_port").(string))
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}
<commit_msg>syntax error<commit_after>package main

import (
	"log"
	"time"
	"strings"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceContainerReplicaController() *schema.Resource {
	return &schema.Resource{
		Create: resourceContainerReplicaControllerCreate,
		Read:   resourceContainerReplicaControllerRead,
		Delete: resourceContainerReplicaControllerDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"docker_image": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"container_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"zone": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"external_port": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"resource_version": &schema.Schema{
				Type:     schema.TypeList,
				ForceNew: true,
				Optional: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"optional_args": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem:	  schema.TypeString,
			},

			"env_args": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem:	  schema.TypeString,
			},

			"external_ip": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

		},
	}
}

func resourceContainerReplicaControllerCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	err := config.initKubectl(d.Get("container_name").(string), d.Get("zone").(string))
	if err != nil {
		return err
	}

	optional_args := cleanAdditionalArgs(d.Get("optional_args").(map[string]interface{}))
	env_args := cleanAdditionalArgs(d.Get("env_args").(map[string]interface{}))
	uid, err := CreateKubeRC(d.Get("name").(string), d.Get("docker_image").(string), d.Get("external_port").(string), optional_args, env_args)
	if err != nil {
		return err
	}

	err = resourceContainerReplicaControllerRead(d, meta)
	if err != nil {
		return err
	}

	d.SetId(uid)

	return nil
}

// if the error string has a 'code=404' in it, the owning cluster is gone.
//  remove the rc from the tfstate file
func checkMissingCluster(d *schema.ResourceData, err error) error {
	if strings.Contains(err.Error(), "code=404") {
		//  the owning cluster doesn't exist, the container can't
		d.SetId("")
		return nil
	}
	return err
}

func resourceContainerReplicaControllerRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	err := config.initKubectl(d.Get("container_name").(string), d.Get("zone").(string))
	if err != nil {
		return checkMissingCluster(d, err)
	}

    //  the endpoint kubectl hits is flaky.  put a loop on it.
	pod_count, external_ip, err := ReadKubeRC(d.Get("name").(string), d.Get("external_port").(string))
	if err != nil {
		is_error := true
		for i := 0; i < (10 * 6) && is_error; i++ {
			time.Sleep(10 * time.Second)
			pod_count, external_ip, err = ReadKubeRC(d.Get("name").(string), d.Get("external_port").(string))
			if err == nil {
				is_error = false
			}
		}
		if err != nil {
			return err
		}
	}

	if pod_count == 0 {
		//  something has gone awry, there should always be at least one pod
		log.Printf("There are no pods associated with this Replica Controller.  This is unexpected and probably wrong.  Please investigate")
	}

	if external_ip != "" {
		d.Set("external_ip", external_ip)
	}

	return nil
}

func resourceContainerReplicaControllerDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	err := config.initKubectl(d.Get("container_name").(string), d.Get("zone").(string))
	if err != nil {
		return checkMissingCluster(d, err)
	}

	err = DeleteKubeRC(d.Get("name").(string),d.Get("external_port").(string))
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}
<|endoftext|>


<commit_before>package exchange

import (
	"os"
	"testing"

	"github.com/qor/qor"
	"github.com/qor/qor/resource"

	"github.com/jinzhu/gorm"
	_ "github.com/mattn/go-sqlite3"
)

type User struct {
	Id   int64
	Name string
	Age  int
}

var testdb = func() *gorm.DB {
	db, err := gorm.Open("sqlite3", "/tmp/qor_exchange_test.db")
	if err != nil {
		panic(err)
	}

	// db.LogMode(true)
	db.DropTable(&User{})
	db.AutoMigrate(&User{})

	return &db
}()

func TestImport(t *testing.T) {
	ex := &Exchange{DB: testdb}
	userRes := ex.NewResource(User{})
	userRes.RegisterMeta(&Meta{Meta: resource.Meta{Name: "Name", Label: "Name"}})
	userRes.RegisterMeta(&Meta{Meta: resource.Meta{Name: "Age", Label: "Age"}})
	r, err := os.Open("simple.xlsx")
	if err != nil {
		t.Fatal(err)
	}
	err = userRes.Import(r, &qor.Context{DB: ex.DB})
	if err != nil {
		t.Fatal(err)
	}
	var users []User
	testdb.Find(&users)
	if len(users) != 3 {
		t.Fatal("should get 3 records")
	}
}
<commit_msg>exchange: remove comments<commit_after>package exchange

import (
	"os"
	"testing"

	"github.com/qor/qor"
	"github.com/qor/qor/resource"

	"github.com/jinzhu/gorm"
	_ "github.com/mattn/go-sqlite3"
)

type User struct {
	Id   int64
	Name string
	Age  int
}

var testdb = func() *gorm.DB {
	db, err := gorm.Open("sqlite3", "/tmp/qor_exchange_test.db")
	if err != nil {
		panic(err)
	}

	db.DropTable(&User{})
	db.AutoMigrate(&User{})

	return &db
}()

func TestImport(t *testing.T) {
	ex := &Exchange{DB: testdb}
	userRes := ex.NewResource(User{})
	userRes.RegisterMeta(&Meta{Meta: resource.Meta{Name: "Name", Label: "Name"}})
	userRes.RegisterMeta(&Meta{Meta: resource.Meta{Name: "Age", Label: "Age"}})
	r, err := os.Open("simple.xlsx")
	if err != nil {
		t.Fatal(err)
	}
	err = userRes.Import(r, &qor.Context{DB: ex.DB})
	if err != nil {
		t.Fatal(err)
	}
	var users []User
	testdb.Find(&users)
	if len(users) != 3 {
		t.Fatal("should get 3 records")
	}
}
<|endoftext|>


<commit_before>package db

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	bytes      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

var (
	FileNotFound = errors.New("Couldn't find .trago.db")
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	// TODO: use a hash as well
}

type FileTag uint8

const (
	File = FileTag(iota)
	Conflict
	Directory
	Deleted
)

func Parse(data string) (*TraDb, error) {
	tradb := &TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version
			if len(fields) != 5 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return tradb, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return tradb, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return tradb, err
			}

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return tradb, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

func ParseFile() (*TraDb, error) {
	tradb := &TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println(FileNotFound.Error())
		tradb = New()

		return tradb, FileNotFound
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return tradb, err
	}

	return Parse(string(bs))
}

func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = bytes[rand.Intn(len(bytes))]
	}
	versionVector[string(replicaId)] = 1 // TODO: check for duplicates

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
		}
		filemap[filename] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	db.VersionVec[db.ReplicaId] += 1
	ourVersion := db.VersionVec[db.ReplicaId]

	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue
		}

		dbRecord := db.Files[filename]
		if dbRecord.MTime == 0 {
			log.Printf("found a new file: %s\n", filename)
			db.Files[filename] = FileState{
				Size:    int(file.Size()),
				MTime:   file.ModTime().UTC().UnixNano(),
				Version: ourVersion,
				Replica: db.ReplicaId,
			}
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = ourVersion
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
	}

	return nil
}

func (local *TraDb) Compare(remote *TraDb) map[string]FileTag {
	tags := make(map[string]FileTag)
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			if state.Version <= remote.VersionVec[state.Replica] {
				log.Printf("deleting: %s\n", file)
				tags[file] = Deleted
			}
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				log.Printf("keeping: %s\n", file)
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				tags[file] = File
			} else {
				log.Printf("conflict: %s\n", file)
				tags[file] = Conflict
			}
		} else {
			log.Printf("unchanged: %s\n", file)
		}
	}

	for file, state := range remoteFiles {
		if local.Files[file].Version > 0 {
			continue
		} else if state.Version > local.VersionVec[state.Replica] {
			log.Printf("downloading: %s\n", file)
			tags[file] = File
		}
	}

	combineVectors(local.VersionVec, remote.VersionVec)
	log.Println(local.VersionVec)
	return tags
}

func combineVectors(v1 map[string]int, v2 map[string]int) {
	for replica, version := range v1 {
		if v2[replica] > version {
			v1[replica] = v2[replica]
		}
	}

	for replica, version := range v2 {
		if v1[replica] < version {
			v1[replica] = version
		}
	}
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		return true
	}
	return false
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>some documentation<commit_after>package db

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	bytes      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

var (
	FileNotFound = errors.New("Couldn't find .trago.db")
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	// TODO: use a hash as well
}

type FileTag uint8

const (
	File = FileTag(iota)
	Conflict
	Directory
	Deleted
)

// Parses a TraDb structure.
// Fails if the given string is not in the correct format.
func Parse(data string) (*TraDb, error) {
	tradb := &TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version
			if len(fields) != 5 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return tradb, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return tradb, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return tradb, err
			}

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return tradb, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

// Parses a TraDb from a file.
func ParseFile() (*TraDb, error) {
	tradb := &TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println(FileNotFound.Error())
		tradb = New()

		return tradb, FileNotFound
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return tradb, err
	}

	return Parse(string(bs))
}

// Creates a new TraDb.
// The replica ID is a random string, and the version
// number is set to 1. Checks for files in the current
// directory and stores relevant file state in a map.
func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = bytes[rand.Intn(len(bytes))]
	}
	versionVector[string(replicaId)] = 1 // TODO: check for duplicates

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
		}
		filemap[filename] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

// Writes a TraDb to the db file .trago.db.
func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

// Looks for modified files in the current directory
// and updates the filemap accordingly.
func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	db.VersionVec[db.ReplicaId] += 1
	ourVersion := db.VersionVec[db.ReplicaId]

	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue
		}

		dbRecord := db.Files[filename]
		if dbRecord.MTime == 0 {
			log.Printf("found a new file: %s\n", filename)
			db.Files[filename] = FileState{
				Size:    int(file.Size()),
				MTime:   file.ModTime().UTC().UnixNano(),
				Version: ourVersion,
				Replica: db.ReplicaId,
			}
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = ourVersion
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
	}

	return nil
}

// Compares two TraDbs.
func (local *TraDb) Compare(remote *TraDb) map[string]FileTag {
	tags := make(map[string]FileTag)
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			if state.Version <= remote.VersionVec[state.Replica] {
				log.Printf("deleting: %s\n", file)
				tags[file] = Deleted
			}
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				log.Printf("keeping: %s\n", file)
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				tags[file] = File
			} else {
				log.Printf("conflict: %s\n", file)
				tags[file] = Conflict
			}
		} else {
			log.Printf("unchanged: %s\n", file)
		}
	}

	for file, state := range remoteFiles {
		if local.Files[file].Version > 0 {
			continue
		} else if state.Version > local.VersionVec[state.Replica] {
			log.Printf("downloading: %s\n", file)
			tags[file] = File
		}
	}

	combineVectors(local.VersionVec, remote.VersionVec)
	log.Println(local.VersionVec)
	return tags
}

func combineVectors(v1 map[string]int, v2 map[string]int) {
	for replica, version := range v1 {
		if v2[replica] > version {
			v1[replica] = v2[replica]
		}
	}

	for replica, version := range v2 {
		if v1[replica] < version {
			v1[replica] = version
		}
	}
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		return true
	}
	return false
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package dynamic

import (
	"reflect"
	"testing"

	"github.com/golang/protobuf/proto"

	"github.com/jhump/protoreflect/desc"
	"github.com/jhump/protoreflect/internal/testprotos"
	"github.com/jhump/protoreflect/internal/testutil"
)

func TestBinaryUnaryFields(t *testing.T) {
	binaryTranslationParty(t, unaryFieldsPosMsg)
	binaryTranslationParty(t, unaryFieldsNegMsg)
}

func TestBinaryRepeatedFields(t *testing.T) {
	binaryTranslationParty(t, repeatedFieldsMsg)
}

func TestBinaryPackedRepeatedFields(t *testing.T) {
	binaryTranslationParty(t, repeatedPackedFieldsMsg)
}

func TestBinaryMapKeyFields(t *testing.T) {
	// translation party wants deterministic marshalling to bytes
	defaultDeterminism = true
	defer func() {
		defaultDeterminism = false
	}()

	binaryTranslationParty(t, mapKeyFieldsMsg)
}

func TestMarshalMapValueFields(t *testing.T) {
	// translation party wants deterministic marshalling to bytes
	defaultDeterminism = true
	defer func() {
		defaultDeterminism = false
	}()

	binaryTranslationParty(t, mapValueFieldsMsg)
}

func TestBinaryExtensionFields(t *testing.T) {
	// TODO
}

func TestBinaryUnknownFields(t *testing.T) {
	// create a buffer with both known fields:
	b, err := proto.Marshal(&testprotos.TestMessage{
		Nm: &testprotos.TestMessage_NestedMessage{
			Anm: &testprotos.TestMessage_NestedMessage_AnotherNestedMessage{
				Yanm: []*testprotos.TestMessage_NestedMessage_AnotherNestedMessage_YetAnotherNestedMessage{
					{Foo: proto.String("foo"), Bar: proto.Int32(100), Baz: []byte{1, 2, 3, 4}},
				},
			}},
		Ne: []testprotos.TestMessage_NestedEnum{testprotos.TestMessage_VALUE1, testprotos.TestMessage_VALUE1},
	})
	baseLen := len(b)
	testutil.Ok(t, err)
	buf := newCodedBuffer(b)

	// and unknown fields:
	//   varint encoded field
	buf.encodeTagAndWireType(1234, proto.WireVarint)
	buf.encodeVarint(987654)
	//   fixed 64
	buf.encodeTagAndWireType(2345, proto.WireFixed64)
	buf.encodeFixed64(123456789)
	//   fixed 32, also repeated
	buf.encodeTagAndWireType(3456, proto.WireFixed32)
	buf.encodeFixed32(123456)
	buf.encodeTagAndWireType(3456, proto.WireFixed32)
	buf.encodeFixed32(123457)
	buf.encodeTagAndWireType(3456, proto.WireFixed32)
	buf.encodeFixed32(123458)
	buf.encodeTagAndWireType(3456, proto.WireFixed32)
	buf.encodeFixed32(123459)
	//   length-encoded
	buf.encodeTagAndWireType(4567, proto.WireBytes)
	buf.encodeRawBytes([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})
	//   and... group!
	buf.encodeTagAndWireType(5678, proto.WireStartGroup)
	{
		buf.encodeTagAndWireType(1, proto.WireVarint)
		buf.encodeVarint(1)
		buf.encodeTagAndWireType(2, proto.WireFixed32)
		buf.encodeFixed32(2)
		buf.encodeTagAndWireType(3, proto.WireFixed64)
		buf.encodeFixed64(3)
		buf.encodeTagAndWireType(4, proto.WireBytes)
		buf.encodeRawBytes([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})
		// nested group
		buf.encodeTagAndWireType(5, proto.WireStartGroup)
		{
			buf.encodeTagAndWireType(1, proto.WireVarint)
			buf.encodeVarint(1)
			buf.encodeTagAndWireType(1, proto.WireVarint)
			buf.encodeVarint(2)
			buf.encodeTagAndWireType(1, proto.WireVarint)
			buf.encodeVarint(3)
			buf.encodeTagAndWireType(2, proto.WireBytes)
			buf.encodeRawBytes([]byte("lorem ipsum"))
		}
		buf.encodeTagAndWireType(5, proto.WireEndGroup)
	}
	buf.encodeTagAndWireType(5678, proto.WireEndGroup)
	testutil.Require(t, len(buf.buf) > baseLen) // sanity check

	var msg testprotos.TestMessage
	err = proto.Unmarshal(buf.buf, &msg)
	testutil.Ok(t, err)
	// make sure unrecognized fields parsed correctly
	testutil.Eq(t, buf.buf[baseLen:], msg.XXX_unrecognized)

	// make sure dynamic message's round trip generates same bytes
	md, err := desc.LoadMessageDescriptorForMessage((*testprotos.TestMessage)(nil))
	testutil.Ok(t, err)
	dm := NewMessage(md)
	err = dm.Unmarshal(buf.buf)
	testutil.Ok(t, err)
	bb, err := dm.Marshal()
	testutil.Ok(t, err)
	testutil.Eq(t, buf.buf, bb)

	// now try a full translation party to ensure unknown bits remain correct throughout
	binaryTranslationParty(t, &msg)
}

func binaryTranslationParty(t *testing.T, msg proto.Message) {
	marshalAppendSimple := func(m *Message) ([]byte, error) {
		// Declare a function that has the same interface as (*Message.Marshal) but uses
		// MarshalAppend internally so we can reuse the translation party tests to verify
		// the behavior of MarshalAppend in addition to Marshal.
		b := make([]byte, 0, 2048)
		marshaledB, err := m.MarshalAppend(b)

		// Verify it doesn't allocate a new byte slice.
		assertByteSlicesBackedBySameData(t, b, marshaledB)
		return marshaledB, err
	}

	marshalAppendPrefix := func(m *Message) ([]byte, error) {
		// Same thing as MarshalAppendSimple, but we verify that prefix data is retained.
		prefix := "prefix"
		marshaledB, err := m.MarshalAppend([]byte(prefix))

		// Verify the prefix data is retained.
		testutil.Eq(t, prefix, string(marshaledB[:len(prefix)]))
		return marshaledB[len(prefix):], err
	}

	marshalMethods := []func(m *Message) ([]byte, error){
		(*Message).Marshal,
		marshalAppendSimple,
		marshalAppendPrefix,
	}
	for _, marshalFn := range marshalMethods {
		doTranslationParty(t, msg, proto.Marshal, proto.Unmarshal, marshalFn, (*Message).Unmarshal)
	}
}

// byteSlicesBackedBySameData returns a bool indicating if the raw backing bytes
// under the []byte slice point to the same memory.
func assertByteSlicesBackedBySameData(t *testing.T, a, b []byte) {
	origPtr := reflect.ValueOf(a).Pointer()
	resultPtr := reflect.ValueOf(b).Pointer()
	testutil.Eq(t, origPtr, resultPtr)
}
<commit_msg>add newline<commit_after>package dynamic

import (
	"reflect"
	"testing"

	"github.com/golang/protobuf/proto"

	"github.com/jhump/protoreflect/desc"
	"github.com/jhump/protoreflect/internal/testprotos"
	"github.com/jhump/protoreflect/internal/testutil"
)

func TestBinaryUnaryFields(t *testing.T) {
	binaryTranslationParty(t, unaryFieldsPosMsg)
	binaryTranslationParty(t, unaryFieldsNegMsg)
}

func TestBinaryRepeatedFields(t *testing.T) {
	binaryTranslationParty(t, repeatedFieldsMsg)
}

func TestBinaryPackedRepeatedFields(t *testing.T) {
	binaryTranslationParty(t, repeatedPackedFieldsMsg)
}

func TestBinaryMapKeyFields(t *testing.T) {
	// translation party wants deterministic marshalling to bytes
	defaultDeterminism = true
	defer func() {
		defaultDeterminism = false
	}()

	binaryTranslationParty(t, mapKeyFieldsMsg)
}

func TestMarshalMapValueFields(t *testing.T) {
	// translation party wants deterministic marshalling to bytes
	defaultDeterminism = true
	defer func() {
		defaultDeterminism = false
	}()

	binaryTranslationParty(t, mapValueFieldsMsg)
}

func TestBinaryExtensionFields(t *testing.T) {
	// TODO
}

func TestBinaryUnknownFields(t *testing.T) {
	// create a buffer with both known fields:
	b, err := proto.Marshal(&testprotos.TestMessage{
		Nm: &testprotos.TestMessage_NestedMessage{
			Anm: &testprotos.TestMessage_NestedMessage_AnotherNestedMessage{
				Yanm: []*testprotos.TestMessage_NestedMessage_AnotherNestedMessage_YetAnotherNestedMessage{
					{Foo: proto.String("foo"), Bar: proto.Int32(100), Baz: []byte{1, 2, 3, 4}},
				},
			}},
		Ne: []testprotos.TestMessage_NestedEnum{testprotos.TestMessage_VALUE1, testprotos.TestMessage_VALUE1},
	})
	baseLen := len(b)
	testutil.Ok(t, err)
	buf := newCodedBuffer(b)

	// and unknown fields:
	//   varint encoded field
	buf.encodeTagAndWireType(1234, proto.WireVarint)
	buf.encodeVarint(987654)
	//   fixed 64
	buf.encodeTagAndWireType(2345, proto.WireFixed64)
	buf.encodeFixed64(123456789)
	//   fixed 32, also repeated
	buf.encodeTagAndWireType(3456, proto.WireFixed32)
	buf.encodeFixed32(123456)
	buf.encodeTagAndWireType(3456, proto.WireFixed32)
	buf.encodeFixed32(123457)
	buf.encodeTagAndWireType(3456, proto.WireFixed32)
	buf.encodeFixed32(123458)
	buf.encodeTagAndWireType(3456, proto.WireFixed32)
	buf.encodeFixed32(123459)
	//   length-encoded
	buf.encodeTagAndWireType(4567, proto.WireBytes)
	buf.encodeRawBytes([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})
	//   and... group!
	buf.encodeTagAndWireType(5678, proto.WireStartGroup)
	{
		buf.encodeTagAndWireType(1, proto.WireVarint)
		buf.encodeVarint(1)
		buf.encodeTagAndWireType(2, proto.WireFixed32)
		buf.encodeFixed32(2)
		buf.encodeTagAndWireType(3, proto.WireFixed64)
		buf.encodeFixed64(3)
		buf.encodeTagAndWireType(4, proto.WireBytes)
		buf.encodeRawBytes([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})
		// nested group
		buf.encodeTagAndWireType(5, proto.WireStartGroup)
		{
			buf.encodeTagAndWireType(1, proto.WireVarint)
			buf.encodeVarint(1)
			buf.encodeTagAndWireType(1, proto.WireVarint)
			buf.encodeVarint(2)
			buf.encodeTagAndWireType(1, proto.WireVarint)
			buf.encodeVarint(3)
			buf.encodeTagAndWireType(2, proto.WireBytes)
			buf.encodeRawBytes([]byte("lorem ipsum"))
		}
		buf.encodeTagAndWireType(5, proto.WireEndGroup)
	}
	buf.encodeTagAndWireType(5678, proto.WireEndGroup)
	testutil.Require(t, len(buf.buf) > baseLen) // sanity check

	var msg testprotos.TestMessage
	err = proto.Unmarshal(buf.buf, &msg)
	testutil.Ok(t, err)
	// make sure unrecognized fields parsed correctly
	testutil.Eq(t, buf.buf[baseLen:], msg.XXX_unrecognized)

	// make sure dynamic message's round trip generates same bytes
	md, err := desc.LoadMessageDescriptorForMessage((*testprotos.TestMessage)(nil))
	testutil.Ok(t, err)
	dm := NewMessage(md)
	err = dm.Unmarshal(buf.buf)
	testutil.Ok(t, err)
	bb, err := dm.Marshal()
	testutil.Ok(t, err)
	testutil.Eq(t, buf.buf, bb)

	// now try a full translation party to ensure unknown bits remain correct throughout
	binaryTranslationParty(t, &msg)
}

func binaryTranslationParty(t *testing.T, msg proto.Message) {
	marshalAppendSimple := func(m *Message) ([]byte, error) {
		// Declare a function that has the same interface as (*Message.Marshal) but uses
		// MarshalAppend internally so we can reuse the translation party tests to verify
		// the behavior of MarshalAppend in addition to Marshal.
		b := make([]byte, 0, 2048)
		marshaledB, err := m.MarshalAppend(b)

		// Verify it doesn't allocate a new byte slice.
		assertByteSlicesBackedBySameData(t, b, marshaledB)
		return marshaledB, err
	}

	marshalAppendPrefix := func(m *Message) ([]byte, error) {
		// Same thing as MarshalAppendSimple, but we verify that prefix data is retained.
		prefix := "prefix"
		marshaledB, err := m.MarshalAppend([]byte(prefix))

		// Verify the prefix data is retained.
		testutil.Eq(t, prefix, string(marshaledB[:len(prefix)]))
		return marshaledB[len(prefix):], err
	}

	marshalMethods := []func(m *Message) ([]byte, error){
		(*Message).Marshal,
		marshalAppendSimple,
		marshalAppendPrefix,
	}

	for _, marshalFn := range marshalMethods {
		doTranslationParty(t, msg, proto.Marshal, proto.Unmarshal, marshalFn, (*Message).Unmarshal)
	}
}

// byteSlicesBackedBySameData returns a bool indicating if the raw backing bytes
// under the []byte slice point to the same memory.
func assertByteSlicesBackedBySameData(t *testing.T, a, b []byte) {
	origPtr := reflect.ValueOf(a).Pointer()
	resultPtr := reflect.ValueOf(b).Pointer()
	testutil.Eq(t, origPtr, resultPtr)
}
<|endoftext|>


<commit_before>package db

import (
  "gopkg.in/mgo.v2"
  "os"
  "log"
)

var appdb *mgo.Session
var Database = os.Getenv("MONGOLAB_URI")

func init() {
	s, err := mgo.Dial(os.Getenv("MONGO_DB_NAME"))
	if err != nil {
		log.Println("Error connecting to the database: " + err.Error())
	}
	s.SetMode(mgo.Monotonic, true)
	appdb = s
}

func AppDB() *mgo.Session {
	return appdb
}
<commit_msg>Correct env var using is DB package<commit_after>package db

import (
	"log"
	"os"

	"gopkg.in/mgo.v2"
)

var appdb *mgo.Session
var Database = os.Getenv("MONGO_DB_NAME")

func init() {
	s, err := mgo.Dial(os.Getenv("MONGOLAB_URI"))
	if err != nil {
		log.Println("Error connecting to the database: " + err.Error())
	}
	s.SetMode(mgo.Monotonic, true)
	appdb = s
}

func AppDB() *mgo.Session {
	return appdb
}
<|endoftext|>


<commit_before>package logging

import (
	smoke ".."
	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Loggregator:", func() {
	var testConfig = smoke.GetConfig()
	var useExistingApp = (testConfig.LoggingApp != "")
	var appName string

	Describe("cf logs", func() {
		AfterEach(func() {
			if testConfig.Cleanup && !useExistingApp {
				Expect(cf.Cf("delete", appName, "-f", "-r").Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))
			}
		})

		Context("linux", func() {
			BeforeEach(func() {
				if !useExistingApp {
					appName = generator.RandomName()
					Expect(cf.Cf("push", appName, "-p", SIMPLE_RUBY_APP_BITS_PATH, "-d", testConfig.AppsDomain).Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
				}
			})

			It("can see app messages in the logs", func() {
				Eventually(func() *Session {
					appLogsSession := cf.Cf("logs", "--recent", appName)
					Expect(appLogsSession.Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))
					return appLogsSession
				}, CF_TIMEOUT_IN_SECONDS*5).Should(Say(`\[(App|APP)/0\]`))
			})
		})

		Context("windows", func() {
			BeforeEach(func() {
				smoke.SkipIfWindows(testConfig)

				appName = generator.RandomName()
				Expect(cf.Cf("push", appName, "-p", SIMPLE_DOTNET_APP_BITS_PATH, "-d", testConfig.AppsDomain, "-s", "windows2012R2", "-b", "binary_buildpack", "--no-start").Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
				smoke.EnableDiego(appName)
				Expect(cf.Cf("start", appName).Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
			})

			It("can see app messages in the logs", func() {
				Eventually(func() *Session {
					appLogsSession := cf.Cf("logs", "--recent", appName)
					Expect(appLogsSession.Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))
					return appLogsSession
				}, CF_TIMEOUT_IN_SECONDS*5).Should(Say(`\[(App|APP)/0\]`))
			})
		})
	})
})
<commit_msg>Specify ruby_buildpack when deploying ruby app in smoke tests<commit_after>package logging

import (
	smoke ".."
	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Loggregator:", func() {
	var testConfig = smoke.GetConfig()
	var useExistingApp = (testConfig.LoggingApp != "")
	var appName string

	Describe("cf logs", func() {
		AfterEach(func() {
			if testConfig.Cleanup && !useExistingApp {
				Expect(cf.Cf("delete", appName, "-f", "-r").Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))
			}
		})

		Context("linux", func() {
			BeforeEach(func() {
				if !useExistingApp {
					appName = generator.RandomName()
					Expect(cf.Cf("push", appName, "-b", "ruby_buildpack", "-p", SIMPLE_RUBY_APP_BITS_PATH, "-d", testConfig.AppsDomain).Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
				}
			})

			It("can see app messages in the logs", func() {
				Eventually(func() *Session {
					appLogsSession := cf.Cf("logs", "--recent", appName)
					Expect(appLogsSession.Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))
					return appLogsSession
				}, CF_TIMEOUT_IN_SECONDS*5).Should(Say(`\[(App|APP)/0\]`))
			})
		})

		Context("windows", func() {
			BeforeEach(func() {
				smoke.SkipIfWindows(testConfig)

				appName = generator.RandomName()
				Expect(cf.Cf("push", appName, "-p", SIMPLE_DOTNET_APP_BITS_PATH, "-d", testConfig.AppsDomain, "-s", "windows2012R2", "-b", "binary_buildpack", "--no-start").Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
				smoke.EnableDiego(appName)
				Expect(cf.Cf("start", appName).Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
			})

			It("can see app messages in the logs", func() {
				Eventually(func() *Session {
					appLogsSession := cf.Cf("logs", "--recent", appName)
					Expect(appLogsSession.Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))
					return appLogsSession
				}, CF_TIMEOUT_IN_SECONDS*5).Should(Say(`\[(App|APP)/0\]`))
			})
		})
	})
})
<|endoftext|>


<commit_before>package kloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/db/mongodb/modelhelper"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/terraformer"
	tf "koding/kites/terraformer"

	"labix.org/v2/mgo/bson"

	"golang.org/x/net/context"

	"github.com/koding/kite"
	"github.com/mitchellh/mapstructure"
)

type AwsBootstrapOutput struct {
	ACL       string `json:"acl" mapstructure:"acl"`
	CidrBlock string `json:"cidr_block" mapstructure:"cidr_block"`
	IGW       string `json:"igw" mapstructure:"igw"`
	KeyPair   string `json:"key_pair" mapstructure:"key_pair"`
	RTB       string `json:"rtb" mapstructure:"rtb"`
	SG        string `json:"sg" mapstructure:"sg"`
	Subnet    string `json:"subnet" mapstructure:"subnet"`
	VPC       string `json:"vpc" mapstructure:"vpc"`
}

type TerraformBootstrapRequest struct {
	// PublicKeys contains publicKeys to be used with terraform
	PublicKeys []string `json:"publicKeys"`

	// Destroy destroys the bootstrap resource associated with the given public
	// keys
	Destroy bool
}

func (k *Kloud) Bootstrap(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformBootstrapRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if len(args.PublicKeys) == 0 {
		return nil, errors.New("publicKeys are not passed")
	}

	ctx := k.ContextCreator(context.Background())
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	creds, err := fetchCredentials(r.Username, sess.DB, args.PublicKeys)
	if err != nil {
		return nil, err
	}

	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	for _, cred := range creds.Creds {
		// We are going to support more providers in the future, for now only allow aws
		if cred.Provider != "aws" {
			return nil, fmt.Errorf("Bootstrap is only supported for 'aws' provider. Got: '%s'", cred.Provider)
		}

		finalBootstrap, err := cred.appendAWSVariable(fmt.Sprintf(awsBootstrap, k.PublicKeys.PublicKey))
		if err != nil {
			return nil, err
		}

		keyName := "kloud-deployment-" + r.Username
		finalBootstrap, err = appendKeyName(finalBootstrap, keyName)
		if err != nil {
			return nil, err
		}

		k.Log.Debug("[%s] Final bootstrap:", cred.PublicKey)
		k.Log.Debug(finalBootstrap)

		// Important so bootstraping is distributed amongs multiple users. If I
		// use these keys to bootstrap, any other user should be not create
		// again, instead they should be fetch and use the existing bootstrap
		// data.

		// TODO(arslan): change this once we have group context name
		groupName := "koding"
		awsOutput := &AwsBootstrapOutput{}

		if args.Destroy {
			k.Log.Info("Destroying bootstrap resources belonging to public key '%s'", cred.PublicKey)
			_, err := tfKite.Destroy(&tf.TerraformRequest{
				Content:   finalBootstrap,
				ContentID: groupName + "-" + cred.PublicKey,
			})
			if err != nil {
				return nil, err
			}
		} else {
			k.Log.Info("Creating bootstrap resources belonging to public key '%s'", cred.PublicKey)
			state, err := tfKite.Apply(&tf.TerraformRequest{
				Content:   finalBootstrap,
				ContentID: groupName + "-" + cred.PublicKey,
			})
			if err != nil {
				return nil, err
			}

			k.Log.Debug("[%s] state.RootModule().Outputs = %+v\n", cred.PublicKey, state.RootModule().Outputs)
			if err := mapstructure.Decode(state.RootModule().Outputs, &awsOutput); err != nil {
				return nil, err
			}
		}

		k.Log.Debug("[%s] Aws Output: %+v", cred.PublicKey, awsOutput)
		if err := modelhelper.UpdateCredentialData(cred.PublicKey, bson.M{
			"$set": bson.M{
				"meta.acl":        awsOutput.ACL,
				"meta.cidr_block": awsOutput.CidrBlock,
				"meta.igw":        awsOutput.IGW,
				"meta.key_pair":   awsOutput.KeyPair,
				"meta.rtb":        awsOutput.RTB,
				"meta.sg":         awsOutput.SG,
				"meta.subnet":     awsOutput.Subnet,
				"meta.vpc":        awsOutput.VPC,
			},
		}); err != nil {
			return nil, err
		}
	}

	return true, nil
}

func appendKeyName(template, keyName string) (string, error) {
	var data struct {
		Output   map[string]map[string]interface{} `json:"output,omitempty"`
		Resource map[string]map[string]interface{} `json:"resource,omitempty"`
		Provider map[string]map[string]interface{} `json:"provider,omitempty"`
		Variable map[string]map[string]interface{} `json:"variable,omitempty"`
	}

	if err := json.Unmarshal([]byte(template), &data); err != nil {
		return "", err
	}

	data.Variable["key_name"] = map[string]interface{}{
		"default": keyName,
	}

	out, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return "", err
	}

	return string(out), nil
}

var awsBootstrap = `{
    "provider": {
        "aws": {
            "access_key": "${var.access_key}",
            "secret_key": "${var.secret_key}",
            "region": "${var.region}"
        }
    },
    "output": {
        "vpc": {
            "value": "${aws_vpc.vpc.id}"
        },
        "cidr_block": {
            "value": "${aws_vpc.vpc.cidr_block}"
        },
        "rtb": {
            "value": "${aws_vpc.vpc.main_route_table_id}"
        },
        "acl": {
            "value": "${aws_vpc.vpc.default_network_acl_id}"
        },
        "igw": {
            "value": "${aws_internet_gateway.main_vpc_igw.id}"
        },
        "subnet": {
            "value": "${aws_subnet.main_koding_subnet.id}"
        },
        "sg": {
            "value": "${aws_security_group.allow_all.id}"
        },
        "key_pair": {
            "value": "${aws_key_pair.koding_key_pair.key_name}"
        }
    },
    "resource": {
        "aws_vpc": {
            "vpc": {
                "cidr_block": "${var.cidr_block}",
                "tags": {
                    "Name": "${var.environment_name}"
                }
            }
        },
        "aws_internet_gateway": {
            "main_vpc_igw": {
                "tags": {
                    "Name": "${var.environment_name}"
                },
                "vpc_id": "${aws_vpc.vpc.id}"
            }
        },
        "aws_subnet": {
            "main_koding_subnet": {
                "availability_zone": "${lookup(var.aws_availability_zones, var.region)}",
                "cidr_block": "${var.cidr_block}",
                "map_public_ip_on_launch": true,
                "tags": {
                    "Name": "${var.environment_name}",
                    "subnet": "public"
                },
                "vpc_id": "${aws_vpc.vpc.id}"
            }
        },
        "aws_route_table": {
            "public": {
                "route": {
                    "cidr_block": "0.0.0.0/0",
                    "gateway_id": "${aws_internet_gateway.main_vpc_igw.id}"
                },
                "tags": {
                    "Name": "${var.environment_name}",
                    "routeTable": "koding",
                    "subnet": "public"
                },
                "vpc_id": "${aws_vpc.vpc.id}"
            }
        },
        "aws_route_table_association": {
            "public-1": {
                "route_table_id": "${aws_route_table.public.id}",
                "subnet_id": "${aws_subnet.main_koding_subnet.id}"
            }
        },
        "aws_security_group": {
            "allow_all": {
                "description": "Allow all inbound and outbound traffic",
                "ingress": {
                    "from_port": 0,
                    "to_port": 0,
                    "protocol": "-1",
                    "cidr_blocks": ["0.0.0.0/0"],
                    "self": true
                },
                "egress": {
                    "from_port": 0,
                    "to_port": 0,
                    "protocol": "-1",
                    "self": true,
                    "cidr_blocks": ["0.0.0.0/0"]
                },
                "name": "allow_all",
                "tags": {
                    "Name": "${var.environment_name}"
                },
                "vpc_id": "${aws_vpc.vpc.id}"
            }
        },
        "aws_key_pair": {
            "koding_key_pair": {
                "key_name": "${var.key_name}",
                "public_key": "%s"
            }
        }
    },
    "variable": {
        "cidr_block": {
            "default": "10.0.0.0/16"
        },
        "environment_name": {
            "default": "Koding-Bootstrap"
        },
        "aws_availability_zones": {
            "default": {
                "ap-northeast-1": "ap-northeast-1b",
                "ap-southeast-1": "ap-southeast-1b",
                "ap-southeast-2": "ap-southeast-2b",
                "eu-central-1": "eu-central-1b",
                "eu-west-1": "eu-west-1b",
                "sa-east-1": "sa-east-1b",
                "us-east-1": "us-east-1b",
                "us-west-1": "us-west-1b",
                "us-west-2": "us-west-2b"
            }
        }
    }
}`
<commit_msg>bootstrap: add default ubuntu ami for future reference<commit_after>package kloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/db/mongodb/modelhelper"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/terraformer"
	tf "koding/kites/terraformer"

	"labix.org/v2/mgo/bson"

	"golang.org/x/net/context"

	"github.com/koding/kite"
	"github.com/mitchellh/mapstructure"
)

type AwsBootstrapOutput struct {
	ACL       string `json:"acl" mapstructure:"acl"`
	CidrBlock string `json:"cidr_block" mapstructure:"cidr_block"`
	IGW       string `json:"igw" mapstructure:"igw"`
	KeyPair   string `json:"key_pair" mapstructure:"key_pair"`
	RTB       string `json:"rtb" mapstructure:"rtb"`
	SG        string `json:"sg" mapstructure:"sg"`
	Subnet    string `json:"subnet" mapstructure:"subnet"`
	VPC       string `json:"vpc" mapstructure:"vpc"`
	AMI       string `json:"ami" mapstructure:"ami"`
}

type TerraformBootstrapRequest struct {
	// PublicKeys contains publicKeys to be used with terraform
	PublicKeys []string `json:"publicKeys"`

	// Destroy destroys the bootstrap resource associated with the given public
	// keys
	Destroy bool
}

func (k *Kloud) Bootstrap(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformBootstrapRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if len(args.PublicKeys) == 0 {
		return nil, errors.New("publicKeys are not passed")
	}

	ctx := k.ContextCreator(context.Background())
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	creds, err := fetchCredentials(r.Username, sess.DB, args.PublicKeys)
	if err != nil {
		return nil, err
	}

	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	for _, cred := range creds.Creds {
		// We are going to support more providers in the future, for now only allow aws
		if cred.Provider != "aws" {
			return nil, fmt.Errorf("Bootstrap is only supported for 'aws' provider. Got: '%s'", cred.Provider)
		}

		finalBootstrap, err := cred.appendAWSVariable(fmt.Sprintf(awsBootstrap, k.PublicKeys.PublicKey))
		if err != nil {
			return nil, err
		}

		keyName := "kloud-deployment-" + r.Username
		finalBootstrap, err = appendKeyName(finalBootstrap, keyName)
		if err != nil {
			return nil, err
		}

		k.Log.Debug("[%s] Final bootstrap:", cred.PublicKey)
		k.Log.Debug(finalBootstrap)

		// Important so bootstraping is distributed amongs multiple users. If I
		// use these keys to bootstrap, any other user should be not create
		// again, instead they should be fetch and use the existing bootstrap
		// data.

		// TODO(arslan): change this once we have group context name
		groupName := "koding"
		awsOutput := &AwsBootstrapOutput{}

		if args.Destroy {
			k.Log.Info("Destroying bootstrap resources belonging to public key '%s'", cred.PublicKey)
			_, err := tfKite.Destroy(&tf.TerraformRequest{
				Content:   finalBootstrap,
				ContentID: groupName + "-" + cred.PublicKey,
			})
			if err != nil {
				return nil, err
			}
		} else {
			k.Log.Info("Creating bootstrap resources belonging to public key '%s'", cred.PublicKey)
			state, err := tfKite.Apply(&tf.TerraformRequest{
				Content:   finalBootstrap,
				ContentID: groupName + "-" + cred.PublicKey,
			})
			if err != nil {
				return nil, err
			}

			k.Log.Debug("[%s] state.RootModule().Outputs = %+v\n", cred.PublicKey, state.RootModule().Outputs)
			if err := mapstructure.Decode(state.RootModule().Outputs, &awsOutput); err != nil {
				return nil, err
			}
		}

		k.Log.Debug("[%s] Aws Output: %+v", cred.PublicKey, awsOutput)
		if err := modelhelper.UpdateCredentialData(cred.PublicKey, bson.M{
			"$set": bson.M{
				"meta.acl":        awsOutput.ACL,
				"meta.cidr_block": awsOutput.CidrBlock,
				"meta.igw":        awsOutput.IGW,
				"meta.key_pair":   awsOutput.KeyPair,
				"meta.rtb":        awsOutput.RTB,
				"meta.sg":         awsOutput.SG,
				"meta.subnet":     awsOutput.Subnet,
				"meta.vpc":        awsOutput.VPC,
				"meta.ami":        awsOutput.AMI,
			},
		}); err != nil {
			return nil, err
		}
	}

	return true, nil
}

func appendKeyName(template, keyName string) (string, error) {
	var data struct {
		Output   map[string]map[string]interface{} `json:"output,omitempty"`
		Resource map[string]map[string]interface{} `json:"resource,omitempty"`
		Provider map[string]map[string]interface{} `json:"provider,omitempty"`
		Variable map[string]map[string]interface{} `json:"variable,omitempty"`
	}

	if err := json.Unmarshal([]byte(template), &data); err != nil {
		return "", err
	}

	data.Variable["key_name"] = map[string]interface{}{
		"default": keyName,
	}

	out, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return "", err
	}

	return string(out), nil
}

var awsBootstrap = `{
    "provider": {
        "aws": {
            "access_key": "${var.access_key}",
            "secret_key": "${var.secret_key}",
            "region": "${var.region}"
        }
    },
    "output": {
        "vpc": {
            "value": "${aws_vpc.vpc.id}"
        },
        "cidr_block": {
            "value": "${aws_vpc.vpc.cidr_block}"
        },
        "rtb": {
            "value": "${aws_vpc.vpc.main_route_table_id}"
        },
        "acl": {
            "value": "${aws_vpc.vpc.default_network_acl_id}"
        },
        "igw": {
            "value": "${aws_internet_gateway.main_vpc_igw.id}"
        },
        "subnet": {
            "value": "${aws_subnet.main_koding_subnet.id}"
        },
        "sg": {
            "value": "${aws_security_group.allow_all.id}"
        },
        "ami": {
            "value": "${lookup(var.aws_amis, var.region)}"
        },
        "key_pair": {
            "value": "${aws_key_pair.koding_key_pair.key_name}"
        }
    },
    "resource": {
        "aws_vpc": {
            "vpc": {
                "cidr_block": "${var.cidr_block}",
                "tags": {
                    "Name": "${var.environment_name}"
                }
            }
        },
        "aws_internet_gateway": {
            "main_vpc_igw": {
                "tags": {
                    "Name": "${var.environment_name}"
                },
                "vpc_id": "${aws_vpc.vpc.id}"
            }
        },
        "aws_subnet": {
            "main_koding_subnet": {
                "availability_zone": "${lookup(var.aws_availability_zones, var.region)}",
                "cidr_block": "${var.cidr_block}",
                "map_public_ip_on_launch": true,
                "tags": {
                    "Name": "${var.environment_name}",
                    "subnet": "public"
                },
                "vpc_id": "${aws_vpc.vpc.id}"
            }
        },
        "aws_route_table": {
            "public": {
                "route": {
                    "cidr_block": "0.0.0.0/0",
                    "gateway_id": "${aws_internet_gateway.main_vpc_igw.id}"
                },
                "tags": {
                    "Name": "${var.environment_name}",
                    "routeTable": "koding",
                    "subnet": "public"
                },
                "vpc_id": "${aws_vpc.vpc.id}"
            }
        },
        "aws_route_table_association": {
            "public-1": {
                "route_table_id": "${aws_route_table.public.id}",
                "subnet_id": "${aws_subnet.main_koding_subnet.id}"
            }
        },
        "aws_security_group": {
            "allow_all": {
                "description": "Allow all inbound and outbound traffic",
                "ingress": {
                    "from_port": 0,
                    "to_port": 0,
                    "protocol": "-1",
                    "cidr_blocks": ["0.0.0.0/0"],
                    "self": true
                },
                "egress": {
                    "from_port": 0,
                    "to_port": 0,
                    "protocol": "-1",
                    "self": true,
                    "cidr_blocks": ["0.0.0.0/0"]
                },
                "name": "allow_all",
                "tags": {
                    "Name": "${var.environment_name}"
                },
                "vpc_id": "${aws_vpc.vpc.id}"
            }
        },
        "aws_key_pair": {
            "koding_key_pair": {
                "key_name": "${var.key_name}",
                "public_key": "%s"
            }
        }
    },
    "variable": {
        "cidr_block": {
            "default": "10.0.0.0/16"
        },
        "environment_name": {
            "default": "Koding-Bootstrap"
        },
        "aws_availability_zones": {
            "default": {
                "ap-northeast-1": "ap-northeast-1b",
                "ap-southeast-1": "ap-southeast-1b",
                "ap-southeast-2": "ap-southeast-2b",
                "eu-central-1": "eu-central-1b",
                "eu-west-1": "eu-west-1b",
                "sa-east-1": "sa-east-1b",
                "us-east-1": "us-east-1b",
                "us-west-1": "us-west-1b",
                "us-west-2": "us-west-2b"
            }
        },
        "aws_amis": {
            "default": {
                "ap-northeast-1": "ami-9e5cff9e",
                "ap-southeast-1": "ami-ec7879be",
                "ap-southeast-2": "ami-2fce8b15",
                "eu-central-1": "ami-60f9c27d",
                "eu-west-1": "ami-7c4b0a0b",
                "sa-east-1": "ami-cd9518d0",
                "us-east-1": "ami-cf35f3a4",
                "us-west-1": "ami-b33dccf7",
                "us-west-2": "ami-8d5b5dbd"
            }
        }
    }
}`
<|endoftext|>