<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

func Proxy(w http.ResponseWriter, req *http.Request) {
	// put back Host to the header
	req.Header.Set("Host", req.Host)

	displayReq(req)
	displayHeader(req)
	displayBody(req)
	fmt.Println("")

	fmt.Fprintln(w, "ok")
}

func displayReq(req *http.Request) {
	display("%s %s %s\n", req.Method, reqPath(req), req.Proto)
}

func displayHeader(req *http.Request) {
	for k, v := range req.Header {
		display("%s: %s\n", k, strings.Join(v, ","))
	}
}

func displayBody(req *http.Request) {
	content, err := ioutil.ReadAll(req.Body)
	defer req.Body.Close()

	if err == nil {
		fmt.Println(string(content))
	}
}

func display(format string, a ...interface{}) {
	format = fmt.Sprintf("> %s", format)
	fmt.Printf(format, a...)
}

func reqPath(req *http.Request) string {
	path := req.URL.Path
	query := []string{}
	for k, v := range req.URL.Query() {
		query = append(query, fmt.Sprintf("%s=%s", k, strings.Join(v, ",")))
	}

	if len(query) > 0 {
		path = fmt.Sprintf("%s?%s", path, strings.Join(query, "&"))
	}

	return path
}
<commit_msg>Group query by key name<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

func Proxy(w http.ResponseWriter, req *http.Request) {
	// put back Host to the header
	req.Header.Set("Host", req.Host)

	displayReq(req)
	displayHeader(req)
	displayBody(req)
	fmt.Println("")

	fmt.Fprintln(w, "ok")
}

func displayReq(req *http.Request) {
	display("%s %s %s\n", req.Method, reqPath(req), req.Proto)
}

func displayHeader(req *http.Request) {
	for k, v := range req.Header {
		display("%s: %s\n", k, strings.Join(v, ","))
	}
}

func displayBody(req *http.Request) {
	content, err := ioutil.ReadAll(req.Body)
	defer req.Body.Close()

	if err == nil {
		fmt.Println(string(content))
	}
}

func display(format string, a ...interface{}) {
	format = fmt.Sprintf("> %s", format)
	fmt.Printf(format, a...)
}

func reqPath(req *http.Request) string {
	path := req.URL.Path
	query := []string{}
	for k, v := range req.URL.Query() {
		for _, vv := range v {
			query = append(query, fmt.Sprintf("%s=%s", k, vv))
		}
	}

	if len(query) > 0 {
		path = fmt.Sprintf("%s?%s", path, strings.Join(query, "&"))
	}

	return path
}
<|endoftext|>


<commit_before>package goproxy

import (
	"bufio"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"regexp"
	"sync/atomic"
)

// The basic proxy type. Implements http.Handler.
type ProxyHttpServer struct {
	// session variable must be aligned in i386
	// see http://golang.org/src/pkg/sync/atomic/doc.go#L41
	sess int64
	// setting Verbose to true will log information on each request sent to the proxy
	Verbose         bool
	Logger          *log.Logger
	NonproxyHandler http.Handler
	reqHandlers     []ReqHandler
	respHandlers    []RespHandler
	httpsHandlers   []HttpsHandler
	Tr              *http.Transport
	// ConnectDial will be used to create TCP connections for CONNECT requests
	// if nil Tr.Dial will be used
	ConnectDial func(network string, addr string) (net.Conn, error)
}

var hasPort = regexp.MustCompile(`:\d+$`)

func copyHeaders(dst, src http.Header) {
	for k, _ := range dst {
		dst.Del(k)
	}
	for k, vs := range src {
		for _, v := range vs {
			dst.Add(k, v)
		}
	}
}

func isEof(r *bufio.Reader) bool {
	_, err := r.Peek(1)
	if err == io.EOF {
		return true
	}
	return false
}

func (proxy *ProxyHttpServer) filterRequest(r *http.Request, ctx *ProxyCtx) (req *http.Request, resp *http.Response) {
	req = r
	for _, h := range proxy.reqHandlers {
		req, resp = h.Handle(r, ctx)
		// non-nil resp means the handler decided to skip sending the request
		// and return canned response instead.
		if resp != nil {
			break
		}
	}
	return
}
func (proxy *ProxyHttpServer) filterResponse(respOrig *http.Response, ctx *ProxyCtx) (resp *http.Response) {
	resp = respOrig
	for _, h := range proxy.respHandlers {
		ctx.Resp = resp
		resp = h.Handle(resp, ctx)
	}
	return
}

func removeProxyHeaders(ctx *ProxyCtx, r *http.Request) {
	r.RequestURI = "" // this must be reset when serving a request with the client
	ctx.Logf("Sending request %v %v", r.Method, r.URL.String())
	// If no Accept-Encoding header exists, Transport will add the headers it can accept
	// and would wrap the response body with the relevant reader.
	r.Header.Del("Accept-Encoding")
	// curl can add that, see
	// http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/web-proxy-connection-header.html
	r.Header.Del("Proxy-Connection")
	r.Header.Del("Proxy-Authenticate")
	r.Header.Del("Proxy-Authorization")
	// Connection, Authenticate and Authorization are single hop Header:
	// http://www.w3.org/Protocols/rfc2616/rfc2616.txt
	// 14.10 Connection
	//   The Connection general-header field allows the sender to specify
	//   options that are desired for that particular connection and MUST NOT
	//   be communicated by proxies over further connections.
	r.Header.Del("Connection")
}

// Standard net/http function. Shouldn't be used directly, http.Serve will use it.
func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//r.Header["X-Forwarded-For"] = w.RemoteAddr()
	if r.Method == "CONNECT" {
		proxy.handleHttps(w, r)
	} else {
		ctx := &ProxyCtx{Req: r, Session: atomic.AddInt64(&proxy.sess, 1), proxy: proxy}

		var err error
		ctx.Logf("Got request %v %v %v %v", r.URL.Path, r.Host, r.Method, r.URL.String())
		if !r.URL.IsAbs() {
			proxy.NonproxyHandler.ServeHTTP(w, r)
			return
		}
		r, resp := proxy.filterRequest(r, ctx)

		if resp == nil {
			removeProxyHeaders(ctx, r)
			resp, err = ctx.RoundTrip(r)
			if err != nil {
				ctx.Error = err
				resp = proxy.filterResponse(nil, ctx)
				if resp == nil {
					ctx.Logf("error read response %v %v:", r.URL.Host, err.Error())
					http.Error(w, err.Error(), 500)
					return
				}
			}
			ctx.Logf("Received response %v", resp.Status)
		}
		origBody := resp.Body
		resp = proxy.filterResponse(resp, ctx)
		defer origBody.Close()
		ctx.Logf("Copying response to client %v [%d]", resp.Status, resp.StatusCode)
		// http.ResponseWriter will take care of filling the correct response length
		// Setting it now, might impose wrong value, contradicting the actual new
		// body the user returned.
		// We keep the original body to remove the header only if things changed.
		// This will prevent problems with HEAD requests where there's no body, yet,
		// the Content-Length header should be set.
		if origBody != resp.Body {
			resp.Header.Del("Content-Length")
		}
		copyHeaders(w.Header(), resp.Header)
		w.WriteHeader(resp.StatusCode)
		nr, err := io.Copy(w, resp.Body)
		if err := resp.Body.Close(); err != nil {
			ctx.Warnf("Can't close response body %v", err)
		}
		ctx.Logf("Copied %v bytes to client error=%v", nr, err)
	}
}

// New proxy server, logs to StdErr by default
func NewProxyHttpServer() *ProxyHttpServer {
	proxy := ProxyHttpServer{
		Logger:        log.New(os.Stderr, "", log.LstdFlags),
		reqHandlers:   []ReqHandler{},
		respHandlers:  []RespHandler{},
		httpsHandlers: []HttpsHandler{},
		NonproxyHandler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			http.Error(w, "This is a proxy server. Does not respond to non-proxy requests.", 500)
		}),
		Tr: &http.Transport{TLSClientConfig: tlsClientSkipVerify,
			Proxy: http.ProxyFromEnvironment},
	}
	proxy.ConnectDial = dialerFromEnv(&proxy)
	return &proxy
}
<commit_msg>Update stale URL in comment<commit_after>package goproxy

import (
	"bufio"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"regexp"
	"sync/atomic"
)

// The basic proxy type. Implements http.Handler.
type ProxyHttpServer struct {
	// session variable must be aligned in i386
	// see http://golang.org/src/pkg/sync/atomic/doc.go#L41
	sess int64
	// setting Verbose to true will log information on each request sent to the proxy
	Verbose         bool
	Logger          *log.Logger
	NonproxyHandler http.Handler
	reqHandlers     []ReqHandler
	respHandlers    []RespHandler
	httpsHandlers   []HttpsHandler
	Tr              *http.Transport
	// ConnectDial will be used to create TCP connections for CONNECT requests
	// if nil Tr.Dial will be used
	ConnectDial func(network string, addr string) (net.Conn, error)
}

var hasPort = regexp.MustCompile(`:\d+$`)

func copyHeaders(dst, src http.Header) {
	for k, _ := range dst {
		dst.Del(k)
	}
	for k, vs := range src {
		for _, v := range vs {
			dst.Add(k, v)
		}
	}
}

func isEof(r *bufio.Reader) bool {
	_, err := r.Peek(1)
	if err == io.EOF {
		return true
	}
	return false
}

func (proxy *ProxyHttpServer) filterRequest(r *http.Request, ctx *ProxyCtx) (req *http.Request, resp *http.Response) {
	req = r
	for _, h := range proxy.reqHandlers {
		req, resp = h.Handle(r, ctx)
		// non-nil resp means the handler decided to skip sending the request
		// and return canned response instead.
		if resp != nil {
			break
		}
	}
	return
}
func (proxy *ProxyHttpServer) filterResponse(respOrig *http.Response, ctx *ProxyCtx) (resp *http.Response) {
	resp = respOrig
	for _, h := range proxy.respHandlers {
		ctx.Resp = resp
		resp = h.Handle(resp, ctx)
	}
	return
}

func removeProxyHeaders(ctx *ProxyCtx, r *http.Request) {
	r.RequestURI = "" // this must be reset when serving a request with the client
	ctx.Logf("Sending request %v %v", r.Method, r.URL.String())
	// If no Accept-Encoding header exists, Transport will add the headers it can accept
	// and would wrap the response body with the relevant reader.
	r.Header.Del("Accept-Encoding")
	// curl can add that, see
	// https://jdebp.eu./FGA/web-proxy-connection-header.html
	r.Header.Del("Proxy-Connection")
	r.Header.Del("Proxy-Authenticate")
	r.Header.Del("Proxy-Authorization")
	// Connection, Authenticate and Authorization are single hop Header:
	// http://www.w3.org/Protocols/rfc2616/rfc2616.txt
	// 14.10 Connection
	//   The Connection general-header field allows the sender to specify
	//   options that are desired for that particular connection and MUST NOT
	//   be communicated by proxies over further connections.
	r.Header.Del("Connection")
}

// Standard net/http function. Shouldn't be used directly, http.Serve will use it.
func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	//r.Header["X-Forwarded-For"] = w.RemoteAddr()
	if r.Method == "CONNECT" {
		proxy.handleHttps(w, r)
	} else {
		ctx := &ProxyCtx{Req: r, Session: atomic.AddInt64(&proxy.sess, 1), proxy: proxy}

		var err error
		ctx.Logf("Got request %v %v %v %v", r.URL.Path, r.Host, r.Method, r.URL.String())
		if !r.URL.IsAbs() {
			proxy.NonproxyHandler.ServeHTTP(w, r)
			return
		}
		r, resp := proxy.filterRequest(r, ctx)

		if resp == nil {
			removeProxyHeaders(ctx, r)
			resp, err = ctx.RoundTrip(r)
			if err != nil {
				ctx.Error = err
				resp = proxy.filterResponse(nil, ctx)
				if resp == nil {
					ctx.Logf("error read response %v %v:", r.URL.Host, err.Error())
					http.Error(w, err.Error(), 500)
					return
				}
			}
			ctx.Logf("Received response %v", resp.Status)
		}
		origBody := resp.Body
		resp = proxy.filterResponse(resp, ctx)
		defer origBody.Close()
		ctx.Logf("Copying response to client %v [%d]", resp.Status, resp.StatusCode)
		// http.ResponseWriter will take care of filling the correct response length
		// Setting it now, might impose wrong value, contradicting the actual new
		// body the user returned.
		// We keep the original body to remove the header only if things changed.
		// This will prevent problems with HEAD requests where there's no body, yet,
		// the Content-Length header should be set.
		if origBody != resp.Body {
			resp.Header.Del("Content-Length")
		}
		copyHeaders(w.Header(), resp.Header)
		w.WriteHeader(resp.StatusCode)
		nr, err := io.Copy(w, resp.Body)
		if err := resp.Body.Close(); err != nil {
			ctx.Warnf("Can't close response body %v", err)
		}
		ctx.Logf("Copied %v bytes to client error=%v", nr, err)
	}
}

// New proxy server, logs to StdErr by default
func NewProxyHttpServer() *ProxyHttpServer {
	proxy := ProxyHttpServer{
		Logger:        log.New(os.Stderr, "", log.LstdFlags),
		reqHandlers:   []ReqHandler{},
		respHandlers:  []RespHandler{},
		httpsHandlers: []HttpsHandler{},
		NonproxyHandler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			http.Error(w, "This is a proxy server. Does not respond to non-proxy requests.", 500)
		}),
		Tr: &http.Transport{TLSClientConfig: tlsClientSkipVerify,
			Proxy: http.ProxyFromEnvironment},
	}
	proxy.ConnectDial = dialerFromEnv(&proxy)
	return &proxy
}
<|endoftext|>


<commit_before>package compress

import (
	"bytes"
	"compress/flate"
	"io"
)

// Compress returns a compressed byte slice.
func Compress(src []byte) []byte {
	compressedData := new(bytes.Buffer)
	compress(src, compressedData, 9)
	return compressedData.Bytes()
}

// Decompress returns a decompressed byte slice.
func Decompress(src []byte) []byte {
	compressedData := bytes.NewBuffer(src)
	deCompressedData := new(bytes.Buffer)
	decompress(compressedData, deCompressedData)
	return deCompressedData.Bytes()
}

// compress uses flate to compress a byte slice to a corresponding level
func compress(src []byte, dest io.Writer, level int) {
	compressor, _ := flate.NewWriter(dest, level)
	compressor.Write(src)
	compressor.Close()
}

// compress uses flate to decompress an io.Reader
func decompress(src io.Reader, dest io.Writer) {
	decompressor := flate.NewReader(src)
	io.Copy(dest, decompressor)
	decompressor.Close()
}
<commit_msg>Improve compression speed<commit_after>package compress

import (
	"bytes"
	"compress/flate"
	"io"
)

// Compress returns a compressed byte slice.
func Compress(src []byte) []byte {
	compressedData := new(bytes.Buffer)
	compress(src, compressedData, -2)
	return compressedData.Bytes()
}

// Decompress returns a decompressed byte slice.
func Decompress(src []byte) []byte {
	compressedData := bytes.NewBuffer(src)
	deCompressedData := new(bytes.Buffer)
	decompress(compressedData, deCompressedData)
	return deCompressedData.Bytes()
}

// compress uses flate to compress a byte slice to a corresponding level
func compress(src []byte, dest io.Writer, level int) {
	compressor, _ := flate.NewWriter(dest, level)
	compressor.Write(src)
	compressor.Close()
}

// compress uses flate to decompress an io.Reader
func decompress(src io.Reader, dest io.Writer) {
	decompressor := flate.NewReader(src)
	io.Copy(dest, decompressor)
	decompressor.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"gopkg.in/yaml.v2"
	"strings"
	"os/exec"
    "os"
)

const version = "0.1.7"

func check(e error){
	if e != nil {
		panic(e)
	}
}

type CodeCi struct {
	Os string `yaml:"os"`
	Language string `yaml:"language"`
    Image string `yaml:"image"`
	Script []string `yaml:"script"`
}

func dockerfileName() string {
    return "Dockerfile.codeci"
}

func dockercomposeName() string {
    return "docker-compose.codeci.yml"
}

func onlytestName() string {
    return "onlytest.codeci.sh"
}

func testName() string {
    return "test.codeci.sh"
}

func createTestScript(codeci CodeCi) string {
    jobInfo := []string{"echo 'Job Node Info: '", "echo \n", "echo 'uname -a'", "uname -a", "echo \n", "echo 'df -h'", "df -h", "echo \n", "echo 'free -m'", "free -m", "echo \n", "echo 'bash --version'", "bash --version", "echo \n", "echo 'lscpu'", "lscpu", "echo \n", "echo 'lsb_release -a'", "lsb_release -a", "echo \n", "echo 'service --status-all'", "service --status-all", "echo \n", "echo 'dpkg -l'", "dpkg -l", "echo \n", "echo \n"}
    s := []string{"#!/bin/bash", "\n", "\n", strings.Join(jobInfo, "\n") , "\n", "echo 'running your commands: '", "\n", strings.Join(codeci.Script, " && "), "\n"}
    return strings.Join(s, "")
}

func createDockerFile(codeci CodeCi) string{
    if codeci.Image != "" {
        s := []string{"FROM ", codeci.Image, "\n", "ADD . /app\nWORKDIR /app\nCMD [\"bash\", \"", testName(),"\"]", "\n"}
        return strings.Join(s, "")
    }
    if codeci.Language == "none" {
        s := []string{"FROM therickys93/", codeci.Os, "\n", "ADD . /app\nWORKDIR /app\nCMD [\"bash\", \"", testName(),"\"]", "\n"}
        return strings.Join(s, "")
    } else {
        s := []string{"FROM therickys93/", codeci.Os, codeci.Language, "\n", "ADD . /app\nWORKDIR /app\nCMD [\"bash\", \"", testName(),"\"]", "\n"}
        return strings.Join(s, "")
    }
}

func codeCIWhalesay() string {
    return "image: docker/whalesay\nscript:\n   - cowsay Hello CodeCI!"
}

func main() {
    filename := "codeci.yml"
    data := []byte{}
    if len(os.Args) > 1 {
        if os.Args[1] == "--version" {
            fmt.Printf("%s version: %s\n", os.Args[0], version)
            os.Exit(0)
        } else if os.Args[1] == "--help" {
            fmt.Printf("usage: %s --> runs the build and search for the codeci.yml\n", os.Args[0])
            fmt.Printf("usage: %s --version --> show the current version\n", os.Args[0])
            fmt.Printf("usage: %s -f codeci.whateveryouwant.yml --> specify the name of your codeci file\n", os.Args[0])
            os.Exit(0)
        } else if os.Args[1] == "-f" {
            if strings.HasPrefix(os.Args[2], "codeci") && strings.HasSuffix(os.Args[2], ".yml") {
                filename = os.Args[2]
            } else {
                os.Exit(1)
            }
        } else if os.Args[1] == "test" {
            data = []byte(codeCIWhalesay())
        } else {
            os.Exit(1)
        }
    }
    filenames := []string{"./", filename}
    var err error
    if strings.EqualFold(string(data), "") {
        data, err = ioutil.ReadFile(strings.Join(filenames, ""))
        check(err)
    }
    fmt.Printf("reading the provided codeci.yml file...\n\n")
	fmt.Print(string(data))
    fmt.Printf("\n\n")
	var codeci CodeCi

	err = yaml.Unmarshal([]byte(string(data)), &codeci)
	check(err)

    fmt.Printf("Creating temp files...\n")
	// create the test.sh file
	d1 := []byte(createTestScript(codeci))
    err = ioutil.WriteFile(testName(), d1, 0644)
    check(err)

    // create the Dockerfile
    d1 = []byte(createDockerFile(codeci))
    err = ioutil.WriteFile(dockerfileName(), d1, 0644)
    check(err)

    // create the docker-compose.yml file
    s := []string{"sut:\n", "  build: .\n", "  dockerfile: ", dockerfileName(), "\n"}
    d1 = []byte(strings.Join(s, ""))
    err = ioutil.WriteFile(dockercomposeName(), d1, 0644)
    check(err)

    // create the onlytest.sh file
    s = []string{"#!/bin/bash", "\n", "\n", "docker-compose -f ", dockercomposeName() ," -p ci build", "\n", "echo running the script...", "\n", "echo -e '\n'", "\n", "docker-compose -f ", dockercomposeName(), " -p ci up -d", "\n", "docker logs -f ci_sut_1", "\n", "echo -e '\n'", "\n","echo 'BUILD EXIT CODE:'",  "\n", "docker wait ci_sut_1", "\n", "if [ $(docker wait ci_sut_1) == 0 ]; then echo -e '\nBUILD SUCCESS\n'; else echo -e '\nBUILD FAILED\n'; fi", "\n", "docker-compose -f ", dockercomposeName(), " -p ci kill", "\n", "docker rm ci_sut_1", "\n", "docker rmi ci_sut"}
    d1 = []byte(strings.Join(s, ""))
    err = ioutil.WriteFile(onlytestName(), d1, 0644)
    check(err)

    // run the script onlytest.sh
    fmt.Print("run the build...\n")
    cmd := exec.Command("/bin/bash", onlytestName())
    cmd.Stdout = os.Stdout
    cmd.Stderr = os.Stderr
    err = cmd.Start()
    if err != nil {
        fmt.Fprintln(os.Stderr, "error starting command", err)
        return
    }
    err = cmd.Wait()
    if err != nil {
        fmt.Fprintln(os.Stderr, "error waiting command", err)
        return
    }

    // remove all the files
    fmt.Print("removing the temp files...\n")
    // remove test file
    os.Remove(testName())
    // remove Dockerfile
    os.Remove(dockerfileName())
    // remove only test file
    os.Remove(onlytestName())
    // remove docker compose file
    os.Remove(dockercomposeName())

    fmt.Print("done!\n")
}<commit_msg>new version<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"gopkg.in/yaml.v2"
	"strings"
	"os/exec"
    "os"
)

const version = "0.1.8"

func check(e error){
	if e != nil {
		panic(e)
	}
}

type CodeCi struct {
	Os string `yaml:"os"`
	Language string `yaml:"language"`
    Image string `yaml:"image"`
	Script []string `yaml:"script"`
}

func dockerfileName() string {
    return "Dockerfile.codeci"
}

func dockercomposeName() string {
    return "docker-compose.codeci.yml"
}

func onlytestName() string {
    return "onlytest.codeci.sh"
}

func testName() string {
    return "test.codeci.sh"
}

func createTestScript(codeci CodeCi) string {
    jobInfo := []string{"echo 'Job Node Info: '", "echo \n", "echo 'uname -a'", "uname -a", "echo \n", "echo 'df -h'", "df -h", "echo \n", "echo 'free -m'", "free -m", "echo \n", "echo 'bash --version'", "bash --version", "echo \n", "echo 'lscpu'", "lscpu", "echo \n", "echo 'lsb_release -a'", "lsb_release -a", "echo \n", "echo 'service --status-all'", "service --status-all", "echo \n", "echo 'dpkg -l'", "dpkg -l", "echo \n", "echo \n"}
    s := []string{"#!/bin/bash", "\n", "\n", strings.Join(jobInfo, "\n") , "\n", "echo 'running your commands: '", "\n", strings.Join(codeci.Script, " && "), "\n"}
    return strings.Join(s, "")
}

func createDockerFile(codeci CodeCi) string{
    if codeci.Image != "" {
        s := []string{"FROM ", codeci.Image, "\n", "ADD . /app\nWORKDIR /app\nCMD [\"bash\", \"", testName(),"\"]", "\n"}
        return strings.Join(s, "")
    }
    if codeci.Language == "none" {
        s := []string{"FROM therickys93/", codeci.Os, "\n", "ADD . /app\nWORKDIR /app\nCMD [\"bash\", \"", testName(),"\"]", "\n"}
        return strings.Join(s, "")
    } else {
        s := []string{"FROM therickys93/", codeci.Os, codeci.Language, "\n", "ADD . /app\nWORKDIR /app\nCMD [\"bash\", \"", testName(),"\"]", "\n"}
        return strings.Join(s, "")
    }
}

func codeCIWhalesay() string {
    return "image: docker/whalesay\nscript:\n   - cowsay Hello CodeCI!"
}

func main() {
    filename := "codeci.yml"
    data := []byte{}
    if len(os.Args) > 1 {
        if os.Args[1] == "--version" {
            fmt.Printf("%s version: %s\n", os.Args[0], version)
            os.Exit(0)
        } else if os.Args[1] == "--help" {
            fmt.Printf("usage: %s --> runs the build and search for the codeci.yml\n", os.Args[0])
            fmt.Printf("usage: %s --version --> show the current version\n", os.Args[0])
            fmt.Printf("usage: %s -f codeci.whateveryouwant.yml --> specify the name of your codeci file\n", os.Args[0])
            os.Exit(0)
        } else if os.Args[1] == "-f" {
            if strings.HasPrefix(os.Args[2], "codeci") && strings.HasSuffix(os.Args[2], ".yml") {
                filename = os.Args[2]
            } else {
                os.Exit(1)
            }
        } else if os.Args[1] == "test" {
            data = []byte(codeCIWhalesay())
        } else {
            os.Exit(1)
        }
    }
    filenames := []string{"./", filename}
    var err error
    if strings.EqualFold(string(data), "") {
        data, err = ioutil.ReadFile(strings.Join(filenames, ""))
        check(err)
    }
    fmt.Printf("reading the provided codeci.yml file...\n\n")
	fmt.Print(string(data))
    fmt.Printf("\n\n")
	var codeci CodeCi

	err = yaml.Unmarshal([]byte(string(data)), &codeci)
	check(err)

    fmt.Printf("Creating temp files...\n")
	// create the test.sh file
	d1 := []byte(createTestScript(codeci))
    err = ioutil.WriteFile(testName(), d1, 0644)
    check(err)

    // create the Dockerfile
    d1 = []byte(createDockerFile(codeci))
    err = ioutil.WriteFile(dockerfileName(), d1, 0644)
    check(err)

    // create the docker-compose.yml file
    s := []string{"sut:\n", "  build: .\n", "  dockerfile: ", dockerfileName(), "\n"}
    d1 = []byte(strings.Join(s, ""))
    err = ioutil.WriteFile(dockercomposeName(), d1, 0644)
    check(err)

    // create the onlytest.sh file
    s = []string{"#!/bin/bash", "\n", "\n", "docker-compose -f ", dockercomposeName() ," -p ci build", "\n", "echo running the script...", "\n", "echo -e '\n'", "\n", "docker-compose -f ", dockercomposeName(), " -p ci up -d", "\n", "docker logs -f ci_sut_1", "\n", "echo -e '\n'", "\n","echo 'BUILD EXIT CODE:'",  "\n", "docker wait ci_sut_1", "\n", "if [ $(docker wait ci_sut_1) == 0 ]; then echo -e '\nBUILD SUCCESS\n'; else echo -e '\nBUILD FAILED\n'; fi", "\n", "docker-compose -f ", dockercomposeName(), " -p ci kill", "\n", "docker rm ci_sut_1", "\n", "docker rmi ci_sut"}
    d1 = []byte(strings.Join(s, ""))
    err = ioutil.WriteFile(onlytestName(), d1, 0644)
    check(err)

    // run the script onlytest.sh
    fmt.Print("run the build...\n")
    cmd := exec.Command("/bin/bash", onlytestName())
    cmd.Stdout = os.Stdout
    cmd.Stderr = os.Stderr
    err = cmd.Start()
    if err != nil {
        fmt.Fprintln(os.Stderr, "error starting command", err)
        return
    }
    err = cmd.Wait()
    if err != nil {
        fmt.Fprintln(os.Stderr, "error waiting command", err)
        return
    }

    // remove all the files
    fmt.Print("removing the temp files...\n")
    // remove test file
    os.Remove(testName())
    // remove Dockerfile
    os.Remove(dockerfileName())
    // remove only test file
    os.Remove(onlytestName())
    // remove docker compose file
    os.Remove(dockercomposeName())

    fmt.Print("done!\n")
}<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package docker

import (
	"fmt"
	"strings"
)

// PortAssignment describes a port mapping between the host
// and the container.
type PortAssignment struct {
	// External is the port on the host.
	External int
	// Internal is the port on the container.
	Internal int
	// Protocol is the network protocol for the mapping (e.g. tcp, udp).
	Protocol string
}

// String returns a docker-friendly string representation of the mapping.
func (pa PortAssignment) String() string {
	return fmt.Sprintf("%d:%d/%s", pa.External, pa.Internal, pa.Protocol)
}

// MountAssignment describes a volume mount mapping between the host
// and the container.
type MountAssignment struct {
	// External is the volume mount point on the host.
	External string
	// Internal is the volume mount point on the container.
	Internal string
	// Mode is the docker-recognized access mode (e.g. rw, ro).
	Mode string
}

// String returns a docker-friendly string representation of the mapping.
func (ma MountAssignment) String() string {
	return fmt.Sprintf("%s:%s:%s", ma.External, ma.Internal, ma.Mode)
}

// RunArgs contains the data passed to the Run function.
type RunArgs struct {
	// Name is the unique name to assign to the container (optional).
	Name string
	// Image is the container image to use.
	Image string
	// Command is the command to run in the container (optional).
	Command string
	// EnvVars are environment variables to use in the container,
	// if any.
	EnvVars map[string]string
	// Ports is the list of ports to map into the container from
	// the host, if any.
	Ports []PortAssignment
	// Mounts is the list of volumes to map into the container from
	// the host, if any.
	Mounts []MountAssignment
}

// CommandlineArgs converts the RunArgs into a list of strings that may
// be passed to exec.Command as the command args.
func (ra RunArgs) CommandlineArgs() []string {
	args := []string{
		"--detach",
		"--name", ra.Name,
	}

	for k, v := range ra.EnvVars {
		args = append(args, "-e", k+"="+v)
	}

	for _, p := range ra.Ports {
		args = append(args, "-p", p.String())
	}

	for _, m := range ra.Mounts {
		args = append(args, "-v", m.String())
	}

	// Image and Command must come after all options.
	args = append(args, ra.Image)

	if ra.Command != "" {
		args = append(args, strings.Fields(ra.Command)...)
	}

	return args
}
<commit_msg>Clean up doc comments.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package docker

import (
	"fmt"
	"strings"
)

// PortAssignment describes a port mapping between the host
// and the container.
type PortAssignment struct {
	// External is the port on the host.
	External int
	// Internal is the port on the container.
	Internal int
	// Protocol is the network protocol for the mapping (e.g. tcp, udp).
	Protocol string
}

// String returns a docker-friendly string representation of the mapping.
func (pa PortAssignment) String() string {
	return fmt.Sprintf("%d:%d/%s", pa.External, pa.Internal, pa.Protocol)
}

// MountAssignment describes a volume mount mapping between the host
// and the container.
type MountAssignment struct {
	// External is the volume mount point on the host.
	External string
	// Internal is the volume mount point on the container.
	Internal string
	// Mode is the docker-recognized access mode (e.g. rw, ro).
	Mode string
}

// String returns a docker-friendly string representation of the mapping.
func (ma MountAssignment) String() string {
	return fmt.Sprintf("%s:%s:%s", ma.External, ma.Internal, ma.Mode)
}

// RunArgs contains the data passed to the Run function.
type RunArgs struct {
	// Name is the unique name to assign to the container (optional).
	Name string
	// Image is the container image to use.
	Image string
	// Command is the command to run in the container (optional).
	Command string
	// EnvVars holds the environment variables to use in the container,
	// if any.
	EnvVars map[string]string
	// Ports holds the ports info to map into the container from the
	// host, if any.
	Ports []PortAssignment
	// Mounts holds the volumes info to map into the container from the
	// host, if any.
	Mounts []MountAssignment
}

// CommandlineArgs converts the RunArgs into a list of strings that may
// be passed to exec.Command as the command args.
func (ra RunArgs) CommandlineArgs() []string {
	args := []string{
		"--detach",
		"--name", ra.Name,
	}

	for k, v := range ra.EnvVars {
		args = append(args, "-e", k+"="+v)
	}

	for _, p := range ra.Ports {
		args = append(args, "-p", p.String())
	}

	for _, m := range ra.Mounts {
		args = append(args, "-v", m.String())
	}

	// Image and Command must come after all options.
	args = append(args, ra.Image)

	if ra.Command != "" {
		args = append(args, strings.Fields(ra.Command)...)
	}

	return args
}
<|endoftext|>


<commit_before>package adserver

import (
	"io"
	"os"
	"fmt"
	"strings"
	"bufio"
	"strconv"
	"sort"
	"github.com/wenweihu86/ad-server/utils"
	"time"
)

type GeoLocationInfo struct {
	LocId uint64
	Country string
	City string
}

type LocationInfo struct {
	BeginIp uint32
	EndIp uint32
	Country string
	City string
}

type IpDataInfo struct {
	ipPairs utils.IpPairs
	ipLocationMap map[utils.IpPair]*LocationInfo
}

type IpDict struct {
	IpDataArray []*IpDataInfo
	CurrentIndex uint32
	BlockLastModifiedTime int64
	LocationLastModifiedTime int64
}

var LocationDict *IpDict

func init() {
	LocationDict = &IpDict{
		IpDataArray: make([]*IpDataInfo, 2, 2),
		CurrentIndex: 0,
		BlockLastModifiedTime: 0,
		LocationLastModifiedTime: 0,
	}
	for i := 0; i < 2; i++ {
		LocationDict.IpDataArray[i] = NewIpDataInfo()
	}
}

// 初始化之后首次加载Ip字典信息
func (ipDict *IpDict) Load() error {
	ipDataInfo,err := LoadLocationDict(GlobalConfObject.GeoBlockFileName,
		GlobalConfObject.GeoLocationFileName)
	if err != nil {
		return err
	}
	ipDict.IpDataArray[ipDict.CurrentIndex] = ipDataInfo
	blockFileStat, _ := os.Stat(GlobalConfObject.GeoBlockFileName)
    locationFileStat, _ := os.Stat(GlobalConfObject.GeoLocationFileName)
	ipDict.BlockLastModifiedTime = blockFileStat.ModTime().Unix()
	ipDict.LocationLastModifiedTime = locationFileStat.ModTime().Unix()
	return nil
}

func NewIpDataInfo() *IpDataInfo {
	return &IpDataInfo{
		ipPairs: make(utils.IpPairs, 0),
		ipLocationMap: make(map[utils.IpPair]*LocationInfo),
	}
}

func LoadLocationDict(blockFileName, locationFileName string) (*IpDataInfo, error) {
	dictFile, err := os.Open(blockFileName)
	if err != nil {
		AdServerLog.Error(fmt.Sprintf("open file error, name=%s\n", blockFileName))
		return nil, err
	}
	defer dictFile.Close()

	ipDataInfo := NewIpDataInfo()
	geoLocationMap, err := loadGeoLocation(locationFileName)
	if err != nil {
		return nil, err
	}
	br := bufio.NewReader(dictFile)
	for {
		line, _, err := br.ReadLine()
		if err == io.EOF {
			break
		}
		lineString := string(line)
		if len(lineString) == 0 || lineString[0] == '#' {
			continue
		}
		lines := strings.Split(lineString, ",")
		if len(lines) != 3 {
			AdServerLog.Warn(fmt.Sprintf(
				"invalid format, blockFileName=%s, line=%s\n",
				blockFileName, lineString))
			continue
		}

		tmpInt, _ := strconv.ParseUint(strings.Trim(lines[0], "\""), 10 ,32)
		beginIp := uint32(tmpInt)
		tmpInt, _ = strconv.ParseUint(strings.Trim(lines[1], "\""), 10 ,32)
		endIp := uint32(tmpInt)
		locId, _ := strconv.ParseUint(strings.Trim(lines[2], "\""), 10 ,64)
		geoLocationInfo, exist := geoLocationMap[locId]
		if !exist {
			//fmt.Printf("geoLocationInfo not found, locId=%d\n", locId)
			continue
		}
		locationInfo := &LocationInfo{
			BeginIp: beginIp,
			EndIp: endIp,
			Country: geoLocationInfo.Country,
			City: geoLocationInfo.City,
		}
		ipPair := utils.IpPair{
			BeginIp: beginIp,
			EndIp: endIp,
		}
		ipDataInfo.ipPairs = append(ipDataInfo.ipPairs, ipPair)
		ipDataInfo.ipLocationMap[ipPair] = locationInfo
	}

	sort.Sort(ipDataInfo.ipPairs)
	AdServerLog.Info(fmt.Sprintf(
		"read dict success, blockFileName=%s locationFileName=%s\n",
		blockFileName, locationFileName))
	AdServerLog.Info(fmt.Sprintf(
		"location dict size=%d\n", len(ipDataInfo.ipPairs)))

	return ipDataInfo, nil
}

// 启动定时器，用于定期重新加载Ip字典信息
func (locationDict *IpDict) StartReloadTimer() {
	duration := int64(time.Second) * GlobalConfObject.IpFileReloadInterval
	t := time.NewTicker(time.Duration(duration))
	go func() {
		for t1 := range t.C {
			AdServerLog.Debug("IpDict reload timer execute")
			blockFileStat, _ := os.Stat(GlobalConfObject.GeoBlockFileName)
			locationFileStat, _ := os.Stat(GlobalConfObject.GeoLocationFileName)
			blockCurrentModifiedTime := blockFileStat.ModTime().Unix()
			locationCurrentModifiedTime := locationFileStat.ModTime().Unix()
			// 如果文件有更新，则重新加载广告内容
			if blockCurrentModifiedTime > locationDict.BlockLastModifiedTime || locationCurrentModifiedTime > locationDict.LocationLastModifiedTime {
				AdServerLog.Info(fmt.Sprintf("start reload ad info dict at %s",
					t1.Format("2006-01-02 03:04:05")))
				_, err := LoadLocationDict(
					GlobalConfObject.GeoBlockFileName,
					GlobalConfObject.GeoLocationFileName)
				if err != nil {
					continue
				}
				nextIndex := 1 - locationDict.CurrentIndex
				locationDict.CurrentIndex = nextIndex
				locationDict.BlockLastModifiedTime = blockCurrentModifiedTime
				locationDict.LocationLastModifiedTime = locationCurrentModifiedTime
			}
		}
	}()
}

// 获取当前可用的Ip字典信息
func (ipDict *IpDict) GetCurrentIpData() *IpDataInfo {
	return ipDict.IpDataArray[ipDict.CurrentIndex]
}

func (ipDataInfo *IpDataInfo) SearchLocationByIp(ipString string) *LocationInfo {
	ip := utils.StringIpToUint(ipString)
	ipPairs := ipDataInfo.ipPairs
	size := len(ipPairs)
	if size == 0 || ip < ipPairs[0].BeginIp || ip > ipPairs[size - 1].EndIp {
		return nil
	}
	left := 0
	right := size - 1
	for left <= right {
		mid := (left + right) / 2
		if ip >= ipPairs[mid].BeginIp && ip <= ipPairs[mid].EndIp {
			return ipDataInfo.ipLocationMap[ipPairs[mid]];
		} else if ip < ipPairs[mid].BeginIp {
			right = mid - 1
		} else if ip > ipPairs[mid].EndIp {
			left = mid + 1
		}
	}
	return nil
}

func loadGeoLocation(fileName string) (map[uint64]*GeoLocationInfo, error) {
	dictFile, err := os.Open(fileName)
	if err != nil {
		AdServerLog.Error(fmt.Sprintf(
		   "open file error, name=%s\n", fileName))
		return nil, err
	}
	defer dictFile.Close()

	geoLocationMap := make(map[uint64]*GeoLocationInfo)
	br := bufio.NewReader(dictFile)
	lineNum := 0
	for {
		line, _, err := br.ReadLine()
		if err == io.EOF {
			break
		}
		lineString := string(line)
		if len(lineString) == 0 || lineString[0] == '#' {
			continue
		}
		lines := strings.Split(lineString, ",")
		if len(lines) != 9 {
			AdServerLog.Warn(fmt.Sprintf(
				"invalid format, file=%s, line=%s\n",
				fileName, lineString))
			continue
		}
		locId, _ := strconv.ParseUint(lines[0], 10, 64)
		geoLocationInfo := &GeoLocationInfo{
			LocId: locId,
			Country: strings.Trim(lines[1], "\""),
			City: strings.Trim(lines[3], "\""),
		}
		geoLocationMap[locId] = geoLocationInfo
		lineNum++
	}
	AdServerLog.Info(fmt.Sprintf(
		"load dict success, file=%s, lineNum=%d\n",
		fileName, lineNum))
	return geoLocationMap, nil
}
<commit_msg>fix bug: location dict reload timer<commit_after>package adserver

import (
	"io"
	"os"
	"fmt"
	"strings"
	"bufio"
	"strconv"
	"sort"
	"github.com/wenweihu86/ad-server/utils"
	"time"
)

type GeoLocationInfo struct {
	LocId uint64
	Country string
	City string
}

type LocationInfo struct {
	BeginIp uint32
	EndIp uint32
	Country string
	City string
}

type IpDataInfo struct {
	ipPairs utils.IpPairs
	ipLocationMap map[utils.IpPair]*LocationInfo
}

type IpDict struct {
	IpDataArray []*IpDataInfo
	CurrentIndex uint32
	BlockLastModifiedTime int64
	LocationLastModifiedTime int64
}

var LocationDict *IpDict

func init() {
	LocationDict = &IpDict{
		IpDataArray: make([]*IpDataInfo, 2, 2),
		CurrentIndex: 0,
		BlockLastModifiedTime: 0,
		LocationLastModifiedTime: 0,
	}
	for i := 0; i < 2; i++ {
		LocationDict.IpDataArray[i] = NewIpDataInfo()
	}
}

// 初始化之后首次加载Ip字典信息
func (ipDict *IpDict) Load() error {
	ipDataInfo,err := LoadLocationDict(GlobalConfObject.GeoBlockFileName,
		GlobalConfObject.GeoLocationFileName)
	if err != nil {
		return err
	}
	ipDict.IpDataArray[ipDict.CurrentIndex] = ipDataInfo
	blockFileStat, _ := os.Stat(GlobalConfObject.GeoBlockFileName)
    locationFileStat, _ := os.Stat(GlobalConfObject.GeoLocationFileName)
	ipDict.BlockLastModifiedTime = blockFileStat.ModTime().Unix()
	ipDict.LocationLastModifiedTime = locationFileStat.ModTime().Unix()
	return nil
}

func NewIpDataInfo() *IpDataInfo {
	return &IpDataInfo{
		ipPairs: make(utils.IpPairs, 0),
		ipLocationMap: make(map[utils.IpPair]*LocationInfo),
	}
}

func LoadLocationDict(blockFileName, locationFileName string) (*IpDataInfo, error) {
	dictFile, err := os.Open(blockFileName)
	if err != nil {
		AdServerLog.Error(fmt.Sprintf("open file error, name=%s\n", blockFileName))
		return nil, err
	}
	defer dictFile.Close()

	ipDataInfo := NewIpDataInfo()
	geoLocationMap, err := loadGeoLocation(locationFileName)
	if err != nil {
		return nil, err
	}
	br := bufio.NewReader(dictFile)
	for {
		line, _, err := br.ReadLine()
		if err == io.EOF {
			break
		}
		lineString := string(line)
		if len(lineString) == 0 || lineString[0] == '#' {
			continue
		}
		lines := strings.Split(lineString, ",")
		if len(lines) != 3 {
			AdServerLog.Warn(fmt.Sprintf(
				"invalid format, blockFileName=%s, line=%s\n",
				blockFileName, lineString))
			continue
		}

		tmpInt, _ := strconv.ParseUint(strings.Trim(lines[0], "\""), 10 ,32)
		beginIp := uint32(tmpInt)
		tmpInt, _ = strconv.ParseUint(strings.Trim(lines[1], "\""), 10 ,32)
		endIp := uint32(tmpInt)
		locId, _ := strconv.ParseUint(strings.Trim(lines[2], "\""), 10 ,64)
		geoLocationInfo, exist := geoLocationMap[locId]
		if !exist {
			//fmt.Printf("geoLocationInfo not found, locId=%d\n", locId)
			continue
		}
		locationInfo := &LocationInfo{
			BeginIp: beginIp,
			EndIp: endIp,
			Country: geoLocationInfo.Country,
			City: geoLocationInfo.City,
		}
		ipPair := utils.IpPair{
			BeginIp: beginIp,
			EndIp: endIp,
		}
		ipDataInfo.ipPairs = append(ipDataInfo.ipPairs, ipPair)
		ipDataInfo.ipLocationMap[ipPair] = locationInfo
	}

	sort.Sort(ipDataInfo.ipPairs)
	AdServerLog.Info(fmt.Sprintf(
		"read dict success, blockFileName=%s locationFileName=%s\n",
		blockFileName, locationFileName))
	AdServerLog.Info(fmt.Sprintf(
		"location dict size=%d\n", len(ipDataInfo.ipPairs)))

	return ipDataInfo, nil
}

// 启动定时器，用于定期重新加载Ip字典信息
func (locationDict *IpDict) StartReloadTimer() {
	duration := int64(time.Second) * GlobalConfObject.IpFileReloadInterval
	t := time.NewTicker(time.Duration(duration))
	go func() {
		for t1 := range t.C {
			AdServerLog.Debug("IpDict reload timer execute")
			blockFileStat, _ := os.Stat(GlobalConfObject.GeoBlockFileName)
			locationFileStat, _ := os.Stat(GlobalConfObject.GeoLocationFileName)
			blockCurrentModifiedTime := blockFileStat.ModTime().Unix()
			locationCurrentModifiedTime := locationFileStat.ModTime().Unix()
			// 如果文件有更新，则重新加载广告内容
			if blockCurrentModifiedTime > locationDict.BlockLastModifiedTime || locationCurrentModifiedTime > locationDict.LocationLastModifiedTime {
				AdServerLog.Info(fmt.Sprintf("start reload ad info dict at %s",
					t1.Format("2006-01-02 03:04:05")))
				ipDataInfo, err := LoadLocationDict(
					GlobalConfObject.GeoBlockFileName,
					GlobalConfObject.GeoLocationFileName)
				if err != nil {
					continue
				}
				nextIndex := 1 - locationDict.CurrentIndex
				locationDict.IpDataArray[nextIndex] = ipDataInfo
				locationDict.CurrentIndex = nextIndex
				locationDict.BlockLastModifiedTime = blockCurrentModifiedTime
				locationDict.LocationLastModifiedTime = locationCurrentModifiedTime
			}
		}
	}()
}

// 获取当前可用的Ip字典信息
func (ipDict *IpDict) GetCurrentIpData() *IpDataInfo {
	return ipDict.IpDataArray[ipDict.CurrentIndex]
}

func (ipDataInfo *IpDataInfo) SearchLocationByIp(ipString string) *LocationInfo {
	ip := utils.StringIpToUint(ipString)
	ipPairs := ipDataInfo.ipPairs
	size := len(ipPairs)
	if size == 0 || ip < ipPairs[0].BeginIp || ip > ipPairs[size - 1].EndIp {
		return nil
	}
	left := 0
	right := size - 1
	for left <= right {
		mid := (left + right) / 2
		if ip >= ipPairs[mid].BeginIp && ip <= ipPairs[mid].EndIp {
			return ipDataInfo.ipLocationMap[ipPairs[mid]];
		} else if ip < ipPairs[mid].BeginIp {
			right = mid - 1
		} else if ip > ipPairs[mid].EndIp {
			left = mid + 1
		}
	}
	return nil
}

func loadGeoLocation(fileName string) (map[uint64]*GeoLocationInfo, error) {
	dictFile, err := os.Open(fileName)
	if err != nil {
		AdServerLog.Error(fmt.Sprintf(
		   "open file error, name=%s\n", fileName))
		return nil, err
	}
	defer dictFile.Close()

	geoLocationMap := make(map[uint64]*GeoLocationInfo)
	br := bufio.NewReader(dictFile)
	lineNum := 0
	for {
		line, _, err := br.ReadLine()
		if err == io.EOF {
			break
		}
		lineString := string(line)
		if len(lineString) == 0 || lineString[0] == '#' {
			continue
		}
		lines := strings.Split(lineString, ",")
		if len(lines) != 9 {
			AdServerLog.Warn(fmt.Sprintf(
				"invalid format, file=%s, line=%s\n",
				fileName, lineString))
			continue
		}
		locId, _ := strconv.ParseUint(lines[0], 10, 64)
		geoLocationInfo := &GeoLocationInfo{
			LocId: locId,
			Country: strings.Trim(lines[1], "\""),
			City: strings.Trim(lines[3], "\""),
		}
		geoLocationMap[locId] = geoLocationInfo
		lineNum++
	}
	AdServerLog.Info(fmt.Sprintf(
		"load dict success, file=%s, lineNum=%d\n",
		fileName, lineNum))
	return geoLocationMap, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"

	"flag"
	"github.com/br0r/hydra/config"
)

const BASE_DIR string = ".hydra"
const help string = `Usage:
hydra [OPTIONS] COMMAND

Commands:
  init - Create hydra project [--clean]
  start - Start hydra servers
  stop - Stops started servers
  ls - Show started servers
  logs [name] - Show logs for servers, or for specific server given by name
`

func initialize(c config.Config, clean bool) {
	if clean {
		_, e := exec.Command("rm", "-rf", ".hydra").Output()
		if e != nil {
			log.Fatal(e)
		}
	}

	exec.Command("mkdir", ".hydra").Output()

	for i := 0; i < len(c.Services); i += 1 {
		service := c.Services[i]
		p := service.Path
		name := service.Name
		base := path.Join(".hydra/", name)
		if _, err := os.Stat(base); os.IsNotExist(err) {
			fmt.Println("Creating", name)

			var e error
			if strings.Index(p, "git@") == 0 {
				e = exec.Command("git", "clone", p, base).Run()
			} else {
				e = exec.Command("cp", "-R", p, base).Run()
			}

			if e != nil {
				log.Fatal(e)
			}

			git := path.Join(base, ".git")
			exec.Command("rm", "-rf", git).Run()

			if service.Config.Dest != "" && service.Config.Src != "" {
				u := path.Join(base, service.Config.Dest)
				cf := service.Config.Src

				err := exec.Command("cp", cf, u).Run()
				if err != nil {
					log.Fatal(err)
				}
			}

			if service.Install != "" {
				args := strings.Split(service.Install, " ")
				cmd := exec.Command(args[0], args[1:]...)

				cmd.Dir = base
				e := cmd.Run()
				if e != nil {
					log.Fatal(e)
				}
			}
		}
	}
}

func start(conf config.Config) {
	for i := 0; i < len(conf.Services); i += 1 {
		service := conf.Services[i]
		name := service.Name

		cmdRunDir := path.Join(".hydra", name)
		pid_file := path.Join(cmdRunDir, "pid")
		log_file_path := path.Join(cmdRunDir, "log")

		if _, err := os.Stat(pid_file); !os.IsNotExist(err) {
			fmt.Printf("%s is already started, run hydra stop if you want to restart\n", name)
			continue
		}

		env := os.Environ()
		for k, v := range service.Env {
			env = append(env, fmt.Sprintf("%s=%s", k, v))
		}

		fmt.Println("Starting", name)
		args := strings.Split(conf.Services[i].Start, " ")
		args = append(args, "2>&1 1> log")
		cmd := exec.Command(args[0], args[1:]...)
		cmd.Env = env
		cmd.Dir = cmdRunDir

		log_file, e := os.Create(log_file_path)
		if e != nil {
			log.Fatal(e)
		}

		_, e = log_file.WriteString(fmt.Sprintf("%s:\n", name))
		if e != nil {
			log.Fatal(e)
		}

		defer log_file.Close()
		cmd.Stdout = log_file
		cmd.Stderr = log_file

		e = cmd.Start()
		pid := []byte(strconv.Itoa(cmd.Process.Pid))
		ioutil.WriteFile(pid_file, pid, 0440)

		if e != nil {
			os.Remove("pid")
			log.Fatal("Error with", name, e)
		}

	}
}

func kill(conf config.Config) {
	for i := 0; i < len(conf.Services); i += 1 {
		service := conf.Services[i]
		name := service.Name
		pid_file := path.Join(".hydra", name, "pid")
		if _, err := os.Stat(pid_file); !os.IsNotExist(err) {
			bytes, e := ioutil.ReadFile(pid_file)
			if e != nil {
				log.Fatalf("Error when reading file %v", e)
			}
			pid := string(bytes)
			exec.Command("kill", pid).Run()
			if e == nil {
				fmt.Printf("Killed %s\n", name)
			}
			os.Remove(pid_file)
		}
	}
}

func ls(conf config.Config) {
	for _, service := range conf.Services {
		name := service.Name
		pid_path := path.Join(BASE_DIR, name, "pid")
		if _, err := os.Stat(pid_path); !os.IsNotExist(err) {
			bytes, e := ioutil.ReadFile(pid_path)
			if e != nil {
				log.Fatal(e)
			}
			pid := string(bytes)
			fmt.Printf("%s running on pid: %s\n", name, pid)
		}
	}
}

func logs(conf config.Config, servers []string) {
	for _, service := range conf.Services {
		if len(servers) > 0 {
			err := false
			for _, name := range servers {
				if name != service.Name {
					err = true
					break
				}
			}
			if err == true {
				continue
			}
		}

		logfile := path.Join(".hydra/", service.Name, "log")
		data, err := ioutil.ReadFile(logfile)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(string(data))
	}
}

func main() {
	var c config.Config = config.ReadConfig()
	var clean = flag.Bool("clean", false, "If we want a clean run")
	flag.Parse()

	if cmd := flag.Arg(0); cmd != "" {
		switch cmd {
		case "init":
			initialize(c, *clean)
		case "start":
			start(c)
		case "stop":
			kill(c)
		case "ls":
			ls(c)
		case "logs":
			logs(c, flag.Args()[1:])
		default:
			fmt.Println(help)
		}
	} else {
		fmt.Println(help)
	}
}
<commit_msg>added msg for not running services<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"

	"flag"
	"github.com/br0r/hydra/config"
)

const BASE_DIR string = ".hydra"
const help string = `Usage:
hydra [OPTIONS] COMMAND

Commands:
  init - Create hydra project [--clean]
  start - Start hydra servers
  stop - Stops started servers
  ls - Show started servers
  logs [name] - Show logs for servers, or for specific server given by name
`

func initialize(c config.Config, clean bool) {
	if clean {
		_, e := exec.Command("rm", "-rf", ".hydra").Output()
		if e != nil {
			log.Fatal(e)
		}
	}

	exec.Command("mkdir", ".hydra").Output()

	for i := 0; i < len(c.Services); i += 1 {
		service := c.Services[i]
		p := service.Path
		name := service.Name
		base := path.Join(".hydra/", name)
		if _, err := os.Stat(base); os.IsNotExist(err) {
			fmt.Println("Creating", name)

			var e error
			if strings.Index(p, "git@") == 0 {
				e = exec.Command("git", "clone", p, base).Run()
			} else {
				e = exec.Command("cp", "-R", p, base).Run()
			}

			if e != nil {
				log.Fatal(e)
			}

			git := path.Join(base, ".git")
			exec.Command("rm", "-rf", git).Run()

			if service.Config.Dest != "" && service.Config.Src != "" {
				u := path.Join(base, service.Config.Dest)
				cf := service.Config.Src

				err := exec.Command("cp", cf, u).Run()
				if err != nil {
					log.Fatal(err)
				}
			}

			if service.Install != "" {
				args := strings.Split(service.Install, " ")
				cmd := exec.Command(args[0], args[1:]...)

				cmd.Dir = base
				e := cmd.Run()
				if e != nil {
					log.Fatal(e)
				}
			}
		}
	}
}

func start(conf config.Config) {
	for i := 0; i < len(conf.Services); i += 1 {
		service := conf.Services[i]
		name := service.Name

		cmdRunDir := path.Join(".hydra", name)
		pid_file := path.Join(cmdRunDir, "pid")
		log_file_path := path.Join(cmdRunDir, "log")

		if _, err := os.Stat(pid_file); !os.IsNotExist(err) {
			fmt.Printf("%s is already started, run hydra stop if you want to restart\n", name)
			continue
		}

		env := os.Environ()
		for k, v := range service.Env {
			env = append(env, fmt.Sprintf("%s=%s", k, v))
		}

		fmt.Println("Starting", name)
		args := strings.Split(conf.Services[i].Start, " ")
		args = append(args, "2>&1 1> log")
		cmd := exec.Command(args[0], args[1:]...)
		cmd.Env = env
		cmd.Dir = cmdRunDir

		log_file, e := os.Create(log_file_path)
		if e != nil {
			log.Fatal(e)
		}

		_, e = log_file.WriteString(fmt.Sprintf("%s:\n", name))
		if e != nil {
			log.Fatal(e)
		}

		defer log_file.Close()
		cmd.Stdout = log_file
		cmd.Stderr = log_file

		e = cmd.Start()
		pid := []byte(strconv.Itoa(cmd.Process.Pid))
		ioutil.WriteFile(pid_file, pid, 0440)

		if e != nil {
			os.Remove("pid")
			log.Fatal("Error with", name, e)
		}

	}
}

func kill(conf config.Config) {
	for i := 0; i < len(conf.Services); i += 1 {
		service := conf.Services[i]
		name := service.Name
		pid_file := path.Join(".hydra", name, "pid")
		if _, err := os.Stat(pid_file); !os.IsNotExist(err) {
			bytes, e := ioutil.ReadFile(pid_file)
			if e != nil {
				log.Fatalf("Error when reading file %v", e)
			}
			pid := string(bytes)
			exec.Command("kill", pid).Run()
			if e == nil {
				fmt.Printf("Killed %s\n", name)
			}
			os.Remove(pid_file)
		}
	}
}

func ls(conf config.Config) {
	for _, service := range conf.Services {
		name := service.Name
		pid_path := path.Join(BASE_DIR, name, "pid")
		if _, err := os.Stat(pid_path); !os.IsNotExist(err) {
			bytes, e := ioutil.ReadFile(pid_path)
			if e != nil {
				log.Fatal(e)
			}
			pid := string(bytes)
			fmt.Printf("%s running on pid: %s\n", name, pid)
		} else {
			fmt.Printf("%s is not running\n", name)
		}
	}
}

func logs(conf config.Config, servers []string) {
	for _, service := range conf.Services {
		if len(servers) > 0 {
			err := false
			for _, name := range servers {
				if name != service.Name {
					err = true
					break
				}
			}
			if err == true {
				continue
			}
		}

		logfile := path.Join(".hydra/", service.Name, "log")
		data, err := ioutil.ReadFile(logfile)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(string(data))
	}
}

func main() {
	var c config.Config = config.ReadConfig()
	var clean = flag.Bool("clean", false, "If we want a clean run")
	flag.Parse()

	if cmd := flag.Arg(0); cmd != "" {
		switch cmd {
		case "init":
			initialize(c, *clean)
		case "start":
			start(c)
		case "stop":
			kill(c)
		case "ls":
			ls(c)
		case "logs":
			logs(c, flag.Args()[1:])
		default:
			fmt.Println(help)
		}
	} else {
		fmt.Println(help)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"

	"gopkg.in/yaml.v2"
)

type Icons struct {
	Icons []Icon
}

type Icon struct {
	Name       string
	Id         string
	Unicode    string
	Created    string
	Filter     []string
	Categories []string
}

func NewIcons() *Icons {
	return new(Icons).init()
}

func (ics *Icons) init() *Icons {
	path := os.Getenv("FAW_ICONS_YAML_PATH") // for testing
	if path == "" {
		path = "icons.yml" // default
	}

	b, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err) // FIXME
	}

	err = yaml.Unmarshal([]byte(b), &ics)
	if err != nil {
		panic(err) // FIXME
	}

	return ics
}

func (ics *Icons) Find(terms []string) []Icon {
	icons := NewIcons().Icons

	var r []Icon
	for _, icon := range icons {
		// FIXME: ContainTerms function dependency
		if ContainTerms(terms, icon.Id) {
			r = append(r, icon)
		}
	}

	return r
}
<commit_msg>Implement Icons#Sort()<commit_after>package main

import (
	"io/ioutil"
	"os"
	"sort"

	"gopkg.in/yaml.v2"
)

type IconsYaml struct {
	Icons Icons
}

type Icons []Icon

type Icon struct {
	Name       string
	Id         string
	Unicode    string
	Created    string
	Filter     []string
	Categories []string
}

func NewIcons() Icons {
	path := os.Getenv("FAW_ICONS_YAML_PATH") // for testing
	if path == "" {
		path = "icons.yml" // default
	}

	b, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err) // FIXME
	}

	var y IconsYaml
	err = yaml.Unmarshal([]byte(b), &y)
	if err != nil {
		panic(err) // FIXME
	}

	return y.Icons.Sort()
}

func (ics Icons) Find(terms []string) Icons {
	var r Icons
	for _, icon := range ics {
		// FIXME: ContainTerms function dependency
		if ContainTerms(terms, icon.Id) {
			r = append(r, icon)
		}
	}

	return r
}

// Len for sort
func (ics Icons) Len() int {
	return len(ics)
}

// Less for sort
func (ics Icons) Less(i, j int) bool {
	return ics[i].Id < ics[j].Id
}

// Swap for sort
func (ics Icons) Swap(i, j int) {
	ics[i], ics[j] = ics[j], ics[i]
}

func (ics Icons) Sort() Icons {
	sort.Sort(ics)
	return ics
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"context"
	"fmt"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/rootfs"
	digest "github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

// Image describes an image used by containers
type Image interface {
	// Name of the image
	Name() string
	// Target descriptor for the image content
	Target() ocispec.Descriptor
	// Unpack unpacks the image's content into a snapshot
	Unpack(context.Context, string) error
	// RootFS returns the unpacked diffids that make up images rootfs.
	RootFS(ctx context.Context) ([]digest.Digest, error)
	// Size returns the total size of the image's packed resources.
	Size(ctx context.Context) (int64, error)
	// Config descriptor for the image.
	Config(ctx context.Context) (ocispec.Descriptor, error)
	// IsUnpacked returns whether or not an image is unpacked.
	IsUnpacked(context.Context, string) (bool, error)
	// ContentStore provides a content store which contains image blob data
	ContentStore() content.Store
}

var _ = (Image)(&image{})

type image struct {
	client *Client

	i images.Image
}

func (i *image) Name() string {
	return i.i.Name
}

func (i *image) Target() ocispec.Descriptor {
	return i.i.Target
}

func (i *image) RootFS(ctx context.Context) ([]digest.Digest, error) {
	provider := i.client.ContentStore()
	return i.i.RootFS(ctx, provider, platforms.Default())
}

func (i *image) Size(ctx context.Context) (int64, error) {
	provider := i.client.ContentStore()
	return i.i.Size(ctx, provider, platforms.Default())
}

func (i *image) Config(ctx context.Context) (ocispec.Descriptor, error) {
	provider := i.client.ContentStore()
	return i.i.Config(ctx, provider, platforms.Default())
}

func (i *image) IsUnpacked(ctx context.Context, snapshotterName string) (bool, error) {
	sn := i.client.SnapshotService(snapshotterName)
	cs := i.client.ContentStore()

	diffs, err := i.i.RootFS(ctx, cs, platforms.Default())
	if err != nil {
		return false, err
	}

	chainID := identity.ChainID(diffs)
	_, err = sn.Stat(ctx, chainID.String())
	if err == nil {
		return true, nil
	} else if !errdefs.IsNotFound(err) {
		return false, err
	}

	return false, nil
}

func (i *image) Unpack(ctx context.Context, snapshotterName string) error {
	ctx, done, err := i.client.WithLease(ctx)
	if err != nil {
		return err
	}
	defer done(ctx)

	layers, err := i.getLayers(ctx, platforms.Default())
	if err != nil {
		return err
	}

	var (
		sn = i.client.SnapshotService(snapshotterName)
		a  = i.client.DiffService()
		cs = i.client.ContentStore()

		chain    []digest.Digest
		unpacked bool
	)
	for _, layer := range layers {
		unpacked, err = rootfs.ApplyLayer(ctx, layer, chain, sn, a)
		if err != nil {
			return err
		}

		if unpacked {
			// Set the uncompressed label after the uncompressed
			// digest has been verified through apply.
			cinfo := content.Info{
				Digest: layer.Blob.Digest,
				Labels: map[string]string{
					"containerd.io/uncompressed": layer.Diff.Digest.String(),
				},
			}
			if _, err := cs.Update(ctx, cinfo, "labels.containerd.io/uncompressed"); err != nil {
				return err
			}
		}

		chain = append(chain, layer.Diff.Digest)
	}

	if unpacked {
		desc, err := i.i.Config(ctx, cs, platforms.Default())
		if err != nil {
			return err
		}

		rootfs := identity.ChainID(chain).String()

		cinfo := content.Info{
			Digest: desc.Digest,
			Labels: map[string]string{
				fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotterName): rootfs,
			},
		}
		if _, err := cs.Update(ctx, cinfo, fmt.Sprintf("labels.containerd.io/gc.ref.snapshot.%s", snapshotterName)); err != nil {
			return err
		}
	}

	return nil
}

func (i *image) getLayers(ctx context.Context, platform string) ([]rootfs.Layer, error) {
	cs := i.client.ContentStore()

	manifest, err := images.Manifest(ctx, cs, i.i.Target, platform)
	if err != nil {
		return nil, err
	}

	diffIDs, err := i.i.RootFS(ctx, cs, platform)
	if err != nil {
		return nil, errors.Wrap(err, "failed to resolve rootfs")
	}
	if len(diffIDs) != len(manifest.Layers) {
		return nil, errors.Errorf("mismatched image rootfs and manifest layers")
	}
	layers := make([]rootfs.Layer, len(diffIDs))
	for i := range diffIDs {
		layers[i].Diff = ocispec.Descriptor{
			// TODO: derive media type from compressed type
			MediaType: ocispec.MediaTypeImageLayer,
			Digest:    diffIDs[i],
		}
		layers[i].Blob = manifest.Layers[i]
	}
	return layers, nil
}

func (i *image) ContentStore() content.Store {
	return i.client.ContentStore()
}
<commit_msg>Add NewImage to return a client Image impl<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"context"
	"fmt"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/rootfs"
	digest "github.com/opencontainers/go-digest"
	"github.com/opencontainers/image-spec/identity"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

// Image describes an image used by containers
type Image interface {
	// Name of the image
	Name() string
	// Target descriptor for the image content
	Target() ocispec.Descriptor
	// Unpack unpacks the image's content into a snapshot
	Unpack(context.Context, string) error
	// RootFS returns the unpacked diffids that make up images rootfs.
	RootFS(ctx context.Context) ([]digest.Digest, error)
	// Size returns the total size of the image's packed resources.
	Size(ctx context.Context) (int64, error)
	// Config descriptor for the image.
	Config(ctx context.Context) (ocispec.Descriptor, error)
	// IsUnpacked returns whether or not an image is unpacked.
	IsUnpacked(context.Context, string) (bool, error)
	// ContentStore provides a content store which contains image blob data
	ContentStore() content.Store
}

var _ = (Image)(&image{})

// NewImage returns a client image object from the metadata image
func NewImage(client *Client, i images.Image) Image {
	return &image{
		client: client,
		i:      i,
	}
}

type image struct {
	client *Client

	i images.Image
}

func (i *image) Name() string {
	return i.i.Name
}

func (i *image) Target() ocispec.Descriptor {
	return i.i.Target
}

func (i *image) RootFS(ctx context.Context) ([]digest.Digest, error) {
	provider := i.client.ContentStore()
	return i.i.RootFS(ctx, provider, platforms.Default())
}

func (i *image) Size(ctx context.Context) (int64, error) {
	provider := i.client.ContentStore()
	return i.i.Size(ctx, provider, platforms.Default())
}

func (i *image) Config(ctx context.Context) (ocispec.Descriptor, error) {
	provider := i.client.ContentStore()
	return i.i.Config(ctx, provider, platforms.Default())
}

func (i *image) IsUnpacked(ctx context.Context, snapshotterName string) (bool, error) {
	sn := i.client.SnapshotService(snapshotterName)
	cs := i.client.ContentStore()

	diffs, err := i.i.RootFS(ctx, cs, platforms.Default())
	if err != nil {
		return false, err
	}

	chainID := identity.ChainID(diffs)
	_, err = sn.Stat(ctx, chainID.String())
	if err == nil {
		return true, nil
	} else if !errdefs.IsNotFound(err) {
		return false, err
	}

	return false, nil
}

func (i *image) Unpack(ctx context.Context, snapshotterName string) error {
	ctx, done, err := i.client.WithLease(ctx)
	if err != nil {
		return err
	}
	defer done(ctx)

	layers, err := i.getLayers(ctx, platforms.Default())
	if err != nil {
		return err
	}

	var (
		sn = i.client.SnapshotService(snapshotterName)
		a  = i.client.DiffService()
		cs = i.client.ContentStore()

		chain    []digest.Digest
		unpacked bool
	)
	for _, layer := range layers {
		unpacked, err = rootfs.ApplyLayer(ctx, layer, chain, sn, a)
		if err != nil {
			return err
		}

		if unpacked {
			// Set the uncompressed label after the uncompressed
			// digest has been verified through apply.
			cinfo := content.Info{
				Digest: layer.Blob.Digest,
				Labels: map[string]string{
					"containerd.io/uncompressed": layer.Diff.Digest.String(),
				},
			}
			if _, err := cs.Update(ctx, cinfo, "labels.containerd.io/uncompressed"); err != nil {
				return err
			}
		}

		chain = append(chain, layer.Diff.Digest)
	}

	if unpacked {
		desc, err := i.i.Config(ctx, cs, platforms.Default())
		if err != nil {
			return err
		}

		rootfs := identity.ChainID(chain).String()

		cinfo := content.Info{
			Digest: desc.Digest,
			Labels: map[string]string{
				fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotterName): rootfs,
			},
		}
		if _, err := cs.Update(ctx, cinfo, fmt.Sprintf("labels.containerd.io/gc.ref.snapshot.%s", snapshotterName)); err != nil {
			return err
		}
	}

	return nil
}

func (i *image) getLayers(ctx context.Context, platform string) ([]rootfs.Layer, error) {
	cs := i.client.ContentStore()

	manifest, err := images.Manifest(ctx, cs, i.i.Target, platform)
	if err != nil {
		return nil, err
	}

	diffIDs, err := i.i.RootFS(ctx, cs, platform)
	if err != nil {
		return nil, errors.Wrap(err, "failed to resolve rootfs")
	}
	if len(diffIDs) != len(manifest.Layers) {
		return nil, errors.Errorf("mismatched image rootfs and manifest layers")
	}
	layers := make([]rootfs.Layer, len(diffIDs))
	for i := range diffIDs {
		layers[i].Diff = ocispec.Descriptor{
			// TODO: derive media type from compressed type
			MediaType: ocispec.MediaTypeImageLayer,
			Digest:    diffIDs[i],
		}
		layers[i].Blob = manifest.Layers[i]
	}
	return layers, nil
}

func (i *image) ContentStore() content.Store {
	return i.client.ContentStore()
}
<|endoftext|>


<commit_before>package profitbricks

import (
	"net/http"
	"strconv"
)

//Image object
type Image struct {
	ID         string          `json:"id,omitempty"`
	PBType     string          `json:"type,omitempty"`
	Href       string          `json:"href,omitempty"`
	Metadata   *Metadata       `json:"metadata,omitempty"`
	Properties ImageProperties `json:"properties,omitempty"`
	Response   string          `json:"Response,omitempty"`
	Headers    *http.Header    `json:"headers,omitempty"`
	StatusCode int             `json:"statuscode,omitempty"`
}

//ImageProperties object
type ImageProperties struct {
	Name                string       `json:"name,omitempty"`
	Description         string       `json:"description,omitempty"`
	Location            string       `json:"location,omitempty"`
	Size                float64      `json:"size,omitempty"`
	CPUHotPlug          bool         `json:"cpuHotPlug,omitempty"`
	CPUHotUnplug        bool         `json:"cpuHotUnplug,omitempty"`
	RAMHotPlug          bool         `json:"ramHotPlug,omitempty"`
	RAMHotUnplug        bool         `json:"ramHotUnplug,omitempty"`
	NicHotPlug          bool         `json:"nicHotPlug,omitempty"`
	NicHotUnplug        bool         `json:"nicHotUnplug,omitempty"`
	DiscVirtioHotPlug   bool         `json:"discVirtioHotPlug,omitempty"`
	DiscVirtioHotUnplug bool         `json:"discVirtioHotUnplug,omitempty"`
	DiscScsiHotPlug     bool         `json:"discScsiHotPlug,omitempty"`
	DiscScsiHotUnplug   bool         `json:"discScsiHotUnplug,omitempty"`
	LicenceType         string       `json:"licenceType,omitempty"`
	ImageType           string       `json:"imageType,omitempty"`
	Public              bool         `json:"public,omitempty"`
	Response            string       `json:"Response,omitempty"`
	Headers             *http.Header `json:"headers,omitempty"`
	StatusCode          int          `json:"statuscode,omitempty"`
}

//Images object
type Images struct {
	ID         string       `json:"id,omitempty"`
	PBType     string       `json:"type,omitempty"`
	Href       string       `json:"href,omitempty"`
	Items      []Image      `json:"items,omitempty"`
	Response   string       `json:"Response,omitempty"`
	Headers    *http.Header `json:"headers,omitempty"`
	StatusCode int          `json:"statuscode,omitempty"`
}

//Cdroms object
type Cdroms struct {
	ID         string       `json:"id,omitempty"`
	PBType     string       `json:"type,omitempty"`
	Href       string       `json:"href,omitempty"`
	Items      []Image      `json:"items,omitempty"`
	Response   string       `json:"Response,omitempty"`
	Headers    *http.Header `json:"headers,omitempty"`
	StatusCode int          `json:"statuscode,omitempty"`
}

// ListImages returns an Collection struct
func (c *Client) ListImages() (*Images, error) {
	url := imageColPath() + `?depth=` + c.client.depth + `&pretty=` + strconv.FormatBool(c.client.pretty)
	ret := &Images{}
	err := c.client.Get(url, ret, http.StatusOK)
	return ret, err
}

// GetImage returns an Instance struct where id ==imageid
func (c *Client) GetImage(imageid string) (*Image, error) {
	url := imagePath(imageid)
	ret := &Image{}
	err := c.client.Get(url, ret, http.StatusOK)
	return ret, err
}
<commit_msg>Add parameter imageAliases to ImageProperties.<commit_after>package profitbricks

import (
	"net/http"
	"strconv"
)

//Image object
type Image struct {
	ID         string          `json:"id,omitempty"`
	PBType     string          `json:"type,omitempty"`
	Href       string          `json:"href,omitempty"`
	Metadata   *Metadata       `json:"metadata,omitempty"`
	Properties ImageProperties `json:"properties,omitempty"`
	Response   string          `json:"Response,omitempty"`
	Headers    *http.Header    `json:"headers,omitempty"`
	StatusCode int             `json:"statuscode,omitempty"`
}

//ImageProperties object
type ImageProperties struct {
	Name                string       `json:"name,omitempty"`
	Description         string       `json:"description,omitempty"`
	Location            string       `json:"location,omitempty"`
	Size                float64      `json:"size,omitempty"`
	CPUHotPlug          bool         `json:"cpuHotPlug,omitempty"`
	CPUHotUnplug        bool         `json:"cpuHotUnplug,omitempty"`
	RAMHotPlug          bool         `json:"ramHotPlug,omitempty"`
	RAMHotUnplug        bool         `json:"ramHotUnplug,omitempty"`
	NicHotPlug          bool         `json:"nicHotPlug,omitempty"`
	NicHotUnplug        bool         `json:"nicHotUnplug,omitempty"`
	DiscVirtioHotPlug   bool         `json:"discVirtioHotPlug,omitempty"`
	DiscVirtioHotUnplug bool         `json:"discVirtioHotUnplug,omitempty"`
	DiscScsiHotPlug     bool         `json:"discScsiHotPlug,omitempty"`
	DiscScsiHotUnplug   bool         `json:"discScsiHotUnplug,omitempty"`
	LicenceType         string       `json:"licenceType,omitempty"`
	ImageType           string       `json:"imageType,omitempty"`
	ImageAliases        []string     `json:"imageAliases,omitempty"`
	Public              bool         `json:"public,omitempty"`
	Response            string       `json:"Response,omitempty"`
	Headers             *http.Header `json:"headers,omitempty"`
	StatusCode          int          `json:"statuscode,omitempty"`
}

//Images object
type Images struct {
	ID         string       `json:"id,omitempty"`
	PBType     string       `json:"type,omitempty"`
	Href       string       `json:"href,omitempty"`
	Items      []Image      `json:"items,omitempty"`
	Response   string       `json:"Response,omitempty"`
	Headers    *http.Header `json:"headers,omitempty"`
	StatusCode int          `json:"statuscode,omitempty"`
}

//Cdroms object
type Cdroms struct {
	ID         string       `json:"id,omitempty"`
	PBType     string       `json:"type,omitempty"`
	Href       string       `json:"href,omitempty"`
	Items      []Image      `json:"items,omitempty"`
	Response   string       `json:"Response,omitempty"`
	Headers    *http.Header `json:"headers,omitempty"`
	StatusCode int          `json:"statuscode,omitempty"`
}

// ListImages returns an Collection struct
func (c *Client) ListImages() (*Images, error) {
	url := imageColPath() + `?depth=` + c.client.depth + `&pretty=` + strconv.FormatBool(c.client.pretty)
	ret := &Images{}
	err := c.client.Get(url, ret, http.StatusOK)
	return ret, err
}

// GetImage returns an Instance struct where id ==imageid
func (c *Client) GetImage(imageid string) (*Image, error) {
	url := imagePath(imageid)
	ret := &Image{}
	err := c.client.Get(url, ret, http.StatusOK)
	return ret, err
}
<|endoftext|>


<commit_before>package resolver // import "pault.ag/go/resolver"

import (
	"bufio"
	"fmt"
	"io"
	"strings"

	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/debian/version"
)

type Candidates map[string][]control.BinaryIndex

func (can *Candidates) AppendBinaryIndexReader(in io.Reader) error {
	reader := bufio.NewReader(in)
	index, err := control.ParseBinaryIndex(reader)
	if err != nil {
		return err
	}
	can.AppendBinaryIndex(index)
	return nil
}

func (can *Candidates) AppendBinaryIndex(index []control.BinaryIndex) {
	for _, entry := range index {
		(*can)[entry.Package] = append((*can)[entry.Package], entry)
	}
}

func NewCandidates(index []control.BinaryIndex) Candidates {
	ret := Candidates{}
	ret.AppendBinaryIndex(index)
	return ret
}

func ReadFromBinaryIndex(in io.Reader) (*Candidates, error) {
	reader := bufio.NewReader(in)
	index, err := control.ParseBinaryIndex(reader)
	if err != nil {
		return nil, err
	}
	can := NewCandidates(index)
	return &can, nil
}

func (can Candidates) ExplainSatisfiesBuildDepends(arch dependency.Arch, depends dependency.Dependency) (bool, string) {
	for _, possi := range depends.GetPossibilities(arch) {
		can, why, _ := can.ExplainSatisfies(arch, possi)
		if !can {
			return false, fmt.Sprintf("Possi %s can't be satisfied - %s", possi.Name, why)
		}
	}
	return true, "All relations are a go"
}

func (can Candidates) SatisfiesBuildDepends(arch dependency.Arch, depends dependency.Dependency) bool {
	ret, _ := can.ExplainSatisfiesBuildDepends(arch, depends)
	return ret
}

func (can Candidates) Satisfies(arch dependency.Arch, possi dependency.Possibility) bool {
	ret, _, _ := can.ExplainSatisfies(arch, possi)
	return ret
}

func (can Candidates) ExplainSatisfies(arch dependency.Arch, possi dependency.Possibility) (bool, string, []control.BinaryIndex) {
	entries, ok := can[possi.Name]
	if !ok { // no known entries in the Index
		return false, fmt.Sprintf("Totally unknown package: %s", possi.Name), nil
	}

	if possi.Arch != nil {
		satisfied := false
		archEntries := []control.BinaryIndex{}
		for _, installable := range entries {
			if installable.Architecture.Is(possi.Arch) {
				archEntries = append(archEntries, installable)
				satisfied = true
			}
		}
		if !satisfied {
			return false, fmt.Sprintf(
				"Relation depends on multiarch arch %s-%s-%s. Not found",
				possi.Arch.ABI,
				possi.Arch.OS,
				possi.Arch.CPU,
			), nil
		}
		entries = archEntries
	}

	if possi.Version == nil {
		return true, "Relation exists, no version constraint", entries
	}

	// OK, so we have to play with versions now.
	vr := *possi.Version
	relatioNumber, _ := version.Parse(vr.Number)
	satisfied := false
	seenRealtions := []string{}

	for _, installable := range entries {
		q := version.Compare(installable.Version, relatioNumber)
		seenRealtions = append(seenRealtions, installable.Version.String())

		switch vr.Operator {
		case ">=":
			satisfied = q >= 0
		case "<=":
			satisfied = q <= 0
		case ">>":
			satisfied = q > 0
		case "<<":
			satisfied = q < 0
		case "=":
			satisfied = q == 0
		default:
			return false, "Unknown operator D:", nil // XXX: WHAT THE SHIT
		}

		if satisfied {
			return true, "Relation exists with a satisfied version constraint", []control.BinaryIndex{installable} // TODO gather the full list of version-constrained satisfiers
		}
	}

	return false, fmt.Sprintf(
		"%s is version constrainted %s %s. Valid options: %s",
		possi.Name,
		vr.Operator,
		vr.Number,
		strings.Join(seenRealtions, ", "),
	), nil
}
<commit_msg>Implement "version constraint multi-return" TODO<commit_after>package resolver // import "pault.ag/go/resolver"

import (
	"bufio"
	"fmt"
	"io"
	"strings"

	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/debian/version"
)

type Candidates map[string][]control.BinaryIndex

func (can *Candidates) AppendBinaryIndexReader(in io.Reader) error {
	reader := bufio.NewReader(in)
	index, err := control.ParseBinaryIndex(reader)
	if err != nil {
		return err
	}
	can.AppendBinaryIndex(index)
	return nil
}

func (can *Candidates) AppendBinaryIndex(index []control.BinaryIndex) {
	for _, entry := range index {
		(*can)[entry.Package] = append((*can)[entry.Package], entry)
	}
}

func NewCandidates(index []control.BinaryIndex) Candidates {
	ret := Candidates{}
	ret.AppendBinaryIndex(index)
	return ret
}

func ReadFromBinaryIndex(in io.Reader) (*Candidates, error) {
	reader := bufio.NewReader(in)
	index, err := control.ParseBinaryIndex(reader)
	if err != nil {
		return nil, err
	}
	can := NewCandidates(index)
	return &can, nil
}

func (can Candidates) ExplainSatisfiesBuildDepends(arch dependency.Arch, depends dependency.Dependency) (bool, string) {
	for _, possi := range depends.GetPossibilities(arch) {
		can, why, _ := can.ExplainSatisfies(arch, possi)
		if !can {
			return false, fmt.Sprintf("Possi %s can't be satisfied - %s", possi.Name, why)
		}
	}
	return true, "All relations are a go"
}

func (can Candidates) SatisfiesBuildDepends(arch dependency.Arch, depends dependency.Dependency) bool {
	ret, _ := can.ExplainSatisfiesBuildDepends(arch, depends)
	return ret
}

func (can Candidates) Satisfies(arch dependency.Arch, possi dependency.Possibility) bool {
	ret, _, _ := can.ExplainSatisfies(arch, possi)
	return ret
}

func (can Candidates) ExplainSatisfies(arch dependency.Arch, possi dependency.Possibility) (bool, string, []control.BinaryIndex) {
	entries, ok := can[possi.Name]
	if !ok { // no known entries in the Index
		return false, fmt.Sprintf("Totally unknown package: %s", possi.Name), nil
	}

	if possi.Arch != nil {
		satisfied := false
		archEntries := []control.BinaryIndex{}
		for _, installable := range entries {
			if installable.Architecture.Is(possi.Arch) {
				archEntries = append(archEntries, installable)
				satisfied = true
			}
		}
		if !satisfied {
			return false, fmt.Sprintf(
				"Relation depends on multiarch arch %s-%s-%s. Not found",
				possi.Arch.ABI,
				possi.Arch.OS,
				possi.Arch.CPU,
			), nil
		}
		entries = archEntries
	}

	if possi.Version == nil {
		return true, "Relation exists, no version constraint", entries
	}

	// OK, so we have to play with versions now.
	vr := *possi.Version
	relatioNumber, _ := version.Parse(vr.Number)
	satisfied := false
	seenRealtions := []string{}

	versionEntries := []control.BinaryIndex{}
	for _, installable := range entries {
		q := version.Compare(installable.Version, relatioNumber)
		seenRealtions = append(seenRealtions, installable.Version.String())

		switch vr.Operator {
		case ">=":
			satisfied = q >= 0
		case "<=":
			satisfied = q <= 0
		case ">>":
			satisfied = q > 0
		case "<<":
			satisfied = q < 0
		case "=":
			satisfied = q == 0
		default:
			return false, "Unknown operator D:", nil // XXX: WHAT THE SHIT
		}

		if satisfied {
			versionEntries = append(versionEntries, installable)
		}
	}

	if len(versionEntries) > 0 {
		return true, "Relation exists with a satisfied version constraint", versionEntries
	}

	return false, fmt.Sprintf(
		"%s is version constrainted %s %s. Valid options: %s",
		possi.Name,
		vr.Operator,
		vr.Number,
		strings.Join(seenRealtions, ", "),
	), nil
}
<|endoftext|>


<commit_before>package cete

import (
	"bytes"
	"log"
	"os"

	"github.com/dgraph-io/badger/badger"
	"gopkg.in/vmihailenco/msgpack.v2"
)

// Bounds is the type for variables which represent a bound for Between.
type Bounds int

// Valid bounds.
var (
	MinBounds Bounds = 1
	MaxBounds Bounds = 2
)

// NewIndex creates a new index on the table, using the name as the Query.
// The index name must not be empty, and must be no more than 125 bytes
// long. ErrAlreadyExists will be returned if the index already exists.
//
// NewIndex may take a while if there are already values in the
// table, as it needs to index all the existing values in the table.
func (t *Table) NewIndex(name string) error {
	if name == "" || len(name) > 125 {
		return ErrBadIdentifier
	}

	t.db.configMutex.Lock()

	tableName := t.name()
	tableConfigKey := -1

	for key, table := range t.db.config.Tables {
		if table.TableName == tableName {
			tableConfigKey = key
			for _, index := range table.Indexes {
				if index.IndexName == name {
					t.db.configMutex.Unlock()
					return ErrAlreadyExists
				}
			}
		}
	}

	if tableConfigKey < 0 {
		log.Println("cete: attempt to call new index on a non-existent table")
		t.db.configMutex.Unlock()
		return ErrNotFound
	}

	kv, err := t.db.newKV(Name(tableName), Name(name))
	if err != nil {
		t.db.configMutex.Unlock()
		return err
	}

	indexes := t.db.config.Tables[tableConfigKey].Indexes
	indexes = append(indexes, indexConfig{IndexName: name})
	t.db.config.Tables[tableConfigKey].Indexes = indexes
	if err = t.db.writeConfig(); err != nil {
		t.db.configMutex.Unlock()
		return err
	}

	t.db.configMutex.Unlock()

	idx := &Index{
		index: kv,
		table: t,
	}

	t.indexes[Name(name)] = idx

	if err = idx.indexValues(name); err != nil {
		log.Println("cete: error while indexing \""+
			idx.name()+"\", index likely corrupt:",
			err)
		return nil
	}

	return nil
}

func (i *Index) indexValues(name string) error {
	r := i.table.Between(MinBounds, MaxBounds)

	var entry bufferEntry
	var results []interface{}
	var err error

	for {
		entry = <-r.buffer
		if entry.err == ErrEndOfRange {
			break
		} else if entry.err != nil {
			return entry.err
		}

		results, err = msgpack.NewDecoder(bytes.NewReader(entry.data)).Query(name)
		if err != nil {
			continue
		}

		for _, result := range results {
			err = i.addToIndex(valueToBytes(result), entry.key)
			if err != nil {
				log.Println("cete: index error for index \""+name+"\":", err)
			}
		}
	}

	return nil
}

// One puts the first matching value with the index's key into dst. dst
// must either be a pointer or nil if you would like to only get the key/counter
// and check for existence. Note that indexes are non-unique, a single index key
// can map to multiple values. Use GetAll to get all such matching values.
func (i *Index) One(key interface{}, dst interface{}) (string, int, error) {
	r, err := i.GetAll(key)
	if err != nil {
		return "", 0, err
	}

	defer r.Close()

	tableKey, counter, err := r.Next(dst)
	if err == ErrEndOfRange {
		log.Println("cete: warning: corrupt index detected:", i.name())
		return tableKey, counter, ErrNotFound
	}

	return tableKey, counter, err
}

// GetAll returns all the matching values as a range for the provided index key.
func (i *Index) GetAll(key interface{}) (*Range, error) {
	var item badger.KVItem
	err := i.index.Get(valueToBytes(key), &item)
	if err != nil {
		return nil, err
	}

	if item.Value() == nil {
		return nil, ErrNotFound
	}

	var keys []string
	err = msgpack.Unmarshal(item.Value(), &keys)
	if err != nil {
		log.Println("cete: corrupt index \""+i.name()+"\":", err)
		return nil, ErrIndexError
	}

	if len(keys) == 0 {
		log.Println("cete: corrupt index \""+i.name()+"\":", err)
		return nil, ErrNotFound
	}

	c := 0
	var value []byte

	return newRange(func() (string, []byte, int, error) {
		for {
			if c >= len(keys) {
				return "", nil, 0, ErrEndOfRange
			}

			err = i.table.data.Get([]byte(keys[c]), &item)
			if err != nil {
				return "", nil, 0, err
			}

			if item.Value() == nil {
				c++
				continue
			}

			value = make([]byte, len(item.Value()))
			copy(value, item.Value())

			c++
			return keys[c-1], value, int(item.Counter()), nil
		}
	}, func() {}), nil
}

// Between returns a Range of documents between the lower and upper index values
// provided. The range will be sorted in ascending order by index value. You can
// reverse the sorting by specifying true to the optional reverse parameter.
// The bounds are inclusive on both ends. It is possible to have
// duplicate documents if the same document has multiple unique index values.
//
// You can use cete.MinBounds and cete.MaxBounds to specify minimum and maximum
// bound values.
func (i *Index) Between(lower, upper interface{},
	reverse ...bool) *Range {
	shouldReverse := (len(reverse) > 0) && reverse[0]

	itOpts := badger.DefaultIteratorOptions
	itOpts.PrefetchSize = 5
	itOpts.Reverse = shouldReverse
	it := i.index.NewIterator(itOpts)

	upperBytes := valueToBytes(upper)
	lowerBytes := valueToBytes(lower)

	if !shouldReverse {
		if lower == MinBounds {
			it.Rewind()
		} else {
			it.Seek(lowerBytes)
		}
	} else {
		if upper == MaxBounds {
			it.Rewind()
		} else {
			it.Seek(upperBytes)
		}
	}

	var lastRange *Range

	return newRange(i.betweenNext(it, lastRange, shouldReverse, lower, upper),
		func() {
			if lastRange != nil {
				lastRange.Close()
			}
			it.Close()
		})
}

func (i *Index) betweenNext(it *badger.Iterator, lastRange *Range,
	shouldReverse bool, lower,
	upper interface{}) func() (string, []byte, int, error) {
	upperBytes := valueToBytes(upper)
	lowerBytes := valueToBytes(lower)

	var entry bufferEntry

	return func() (string, []byte, int, error) {
		if lastRange != nil {
			entry = <-lastRange.buffer
			if entry.err != ErrEndOfRange {
				return entry.key, entry.data, entry.counter, entry.err
			}

			lastRange.Close()
		}

		for it.Valid() {
			if !shouldReverse && upper != MaxBounds &&
				bytes.Compare(it.Item().Key(), upperBytes) > 0 {
				return "", nil, 0, ErrEndOfRange
			} else if shouldReverse && lower != MinBounds &&
				bytes.Compare(it.Item().Key(), lowerBytes) < 0 {
				return "", nil, 0, ErrEndOfRange
			}

			r, err := i.GetAll(it.Item().Key())
			it.Next()
			if err != nil {
				continue
			}

			lastRange = r

			entry = <-lastRange.buffer
			if entry.err != ErrEndOfRange {
				return entry.key, entry.data, entry.counter, entry.err
			}

			lastRange.Close()
		}

		return "", nil, 0, ErrEndOfRange
	}
}

// All returns all the documents which have an index value. It is shorthand
// for Between(MinBounds, MaxBounds, reverse...)
func (i *Index) All(reverse ...bool) *Range {
	return i.Between(MinBounds, MaxBounds, reverse...)
}

// Drop drops the index from the table, deleting its folder from the disk.
// All further calls to the index will result in undefined behaviour.
// Note that table.Index("deleted index") will be nil.
func (i *Index) Drop() error {
	i.table.db.configMutex.Lock()
	defer i.table.db.configMutex.Unlock()

	tableName := i.table.name()

	var indexName string

	for idxName, index := range i.table.indexes {
		if index == i {
			indexName = string(idxName)
		}
	}

	if indexName == "" {
		return ErrNotFound
	}

tableLoop:
	for key, table := range i.table.db.config.Tables {
		if table.TableName == tableName {
			for indexKey, index := range table.Indexes {
				if index.IndexName == indexName {
					indexes := i.table.db.config.Tables[key].Indexes
					indexes = append(indexes[:indexKey], indexes[indexKey+1:]...)
					i.table.db.config.Tables[key].Indexes = indexes
					break tableLoop
				}
			}
		}
	}

	if err := i.table.db.writeConfig(); err != nil {
		return err
	}

	i.index.Close()

	delete(i.table.indexes, Name(indexName))

	return os.RemoveAll(i.table.db.path + "/" + Name(tableName).Hex() + "/" +
		Name(indexName).Hex())
}
<commit_msg>Move back new line<commit_after>package cete

import (
	"bytes"
	"log"
	"os"

	"github.com/dgraph-io/badger/badger"
	"gopkg.in/vmihailenco/msgpack.v2"
)

// Bounds is the type for variables which represent a bound for Between.
type Bounds int

// Valid bounds.
var (
	MinBounds Bounds = 1
	MaxBounds Bounds = 2
)

// NewIndex creates a new index on the table, using the name as the Query.
// The index name must not be empty, and must be no more than 125 bytes
// long. ErrAlreadyExists will be returned if the index already exists.
//
// NewIndex may take a while if there are already values in the
// table, as it needs to index all the existing values in the table.
func (t *Table) NewIndex(name string) error {
	if name == "" || len(name) > 125 {
		return ErrBadIdentifier
	}

	t.db.configMutex.Lock()

	tableName := t.name()
	tableConfigKey := -1

	for key, table := range t.db.config.Tables {
		if table.TableName == tableName {
			tableConfigKey = key
			for _, index := range table.Indexes {
				if index.IndexName == name {
					t.db.configMutex.Unlock()
					return ErrAlreadyExists
				}
			}
		}
	}

	if tableConfigKey < 0 {
		log.Println("cete: attempt to call new index on a non-existent table")
		t.db.configMutex.Unlock()
		return ErrNotFound
	}

	kv, err := t.db.newKV(Name(tableName), Name(name))
	if err != nil {
		t.db.configMutex.Unlock()
		return err
	}

	indexes := t.db.config.Tables[tableConfigKey].Indexes
	indexes = append(indexes, indexConfig{IndexName: name})
	t.db.config.Tables[tableConfigKey].Indexes = indexes
	if err = t.db.writeConfig(); err != nil {
		t.db.configMutex.Unlock()
		return err
	}

	t.db.configMutex.Unlock()

	idx := &Index{
		index: kv,
		table: t,
	}

	t.indexes[Name(name)] = idx

	if err = idx.indexValues(name); err != nil {
		log.Println("cete: error while indexing \""+
			idx.name()+"\", index likely corrupt:", err)
		return nil
	}

	return nil
}

func (i *Index) indexValues(name string) error {
	r := i.table.Between(MinBounds, MaxBounds)

	var entry bufferEntry
	var results []interface{}
	var err error

	for {
		entry = <-r.buffer
		if entry.err == ErrEndOfRange {
			break
		} else if entry.err != nil {
			return entry.err
		}

		results, err = msgpack.NewDecoder(bytes.NewReader(entry.data)).Query(name)
		if err != nil {
			continue
		}

		for _, result := range results {
			err = i.addToIndex(valueToBytes(result), entry.key)
			if err != nil {
				log.Println("cete: index error for index \""+name+"\":", err)
			}
		}
	}

	return nil
}

// One puts the first matching value with the index's key into dst. dst
// must either be a pointer or nil if you would like to only get the key/counter
// and check for existence. Note that indexes are non-unique, a single index key
// can map to multiple values. Use GetAll to get all such matching values.
func (i *Index) One(key interface{}, dst interface{}) (string, int, error) {
	r, err := i.GetAll(key)
	if err != nil {
		return "", 0, err
	}

	defer r.Close()

	tableKey, counter, err := r.Next(dst)
	if err == ErrEndOfRange {
		log.Println("cete: warning: corrupt index detected:", i.name())
		return tableKey, counter, ErrNotFound
	}

	return tableKey, counter, err
}

// GetAll returns all the matching values as a range for the provided index key.
func (i *Index) GetAll(key interface{}) (*Range, error) {
	var item badger.KVItem
	err := i.index.Get(valueToBytes(key), &item)
	if err != nil {
		return nil, err
	}

	if item.Value() == nil {
		return nil, ErrNotFound
	}

	var keys []string
	err = msgpack.Unmarshal(item.Value(), &keys)
	if err != nil {
		log.Println("cete: corrupt index \""+i.name()+"\":", err)
		return nil, ErrIndexError
	}

	if len(keys) == 0 {
		log.Println("cete: corrupt index \""+i.name()+"\":", err)
		return nil, ErrNotFound
	}

	c := 0
	var value []byte

	return newRange(func() (string, []byte, int, error) {
		for {
			if c >= len(keys) {
				return "", nil, 0, ErrEndOfRange
			}

			err = i.table.data.Get([]byte(keys[c]), &item)
			if err != nil {
				return "", nil, 0, err
			}

			if item.Value() == nil {
				c++
				continue
			}

			value = make([]byte, len(item.Value()))
			copy(value, item.Value())

			c++
			return keys[c-1], value, int(item.Counter()), nil
		}
	}, func() {}), nil
}

// Between returns a Range of documents between the lower and upper index values
// provided. The range will be sorted in ascending order by index value. You can
// reverse the sorting by specifying true to the optional reverse parameter.
// The bounds are inclusive on both ends. It is possible to have
// duplicate documents if the same document has multiple unique index values.
//
// You can use cete.MinBounds and cete.MaxBounds to specify minimum and maximum
// bound values.
func (i *Index) Between(lower, upper interface{},
	reverse ...bool) *Range {
	shouldReverse := (len(reverse) > 0) && reverse[0]

	itOpts := badger.DefaultIteratorOptions
	itOpts.PrefetchSize = 5
	itOpts.Reverse = shouldReverse
	it := i.index.NewIterator(itOpts)

	upperBytes := valueToBytes(upper)
	lowerBytes := valueToBytes(lower)

	if !shouldReverse {
		if lower == MinBounds {
			it.Rewind()
		} else {
			it.Seek(lowerBytes)
		}
	} else {
		if upper == MaxBounds {
			it.Rewind()
		} else {
			it.Seek(upperBytes)
		}
	}

	var lastRange *Range

	return newRange(i.betweenNext(it, lastRange, shouldReverse, lower, upper),
		func() {
			if lastRange != nil {
				lastRange.Close()
			}
			it.Close()
		})
}

func (i *Index) betweenNext(it *badger.Iterator, lastRange *Range,
	shouldReverse bool, lower,
	upper interface{}) func() (string, []byte, int, error) {
	upperBytes := valueToBytes(upper)
	lowerBytes := valueToBytes(lower)

	var entry bufferEntry

	return func() (string, []byte, int, error) {
		if lastRange != nil {
			entry = <-lastRange.buffer
			if entry.err != ErrEndOfRange {
				return entry.key, entry.data, entry.counter, entry.err
			}

			lastRange.Close()
		}

		for it.Valid() {
			if !shouldReverse && upper != MaxBounds &&
				bytes.Compare(it.Item().Key(), upperBytes) > 0 {
				return "", nil, 0, ErrEndOfRange
			} else if shouldReverse && lower != MinBounds &&
				bytes.Compare(it.Item().Key(), lowerBytes) < 0 {
				return "", nil, 0, ErrEndOfRange
			}

			r, err := i.GetAll(it.Item().Key())
			it.Next()
			if err != nil {
				continue
			}

			lastRange = r

			entry = <-lastRange.buffer
			if entry.err != ErrEndOfRange {
				return entry.key, entry.data, entry.counter, entry.err
			}

			lastRange.Close()
		}

		return "", nil, 0, ErrEndOfRange
	}
}

// All returns all the documents which have an index value. It is shorthand
// for Between(MinBounds, MaxBounds, reverse...)
func (i *Index) All(reverse ...bool) *Range {
	return i.Between(MinBounds, MaxBounds, reverse...)
}

// Drop drops the index from the table, deleting its folder from the disk.
// All further calls to the index will result in undefined behaviour.
// Note that table.Index("deleted index") will be nil.
func (i *Index) Drop() error {
	i.table.db.configMutex.Lock()
	defer i.table.db.configMutex.Unlock()

	tableName := i.table.name()

	var indexName string

	for idxName, index := range i.table.indexes {
		if index == i {
			indexName = string(idxName)
		}
	}

	if indexName == "" {
		return ErrNotFound
	}

tableLoop:
	for key, table := range i.table.db.config.Tables {
		if table.TableName == tableName {
			for indexKey, index := range table.Indexes {
				if index.IndexName == indexName {
					indexes := i.table.db.config.Tables[key].Indexes
					indexes = append(indexes[:indexKey], indexes[indexKey+1:]...)
					i.table.db.config.Tables[key].Indexes = indexes
					break tableLoop
				}
			}
		}
	}

	if err := i.table.db.writeConfig(); err != nil {
		return err
	}

	i.index.Close()

	delete(i.table.indexes, Name(indexName))

	return os.RemoveAll(i.table.db.path + "/" + Name(tableName).Hex() + "/" +
		Name(indexName).Hex())
}
<|endoftext|>