<commit_before>package rpm

import (
	"fmt"

	"github.com/facebookincubator/nvdtools/wfn"
)

// Checker knows how to verify whether some package has been fixed or not
type Checker interface {
	// Check should return whether a given package on distribution is fixed for some CVE
	Check(pkg *Package, distro *wfn.Attributes, cve string) bool
}

// CheckAny returns a Checker which will return true if any of the underlying checkers returns true
func CheckAny(chks ...Checker) Checker {
	return anyChecker(chks)
}

type anyChecker []Checker

// Check is part of the Checker interface
func (c anyChecker) Check(pkg *Package, distro *wfn.Attributes, cve string) bool {
	for _, chk := range c {
		if chk.Check(pkg, distro, cve) {
			return true
		}
	}
	return false
}

// CheckAll returns a Checker which will return true if all of the underlying checkers returns true
func CheckAll(chks ...Checker) Checker {
	return allChecker(chks)
}

type allChecker []Checker

// Check is part of the Checker interface
func (c allChecker) Check(pkg *Package, distro *wfn.Attributes, cve string) bool {
	if len(c) == 0 {
		return false
	}
	for _, chk := range c {
		if !chk.Check(pkg, distro, cve) {
			return false
		}
	}
	return true
}

// MapChecker implements the Checker interface
// calls the checker which is mapped to checked CVE
type MapChecker map[string]Checker // CVE -> Checker

// Check is part of the Checker interface
func (c MapChecker) Check(pkg *Package, distro *wfn.Attributes, cve string) bool {
	if chk, ok := c[cve]; ok {
		return chk.Check(pkg, distro, cve)
	}
	return false
}

// Check will parse package and distro and call given checker to return
func Check(chk Checker, pkg, distro, cve string) (bool, error) {
	p, err := Parse(pkg)
	if err != nil {
		return false, fmt.Errorf("can't parse package %q: %v", pkg, err)
	}

	d, err := wfn.Parse(distro)
	if err != nil {
		return false, fmt.Errorf("can't parse distro cpe %q: %v", distro, err)
	}

	return chk.Check(p, d, cve), nil
}
<commit_msg>fix copyright<commit_after>// Copyright (c) Facebook, Inc. and its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rpm

import (
	"fmt"

	"github.com/facebookincubator/nvdtools/wfn"
)

// Checker knows how to verify whether some package has been fixed or not
type Checker interface {
	// Check should return whether a given package on distribution is fixed for some CVE
	Check(pkg *Package, distro *wfn.Attributes, cve string) bool
}

// CheckAny returns a Checker which will return true if any of the underlying checkers returns true
func CheckAny(chks ...Checker) Checker {
	return anyChecker(chks)
}

type anyChecker []Checker

// Check is part of the Checker interface
func (c anyChecker) Check(pkg *Package, distro *wfn.Attributes, cve string) bool {
	for _, chk := range c {
		if chk.Check(pkg, distro, cve) {
			return true
		}
	}
	return false
}

// CheckAll returns a Checker which will return true if all of the underlying checkers returns true
func CheckAll(chks ...Checker) Checker {
	return allChecker(chks)
}

type allChecker []Checker

// Check is part of the Checker interface
func (c allChecker) Check(pkg *Package, distro *wfn.Attributes, cve string) bool {
	if len(c) == 0 {
		return false
	}
	for _, chk := range c {
		if !chk.Check(pkg, distro, cve) {
			return false
		}
	}
	return true
}

// MapChecker implements the Checker interface
// calls the checker which is mapped to checked CVE
type MapChecker map[string]Checker // CVE -> Checker

// Check is part of the Checker interface
func (c MapChecker) Check(pkg *Package, distro *wfn.Attributes, cve string) bool {
	if chk, ok := c[cve]; ok {
		return chk.Check(pkg, distro, cve)
	}
	return false
}

// Check will parse package and distro and call given checker to return
func Check(chk Checker, pkg, distro, cve string) (bool, error) {
	p, err := Parse(pkg)
	if err != nil {
		return false, fmt.Errorf("can't parse package %q: %v", pkg, err)
	}

	d, err := wfn.Parse(distro)
	if err != nil {
		return false, fmt.Errorf("can't parse distro cpe %q: %v", distro, err)
	}

	return chk.Check(p, d, cve), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fun

import (
	"math"
	"math/cmplx"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/plt"
	"github.com/cpmech/gosl/utl"
)

// Smoothing kinds
var (

	// No smoothing
	SmoNoneKind = io.NewEnum("None", "fun.smoothing", "L", "No smoothing")

	// Lanczos (sinc) smoothing kind
	SmoLanczosKind = io.NewEnum("Lanczos", "fun.smoothing", "L", "Lanczos (sinc) smoothing kind")

	// Cesaro
	SmoCesaroKind = io.NewEnum("Cesaro", "fun.smoothing", "L", "Lanczos (sinc) smoothing kind")

	// Raised Cosine
	SmoRcosKind = io.NewEnum("Rcos", "fun.smoothing", "L", "Lanczos (sinc) smoothing kind")
)

// FourierInterp performs interpolation using truncated Fourier series
//
//                  N/2 - 1
//                   ————          +i k X[j]
//        f(x[j]) =  \     A[k] ⋅ e              with    X[j] = 2 π j / N
//                   /
//                   ————
//                  k = -N/2
//
//   x ϵ [0, 2π]
//
//   Equation (2.1.27) of [1]. Note that f=u in [1] and A[k] is the tilde(u[k]) of [1]
//
//   Reference:
//     [1] Canuto C, Hussaini MY, Quarteroni A, Zang TA (2006) Spectral Methods: Fundamentals in
//         Single Domains. Springer. 563p
//
type FourierInterp struct {
	N int          // number of terms. must be power of 2; i.e. N = 2ⁿ
	A []complex128 // coefficients for interpolation. from FFT
	S []complex128 // smothing coefficients
}

// NewFourierInterp allocates a new FourierInterp object
//
//                    N - 1
//                1   ————             -i k X[j]
//        A[k] = ———  \     f(x[j]) ⋅ e              with    X[j] = 2 π j / N
//                N   /
//                    ————
//                   j = 0
//
//  INPUT:
//    N -- number of terms. must be power of 2; i.e. N = 2ⁿ
//
//    smoothing -- type of smoothing: use SmoNoneKind for no smoothing
//
//  NOTE: (1) x ϵ [0, 2π]
//        (2) remember to call CalcA to calculate coefficients A!
//
//   Equation (2.1.25) of [1]. Note that f=u in [1] and A[k] is the tilde(u[k]) of [1]
//
//   Reference:
//     [1] Canuto C, Hussaini MY, Quarteroni A, Zang TA (2006) Spectral Methods: Fundamentals in
//         Single Domains. Springer. 563p
//
func NewFourierInterp(N int, smoothing io.Enum) (o *FourierInterp, err error) {

	// check
	if N%2 != 0 {
		err = chk.Err("N must be even. N=%d is invalid\n", N)
		return
	}

	// allocate
	o = new(FourierInterp)
	o.N = N
	o.A = make([]complex128, o.N)
	o.S = make([]complex128, o.N)

	// compute smoothing coefficients
	n := float64(o.N)
	σ := func(k float64) float64 { return 1.0 }
	switch smoothing {
	case SmoLanczosKind:
		σ = func(k float64) float64 { return Sinc(2 * k * π / n) }
	case SmoRcosKind:
		σ = func(k float64) float64 { return (1.0 + math.Cos(2*k*π/n)) / 2.0 }
	case SmoCesaroKind:
		σ = func(k float64) float64 { return 1.0 - math.Abs(k)/(1.0+n/2.0) }
	}

	for j := 0; j < o.N; j++ {
		o.S[j] = complex(σ(o.K(j)), 0)
	}
	return
}

// CalcA calculates the coefficients A of the interpolation ousing FFT
func (o *FourierInterp) CalcA(f Ss) (err error) {

	// compute f(x[j]) and set A[j] with f(x[j]) / N
	var fxj float64
	n := float64(o.N)
	for j := 0; j < o.N; j++ {
		xj := 2.0 * math.Pi * float64(j) / n
		fxj, err = f(xj)
		if err != nil {
			return
		}
		o.A[j] = complex(fxj/n, 0)
	}

	// perform Fourier transform to find A[j]
	err = Dft1d(o.A, false)
	return
}

// K computes k-index from j-index where j corresponds to the FFT index
//
//   FFT returns the A coefficients as:
//
//      {A[0], A[1], ..., A[N/2-1], A[-N/2], A[-N/2+1], ... A[-1]}
//
//   k ϵ [-N/2, N/2-1]
//   j ϵ [0, N-1]
//
//   Example with N = 8:
//
//        j=0 ⇒ k=0      j=4 ⇒ k=-4
//        j=1 ⇒ k=1      j=5 ⇒ k=-3
//        j=2 ⇒ k=2      j=6 ⇒ k=-2
//        j=3 ⇒ k=3      j=7 ⇒ k=-1
//
func (o *FourierInterp) K(j int) float64 {
	h := o.N / 2
	k := j - (j/h)*o.N
	return float64(k)
}

// I computes the interpolation
//
//                  N/2 - 1
//                    ————          +i k x
//        I {f}(x) =  \     A[k] ⋅ e
//         N          /
//                    ————
//                   k = -N/2
//
//   x ϵ [0, 2π]
//
//   Equation (2.1.28) of [1]. Note that f=u in [1] and A[k] is the tilde(u[k]) of [1]
//
//   Reference:
//     [1] Canuto C, Hussaini MY, Quarteroni A, Zang TA (2006) Spectral Methods: Fundamentals in
//         Single Domains. Springer. 563p
//
//  NOTE: remember to call CalcA to calculate coefficients A!
//
func (o *FourierInterp) I(x float64) float64 {
	var res complex128
	for j := 0; j < o.N; j++ {
		res += o.S[j] * o.A[j] * cmplx.Exp(complex(0, o.K(j)*x))
	}
	return real(res)
}

// DI computes the p-derivative of the interpolation
//
//                   p       N/2 - 1
//        p         d(I{f})    ————       p           +i k x
//       DI{f}(x) = ——————— =  \     (i⋅k)  ⋅ A[k] ⋅ e
//        N             p      /
//                    dx       ————
//                            k = -N/2
//
//   x ϵ [0, 2π]
//
//  NOTE: remember to call CalcA to calculate coefficients A!
//
func (o *FourierInterp) DI(p int, x float64) float64 {
	var res complex128
	pc := complex(float64(p), 0)
	for j := 0; j < o.N; j++ {
		ik := complex(0, o.K(j))
		ikp := cmplx.Pow(ik, pc)
		res += ikp * o.S[j] * o.A[j] * cmplx.Exp(complex(0, o.K(j)*x))
	}
	return real(res)
}

// Plot plots interpolated curve
//   option -- 1: plot only f(x)
//             2: plot both f(x) and df/dx(x)
//             3: plot all f(x), df/dx(x) and d^2f/dx^2
//             4: plot only df/dx(x)
//             5: plot only d^2f/dx^2(x)
//             6: plot df^p/dx^p
//   p      -- order of the derivative to plot if option == 6
//   dfdx   -- is the analytic df/dx(x) [optional]
//   d2fdx2 -- is the analytic d^2f/dx^2(x) [optional]
func (o *FourierInterp) Plot(option, p int, f, dfdx, d2fdx2 Ss, argsF, argsI, argsD1, argsD2 *plt.A) {

	// set arguments
	if argsF == nil {
		argsF = &plt.A{L: "f(x)", C: plt.C(0, 1), NoClip: true}
	}
	if argsI == nil {
		argsI = &plt.A{L: "I{f}(x)", C: plt.C(1, 1), NoClip: true}
	}
	if argsD1 == nil {
		argsD1 = &plt.A{L: "D1I{f}(x)", C: plt.C(2, 1), NoClip: true}
	}
	if argsD2 == nil {
		argsD2 = &plt.A{L: "D2I{f}(x)", C: plt.C(3, 1), NoClip: true}
	}

	// graph points
	npts := 2001
	xx := utl.LinSpace(0, 2.0*math.Pi, npts)

	// options
	withF := option == 1 || option == 2 || option == 3
	firstD := option == 2 || option == 3 || option == 4
	secondD := option == 3 || option == 5

	// allocate arrays
	var y1, y2 []float64
	if withF {
		y1 = make([]float64, npts)
		y2 = make([]float64, npts)
	}
	var y3, y4 []float64
	if firstD {
		y3 = make([]float64, npts)
		y4 = make([]float64, npts)
	}
	var y5, y6 []float64
	if secondD {
		y5 = make([]float64, npts)
		y6 = make([]float64, npts)
	}
	var y7 []float64
	if option == 6 {
		y7 = make([]float64, npts)
	}

	// compute values
	for i := 0; i < npts; i++ {
		x := xx[i]
		if withF {
			if f != nil {
				fx, err := f(x)
				if err != nil {
					chk.Panic("f(x) failed:\n%v\n", err)
				}
				y1[i] = fx
			}
			y2[i] = o.I(x)
		}
		if firstD {
			if dfdx != nil {
				dfx, err := dfdx(x)
				if err != nil {
					chk.Panic("df/dx(x) failed:\n%v\n", err)
				}
				y3[i] = dfx
			}
			y4[i] = o.DI(1, x)
		}
		if secondD {
			if d2fdx2 != nil {
				ddfx, err := d2fdx2(x)
				if err != nil {
					chk.Panic("d2f/dx2(x) failed:\n%v\n", err)
				}
				y5[i] = ddfx
			}
			y6[i] = o.DI(2, x)
		}
		if option == 6 {
			y7[i] = o.DI(p, x)
		}
	}

	// plot
	if option == 2 {
		plt.Subplot(2, 1, 1)
	}
	if option == 3 {
		plt.Subplot(3, 1, 1)
	}
	if withF {
		if f != nil {
			plt.Plot(xx, y1, argsF)
		}
		plt.Plot(xx, y2, argsI)
		plt.HideTRborders()
		plt.Gll("$x$", "$f(x)$", nil)
	}
	if option == 2 {
		plt.Subplot(2, 1, 2)
	}
	if option == 3 {
		plt.Subplot(3, 1, 2)
	}
	if firstD {
		argsF.L = "dfdx"
		//plt.Plot(X, yX, argsX)
		if dfdx != nil {
			plt.Plot(xx, y3, argsF)
		}
		plt.Plot(xx, y4, argsD1)
		plt.HideTRborders()
		plt.Gll("$x$", "$\\frac{\\mathrm{d}f(x)}{\\mathrm{d}x}$", nil)
	}
	if option == 3 {
		plt.Subplot(3, 1, 3)
	}
	if secondD {
		argsF.L = "d2fdx2"
		if d2fdx2 != nil {
			plt.Plot(xx, y5, argsF)
		}
		plt.Plot(xx, y6, argsD2)
		plt.HideTRborders()
		plt.Gll("$x$", "$\\frac{\\mathrm{d}^2f(x)}{\\mathrm{d}x^2}$", nil)
	}
	if option == 6 {
		argsI.L = io.Sf("D%d", p)
		plt.Plot(xx, y7, argsI)
		plt.HideTRborders()
		plt.Gll("$x$", io.Sf("$\\frac{\\mathrm{d}^{%d}f(x)}{\\mathrm{d}x^{%d}}$", p, p), nil)
	}
}
<commit_msg>Save grid coordinates in FourierInterp<commit_after>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fun

import (
	"math"
	"math/cmplx"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/plt"
	"github.com/cpmech/gosl/utl"
)

// Smoothing kinds
var (

	// No smoothing
	SmoNoneKind = io.NewEnum("None", "fun.smoothing", "L", "No smoothing")

	// Lanczos (sinc) smoothing kind
	SmoLanczosKind = io.NewEnum("Lanczos", "fun.smoothing", "L", "Lanczos (sinc) smoothing kind")

	// Cesaro
	SmoCesaroKind = io.NewEnum("Cesaro", "fun.smoothing", "L", "Lanczos (sinc) smoothing kind")

	// Raised Cosine
	SmoRcosKind = io.NewEnum("Rcos", "fun.smoothing", "L", "Lanczos (sinc) smoothing kind")
)

// FourierInterp performs interpolation using truncated Fourier series
//
//                  N/2 - 1
//                   ————          +i k X[j]
//        f(x[j]) =  \     A[k] ⋅ e              with    X[j] = 2 π j / N
//                   /
//                   ————
//                  k = -N/2
//
//   x ϵ [0, 2π]
//
//   Equation (2.1.27) of [1]. Note that f=u in [1] and A[k] is the tilde(u[k]) of [1]
//
//   Reference:
//     [1] Canuto C, Hussaini MY, Quarteroni A, Zang TA (2006) Spectral Methods: Fundamentals in
//         Single Domains. Springer. 563p
//
type FourierInterp struct {
	N int          // number of terms. must be power of 2; i.e. N = 2ⁿ
	X []float64    // point coordinates == 2⋅π.j/N
	A []complex128 // coefficients for interpolation. from FFT
	S []complex128 // smothing coefficients
}

// NewFourierInterp allocates a new FourierInterp object
//
//                    N - 1
//                1   ————             -i k X[j]
//        A[k] = ———  \     f(x[j]) ⋅ e              with    X[j] = 2 π j / N
//                N   /
//                    ————
//                   j = 0
//
//  INPUT:
//    N -- number of terms. must be power of 2; i.e. N = 2ⁿ
//
//    smoothing -- type of smoothing: use SmoNoneKind for no smoothing
//
//  NOTE: (1) x ϵ [0, 2π]
//        (2) remember to call CalcA to calculate coefficients A!
//
//   Equation (2.1.25) of [1]. Note that f=u in [1] and A[k] is the tilde(u[k]) of [1]
//
//   Reference:
//     [1] Canuto C, Hussaini MY, Quarteroni A, Zang TA (2006) Spectral Methods: Fundamentals in
//         Single Domains. Springer. 563p
//
func NewFourierInterp(N int, smoothing io.Enum) (o *FourierInterp, err error) {

	// check
	if N%2 != 0 {
		err = chk.Err("N must be even. N=%d is invalid\n", N)
		return
	}

	// allocate
	o = new(FourierInterp)
	o.N = N
	o.X = make([]float64, o.N)
	o.A = make([]complex128, o.N)
	o.S = make([]complex128, o.N)

	// point coordinates
	n := float64(o.N)
	for j := 0; j < o.N; j++ {
		o.X[j] = 2.0 * math.Pi * float64(j) / n
	}

	// compute smoothing coefficients
	σ := func(k float64) float64 { return 1.0 }
	switch smoothing {
	case SmoLanczosKind:
		σ = func(k float64) float64 { return Sinc(2 * k * π / n) }
	case SmoRcosKind:
		σ = func(k float64) float64 { return (1.0 + math.Cos(2*k*π/n)) / 2.0 }
	case SmoCesaroKind:
		σ = func(k float64) float64 { return 1.0 - math.Abs(k)/(1.0+n/2.0) }
	}

	for j := 0; j < o.N; j++ {
		o.S[j] = complex(σ(o.K(j)), 0)
	}
	return
}

// CalcA calculates the coefficients A of the interpolation ousing FFT
func (o *FourierInterp) CalcA(f Ss) (err error) {

	// compute f(x[j]) and set A[j] with f(x[j]) / N
	var fxj float64
	n := float64(o.N)
	for j := 0; j < o.N; j++ {
		fxj, err = f(o.X[j])
		if err != nil {
			return
		}
		o.A[j] = complex(fxj/n, 0)
	}

	// perform Fourier transform to find A[j]
	err = Dft1d(o.A, false)
	return
}

// K computes k-index from j-index where j corresponds to the FFT index
//
//   FFT returns the A coefficients as:
//
//      {A[0], A[1], ..., A[N/2-1], A[-N/2], A[-N/2+1], ... A[-1]}
//
//   k ϵ [-N/2, N/2-1]
//   j ϵ [0, N-1]
//
//   Example with N = 8:
//
//        j=0 ⇒ k=0      j=4 ⇒ k=-4
//        j=1 ⇒ k=1      j=5 ⇒ k=-3
//        j=2 ⇒ k=2      j=6 ⇒ k=-2
//        j=3 ⇒ k=3      j=7 ⇒ k=-1
//
func (o *FourierInterp) K(j int) float64 {
	h := o.N / 2
	k := j - (j/h)*o.N
	return float64(k)
}

// I computes the interpolation
//
//                  N/2 - 1
//                    ————          +i k x
//        I {f}(x) =  \     A[k] ⋅ e
//         N          /
//                    ————
//                   k = -N/2
//
//   x ϵ [0, 2π]
//
//   Equation (2.1.28) of [1]. Note that f=u in [1] and A[k] is the tilde(u[k]) of [1]
//
//   Reference:
//     [1] Canuto C, Hussaini MY, Quarteroni A, Zang TA (2006) Spectral Methods: Fundamentals in
//         Single Domains. Springer. 563p
//
//  NOTE: remember to call CalcA to calculate coefficients A!
//
func (o *FourierInterp) I(x float64) float64 {
	var res complex128
	for j := 0; j < o.N; j++ {
		res += o.S[j] * o.A[j] * cmplx.Exp(complex(0, o.K(j)*x))
	}
	return real(res)
}

// DI computes the p-derivative of the interpolation
//
//                   p       N/2 - 1
//        p         d(I{f})    ————       p           +i k x
//       DI{f}(x) = ——————— =  \     (i⋅k)  ⋅ A[k] ⋅ e
//        N             p      /
//                    dx       ————
//                            k = -N/2
//
//   x ϵ [0, 2π]
//
//  NOTE: remember to call CalcA to calculate coefficients A!
//
func (o *FourierInterp) DI(p int, x float64) float64 {
	var res complex128
	pc := complex(float64(p), 0)
	for j := 0; j < o.N; j++ {
		ik := complex(0, o.K(j))
		ikp := cmplx.Pow(ik, pc)
		res += ikp * o.S[j] * o.A[j] * cmplx.Exp(complex(0, o.K(j)*x))
	}
	return real(res)
}

// Plot plots interpolated curve
//   option -- 1: plot only f(x)
//             2: plot both f(x) and df/dx(x)
//             3: plot all f(x), df/dx(x) and d^2f/dx^2
//             4: plot only df/dx(x)
//             5: plot only d^2f/dx^2(x)
//             6: plot df^p/dx^p
//   p      -- order of the derivative to plot if option == 6
//   dfdx   -- is the analytic df/dx(x) [optional]
//   d2fdx2 -- is the analytic d^2f/dx^2(x) [optional]
func (o *FourierInterp) Plot(option, p int, f, dfdx, d2fdx2 Ss, argsF, argsI, argsD1, argsD2 *plt.A) {

	// set arguments
	if argsF == nil {
		argsF = &plt.A{L: "f(x)", C: plt.C(0, 1), NoClip: true}
	}
	if argsI == nil {
		argsI = &plt.A{L: "I{f}(x)", C: plt.C(1, 1), NoClip: true}
	}
	if argsD1 == nil {
		argsD1 = &plt.A{L: "D1I{f}(x)", C: plt.C(2, 1), NoClip: true}
	}
	if argsD2 == nil {
		argsD2 = &plt.A{L: "D2I{f}(x)", C: plt.C(3, 1), NoClip: true}
	}

	// graph points
	npts := 2001
	xx := utl.LinSpace(0, 2.0*math.Pi, npts)

	// options
	withF := option == 1 || option == 2 || option == 3
	firstD := option == 2 || option == 3 || option == 4
	secondD := option == 3 || option == 5

	// allocate arrays
	var y1, y2 []float64
	if withF {
		y1 = make([]float64, npts)
		y2 = make([]float64, npts)
	}
	var y3, y4 []float64
	if firstD {
		y3 = make([]float64, npts)
		y4 = make([]float64, npts)
	}
	var y5, y6 []float64
	if secondD {
		y5 = make([]float64, npts)
		y6 = make([]float64, npts)
	}
	var y7 []float64
	if option == 6 {
		y7 = make([]float64, npts)
	}

	// compute values
	for i := 0; i < npts; i++ {
		x := xx[i]
		if withF {
			if f != nil {
				fx, err := f(x)
				if err != nil {
					chk.Panic("f(x) failed:\n%v\n", err)
				}
				y1[i] = fx
			}
			y2[i] = o.I(x)
		}
		if firstD {
			if dfdx != nil {
				dfx, err := dfdx(x)
				if err != nil {
					chk.Panic("df/dx(x) failed:\n%v\n", err)
				}
				y3[i] = dfx
			}
			y4[i] = o.DI(1, x)
		}
		if secondD {
			if d2fdx2 != nil {
				ddfx, err := d2fdx2(x)
				if err != nil {
					chk.Panic("d2f/dx2(x) failed:\n%v\n", err)
				}
				y5[i] = ddfx
			}
			y6[i] = o.DI(2, x)
		}
		if option == 6 {
			y7[i] = o.DI(p, x)
		}
	}

	// plot
	if option == 2 {
		plt.Subplot(2, 1, 1)
	}
	if option == 3 {
		plt.Subplot(3, 1, 1)
	}
	if withF {
		if f != nil {
			plt.Plot(xx, y1, argsF)
		}
		plt.Plot(xx, y2, argsI)
		plt.HideTRborders()
		plt.Gll("$x$", "$f(x)$", nil)
	}
	if option == 2 {
		plt.Subplot(2, 1, 2)
	}
	if option == 3 {
		plt.Subplot(3, 1, 2)
	}
	if firstD {
		argsF.L = "dfdx"
		//plt.Plot(X, yX, argsX)
		if dfdx != nil {
			plt.Plot(xx, y3, argsF)
		}
		plt.Plot(xx, y4, argsD1)
		plt.HideTRborders()
		plt.Gll("$x$", "$\\frac{\\mathrm{d}f(x)}{\\mathrm{d}x}$", nil)
	}
	if option == 3 {
		plt.Subplot(3, 1, 3)
	}
	if secondD {
		argsF.L = "d2fdx2"
		if d2fdx2 != nil {
			plt.Plot(xx, y5, argsF)
		}
		plt.Plot(xx, y6, argsD2)
		plt.HideTRborders()
		plt.Gll("$x$", "$\\frac{\\mathrm{d}^2f(x)}{\\mathrm{d}x^2}$", nil)
	}
	if option == 6 {
		argsI.L = io.Sf("D%d", p)
		plt.Plot(xx, y7, argsI)
		plt.HideTRborders()
		plt.Gll("$x$", io.Sf("$\\frac{\\mathrm{d}^{%d}f(x)}{\\mathrm{d}x^{%d}}$", p, p), nil)
	}
}
<|endoftext|>


<commit_before>package v1alpha1

import (
	"fmt"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	CRDKind       = "TrainingJob"
	CRDKindPlural = "trainingjobs"
	CRDShortName  = "tj"
	CRDGroup      = "paddlepaddle.org"
	CRDVersion    = "v1alpha1"
)

// CRDName returns name of crd
func CRDName() string {
	return fmt.Sprintf("%s.%s", CRDKindPlural, CRDGroup)
}

// +genclient
// +genclient:noStatus
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +resource:path=trainingjob

// TrainingJob is a specification for a TrainingJob resource
type TrainingJob struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`
	Spec              TrainingJobSpec   `json:"spec"`
	Status            TrainingJobStatus `json:"status"`
}

// TrainingJobSpec is the spec for a TrainingJob resource
type TrainingJobSpec struct {
	// General job attributes.
	Image string `json:"image,omitempty"`
	// If you want to use the hostnetwork instead of container network
	// portmanager is necessary.
	HostNetwork       bool                `json:"host_network,omitempty"`
	Port              int                 `json:"port,omitempty"`
	PortsNum          int                 `json:"ports_num,omitempty"`
	PortsNumForSparse int                 `json:"ports_num_for_sparse,omitempty"`
	FaultTolerant     bool                `json:"fault_tolerant,omitempty"`
	Passes            int                 `json:"passes,omitempty"`
	Volumes           []apiv1.Volume      `json:"volumes"`
	VolumeMounts      []apiv1.VolumeMount `json:"VolumeMounts"`
	//TrainingJob components.
	Master  MasterSpec  `json:"master"`
	Pserver PserverSpec `json:"pserver"`
	Trainer TrainerSpec `json:"trainer"`
}

// MasterSpec is the spec for a master in the paddle job
type MasterSpec struct {
	EtcdEndpoint string                     `json:"etcd-endpoint"`
	Resources    apiv1.ResourceRequirements `json:"resources"`
}

// PserverSpec is the spec for pservers in the paddle job
type PserverSpec struct {
	MinInstance int                        `json:"min-instance"`
	MaxInstance int                        `json:"max-instance"`
	Resources   apiv1.ResourceRequirements `json:"resources"`
}

// TrainerSpec is the spec for trainers in the paddle job
type TrainerSpec struct {
	EtcdEndpoint string                     `json:"etcd-endpoint"`
	Entrypoint   string                     `json:"entrypoint"`
	Workspace    string                     `json:"workspace"`
	MinInstance  int                        `json:"min-instance"`
	MaxInstance  int                        `json:"max-instance"`
	Resources    apiv1.ResourceRequirements `json:"resources"`
}

// TrainingJobPhase is the phase of TrainingJob
type TrainingJobPhase string

const (
	TrainingJobPhaseNone      TrainingJobPhase = ""
	TrainingJobPhaseCreating                   = "creating"
	TrainingJobPhaseRunning                    = "running"
	TrainingJobPhaseSucceeded                  = "succeeded"
	TrainingJobPhaseFailed                     = "failed"
)

// TrainerJobScaleStatus is status of trainer jobs.
type TrainerJobScaleStatus struct {
}

// TrainingResourceType the type of TrainingJob resource, include MASTER PSERVER and TRAINER
type TrainingResourceType string

const (
	MASTER  TrainingResourceType = "MASTER"
	PSERVER TrainingResourceType = "PSERVER"
	TRAINER TrainingResourceType = "TRAINER"
)

// ResourceState is the state of a type of resource
type ResourceState string

const (
	ResourceStateNone      ResourceState = ""
	ResourceStateStarting                = "starting"
	ResourceStateRunning                 = "running"
	ResourceStateFailed                  = "failed"
	ResourceStateSucceeded               = "succeeded"
)

// TrainingResourceStatus is the status of every resource
type TrainingResourceStatus struct {
	// TrainingResourceType the type of TrainingJob resource, include MASTER PSERVER and TRAINER
	TrainingResourceType `json:"training_resource_type"`
	// State is the state of a type of resource
	State ResourceState `json:"state"`
	// ResourceStates is the number of resource in different state
	ResourceStates map[ResourceState]int `json:"resource_states"`
}

// TrainingJobStatus is the status for a TrainingJob resource.
type TrainingJobStatus struct {
	// Phase is phase of TrainingJob
	Phase TrainingJobPhase `json:"phase"`
	// Reason is the reason of job phase failed
	Reason string `json:"reason"`
	// ScaleStatus is autoscale status of trainer jobs
	// TODO(ZhengQi): this will used in autoscale mode in future.
	ScaleStatus TrainerJobScaleStatus `json:"scale_status"`
	// ReplicaStatuses is detail status of resources
	// TODO(ZhengQi): should we only considered trainer job now?
	ReplicaStatuses []*TrainingResourceStatus `json:"replica_statuses"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +resource:path=trainingjobs

// TrainingJobList is a list of TrainingJob resources
type TrainingJobList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	// Items means the list of paddle job/TrainingJob
	Items []TrainingJob `json:"items"`
}
<commit_msg>add a ref for portmanager<commit_after>package v1alpha1

import (
	"fmt"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	CRDKind       = "TrainingJob"
	CRDKindPlural = "trainingjobs"
	CRDShortName  = "tj"
	CRDGroup      = "paddlepaddle.org"
	CRDVersion    = "v1alpha1"
)

// CRDName returns name of crd
func CRDName() string {
	return fmt.Sprintf("%s.%s", CRDKindPlural, CRDGroup)
}

// +genclient
// +genclient:noStatus
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +resource:path=trainingjob

// TrainingJob is a specification for a TrainingJob resource
type TrainingJob struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`
	Spec              TrainingJobSpec   `json:"spec"`
	Status            TrainingJobStatus `json:"status"`
}

// TrainingJobSpec is the spec for a TrainingJob resource
type TrainingJobSpec struct {
	// General job attributes.
	Image string `json:"image,omitempty"`
	// If you want to use the hostnetwork instead of container network
	// portmanager is necessary. About portmanager, please refer to
	// https://github.com/PaddlePaddle/cloud/blob/develop/doc/hostnetwork/hostnetwork.md
	HostNetwork       bool                `json:"host_network,omitempty"`
	Port              int                 `json:"port,omitempty"`
	PortsNum          int                 `json:"ports_num,omitempty"`
	PortsNumForSparse int                 `json:"ports_num_for_sparse,omitempty"`
	FaultTolerant     bool                `json:"fault_tolerant,omitempty"`
	Passes            int                 `json:"passes,omitempty"`
	Volumes           []apiv1.Volume      `json:"volumes"`
	VolumeMounts      []apiv1.VolumeMount `json:"VolumeMounts"`
	//TrainingJob components.
	Master  MasterSpec  `json:"master"`
	Pserver PserverSpec `json:"pserver"`
	Trainer TrainerSpec `json:"trainer"`
}

// MasterSpec is the spec for a master in the paddle job
type MasterSpec struct {
	EtcdEndpoint string                     `json:"etcd-endpoint"`
	Resources    apiv1.ResourceRequirements `json:"resources"`
}

// PserverSpec is the spec for pservers in the paddle job
type PserverSpec struct {
	MinInstance int                        `json:"min-instance"`
	MaxInstance int                        `json:"max-instance"`
	Resources   apiv1.ResourceRequirements `json:"resources"`
}

// TrainerSpec is the spec for trainers in the paddle job
type TrainerSpec struct {
	EtcdEndpoint string                     `json:"etcd-endpoint"`
	Entrypoint   string                     `json:"entrypoint"`
	Workspace    string                     `json:"workspace"`
	MinInstance  int                        `json:"min-instance"`
	MaxInstance  int                        `json:"max-instance"`
	Resources    apiv1.ResourceRequirements `json:"resources"`
}

// TrainingJobPhase is the phase of TrainingJob
type TrainingJobPhase string

const (
	TrainingJobPhaseNone      TrainingJobPhase = ""
	TrainingJobPhaseCreating                   = "creating"
	TrainingJobPhaseRunning                    = "running"
	TrainingJobPhaseSucceeded                  = "succeeded"
	TrainingJobPhaseFailed                     = "failed"
)

// TrainerJobScaleStatus is status of trainer jobs.
type TrainerJobScaleStatus struct {
}

// TrainingResourceType the type of TrainingJob resource, include MASTER PSERVER and TRAINER
type TrainingResourceType string

const (
	MASTER  TrainingResourceType = "MASTER"
	PSERVER TrainingResourceType = "PSERVER"
	TRAINER TrainingResourceType = "TRAINER"
)

// ResourceState is the state of a type of resource
type ResourceState string

const (
	ResourceStateNone      ResourceState = ""
	ResourceStateStarting                = "starting"
	ResourceStateRunning                 = "running"
	ResourceStateFailed                  = "failed"
	ResourceStateSucceeded               = "succeeded"
)

// TrainingResourceStatus is the status of every resource
type TrainingResourceStatus struct {
	// TrainingResourceType the type of TrainingJob resource, include MASTER PSERVER and TRAINER
	TrainingResourceType `json:"training_resource_type"`
	// State is the state of a type of resource
	State ResourceState `json:"state"`
	// ResourceStates is the number of resource in different state
	ResourceStates map[ResourceState]int `json:"resource_states"`
}

// TrainingJobStatus is the status for a TrainingJob resource.
type TrainingJobStatus struct {
	// Phase is phase of TrainingJob
	Phase TrainingJobPhase `json:"phase"`
	// Reason is the reason of job phase failed
	Reason string `json:"reason"`
	// ScaleStatus is autoscale status of trainer jobs
	// TODO(ZhengQi): this will used in autoscale mode in future.
	ScaleStatus TrainerJobScaleStatus `json:"scale_status"`
	// ReplicaStatuses is detail status of resources
	// TODO(ZhengQi): should we only considered trainer job now?
	ReplicaStatuses []*TrainingResourceStatus `json:"replica_statuses"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +resource:path=trainingjobs

// TrainingJobList is a list of TrainingJob resources
type TrainingJobList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	// Items means the list of paddle job/TrainingJob
	Items []TrainingJob `json:"items"`
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"fmt"
	"socialapi/request"
	"time"

	"github.com/koding/bongo"
)

type Interaction struct {
	// unique identifier of the Interaction
	Id int64 `json:"id,string"`

	// Id of the interacted message
	MessageId int64 `json:"messageId,string"      sql:"NOT NULL"`

	// Id of the actor
	AccountId int64 `json:"accountId,string"      sql:"NOT NULL"`

	// holds troll, unsafe, etc
	MetaBits MetaBits `json:"-"`

	// Type of the interaction
	TypeConstant string `json:"typeConstant"      sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// Creation of the interaction
	CreatedAt time.Time `json:"createdAt"         sql:"NOT NULL"`
}

var AllowedInteractions = map[string]struct{}{
	"like":     struct{}{},
	"upvote":   struct{}{},
	"downvote": struct{}{},
}

const (
	Interaction_TYPE_LIKE     = "like"
	Interaction_TYPE_UPVOTE   = "upvote"
	Interaction_TYPE_DONWVOTE = "downvote"
)

func (i Interaction) GetId() int64 {
	return i.Id
}

func (i Interaction) TableName() string {
	return "api.interaction"
}

func NewInteraction() *Interaction {
	return &Interaction{}
}

func (i *Interaction) One(q *bongo.Query) error {
	return bongo.B.One(i, i, q)
}

func (i *Interaction) ById(id int64) error {
	return bongo.B.ById(i, id)
}

func (i *Interaction) Create() error {
	return bongo.B.Create(i)
}

func (i *Interaction) Update() error {
	return bongo.B.Update(i)
}

func (i *Interaction) CreateRaw() error {
	insertSql := "INSERT INTO " +
		i.TableName() +
		` ("message_id","account_id","type_constant","created_at") VALUES ($1,$2,$3,$4) ` +
		"RETURNING ID"

	return bongo.B.DB.CommonDB().
		QueryRow(insertSql, i.MessageId, i.AccountId, i.TypeConstant, i.CreatedAt).
		Scan(&i.Id)
}

func (i *Interaction) AfterCreate() {
	bongo.B.AfterCreate(i)
}

func (i *Interaction) AfterUpdate() {
	bongo.B.AfterUpdate(i)
}

func (i Interaction) AfterDelete() {
	bongo.B.AfterDelete(i)
}

func (i *Interaction) BeforeCreate() error {
	return i.MarkIfExempt()
}

func (i *Interaction) BeforeUpdate() error {
	return i.MarkIfExempt()
}

func (i *Interaction) MarkIfExempt() error {
	isExempt, err := i.isExempt()
	if err != nil {
		return err
	}

	if isExempt {
		i.MetaBits.MarkTroll()
	}

	return nil
}

func (i *Interaction) isExempt() (bool, error) {
	if i.MetaBits.IsTroll() {
		return true, nil
	}

	accountId, err := i.getAccountId()
	if err != nil {
		return false, err
	}

	account, err := ResetAccountCache(accountId)
	if err != nil {
		return false, err
	}

	if account == nil {
		return false, fmt.Errorf("account is nil, accountId:%d", i.AccountId)
	}

	if account.IsTroll {
		return true, nil
	}

	return false, nil
}

func (i *Interaction) getAccountId() (int64, error) {
	if i.AccountId != 0 {
		return i.AccountId, nil
	}

	if i.Id == 0 {
		return 0, fmt.Errorf("couldnt find accountId from content %+v", i)
	}

	ii := NewInteraction()
	if err := ii.ById(i.Id); err != nil {
		return 0, err
	}

	return ii.AccountId, nil
}

func (i *Interaction) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(i, data, q)
}

func (i *Interaction) Delete() error {
	selector := map[string]interface{}{
		"message_id": i.MessageId,
		"account_id": i.AccountId,
	}

	if err := i.One(bongo.NewQS(selector)); err != nil {
		return err
	}

	if err := bongo.B.Delete(i); err != nil {
		return err
	}

	return nil
}

func (i *Interaction) List(query *request.Query) ([]int64, error) {
	var interactions []int64

	if i.MessageId == 0 {
		return interactions, errors.New("Message is not set")
	}

	return i.FetchInteractorIds(query)
}

func (i *Interaction) FetchInteractorIds(query *request.Query) ([]int64, error) {
	interactorIds := make([]int64, 0)
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"message_id":    i.MessageId,
			"type_constant": query.Type,
		},
		Pagination: *bongo.NewPagination(query.Limit, query.Skip),
		Pluck:      "account_id",
		Sort: map[string]string{
			"created_at": "desc",
		},
	}

	q.AddScope(RemoveTrollContent(i, query.ShowExempt))

	if err := i.Some(&interactorIds, q); err != nil {
		// TODO log this error
		return make([]int64, 0), nil
	}

	return interactorIds, nil
}

func (c *Interaction) Count(q *request.Query) (int, error) {
	if c.MessageId == 0 {
		return 0, errors.New("messageId is not set")
	}

	if q.Type == "" {
		return 0, errors.New("query type is not set")
	}

	query := &bongo.Query{
		Selector: map[string]interface{}{
			"message_id":    c.MessageId,
			"type_constant": q.Type,
		},
	}

	query.AddScope(RemoveTrollContent(
		c, q.ShowExempt,
	))

	return c.CountWithQuery(query)
}

func (c *Interaction) CountWithQuery(q *bongo.Query) (int, error) {
	return bongo.B.CountWithQuery(c, q)
}

func (c *Interaction) FetchAll(interactionType string) ([]Interaction, error) {
	var interactions []Interaction

	if c.MessageId == 0 {
		return interactions, errors.New("ChannelId is not set")
	}

	selector := map[string]interface{}{
		"message_id":    c.MessageId,
		"type_constant": interactionType,
	}

	err := c.Some(&interactions, bongo.NewQS(selector))
	if err != nil {
		return interactions, err
	}

	return interactions, nil
}

func (i *Interaction) IsInteracted(accountId int64) (bool, error) {
	if i.MessageId == 0 {
		return false, errors.New("Message Id is not set")
	}

	selector := map[string]interface{}{
		"message_id": i.MessageId,
		"account_id": accountId,
	}

	// do not set
	err := NewInteraction().One(bongo.NewQS(selector))
	if err == nil {
		return true, nil
	}

	if err == bongo.RecordNotFound {
		return false, nil
	}

	return false, err
}

func (i *Interaction) FetchInteractorCount() (int, error) {
	return bongo.B.Count(i, "message_id = ?", i.MessageId)
}
<commit_msg>social: change ordering<commit_after>package models

import (
	"errors"
	"fmt"
	"socialapi/request"
	"time"

	"github.com/koding/bongo"
)

type Interaction struct {
	// unique identifier of the Interaction
	Id int64 `json:"id,string"`

	// Id of the interacted message
	MessageId int64 `json:"messageId,string"      sql:"NOT NULL"`

	// Id of the actor
	AccountId int64 `json:"accountId,string"      sql:"NOT NULL"`

	// holds troll, unsafe, etc
	MetaBits MetaBits `json:"-"`

	// Type of the interaction
	TypeConstant string `json:"typeConstant"      sql:"NOT NULL;TYPE:VARCHAR(100);"`

	// Creation of the interaction
	CreatedAt time.Time `json:"createdAt"         sql:"NOT NULL"`
}

var AllowedInteractions = map[string]struct{}{
	"like":     struct{}{},
	"upvote":   struct{}{},
	"downvote": struct{}{},
}

const (
	Interaction_TYPE_LIKE     = "like"
	Interaction_TYPE_UPVOTE   = "upvote"
	Interaction_TYPE_DONWVOTE = "downvote"
)

func (i *Interaction) BeforeCreate() error {
	return i.MarkIfExempt()
}

func (i *Interaction) BeforeUpdate() error {
	return i.MarkIfExempt()
}

func (i *Interaction) AfterCreate() {
	bongo.B.AfterCreate(i)
}

func (i *Interaction) AfterUpdate() {
	bongo.B.AfterUpdate(i)
}

func (i Interaction) AfterDelete() {
	bongo.B.AfterDelete(i)
}

func (i Interaction) GetId() int64 {
	return i.Id
}

func (i Interaction) TableName() string {
	return "api.interaction"
}

func NewInteraction() *Interaction {
	return &Interaction{}
}

func (i *Interaction) One(q *bongo.Query) error {
	return bongo.B.One(i, i, q)
}

func (i *Interaction) ById(id int64) error {
	return bongo.B.ById(i, id)
}

func (i *Interaction) Create() error {
	return bongo.B.Create(i)
}

func (i *Interaction) Update() error {
	return bongo.B.Update(i)
}

func (i *Interaction) CreateRaw() error {
	insertSql := "INSERT INTO " +
		i.TableName() +
		` ("message_id","account_id","type_constant","created_at") VALUES ($1,$2,$3,$4) ` +
		"RETURNING ID"

	return bongo.B.DB.CommonDB().
		QueryRow(insertSql, i.MessageId, i.AccountId, i.TypeConstant, i.CreatedAt).
		Scan(&i.Id)
}

func (i *Interaction) MarkIfExempt() error {
	isExempt, err := i.isExempt()
	if err != nil {
		return err
	}

	if isExempt {
		i.MetaBits.MarkTroll()
	}

	return nil
}

func (i *Interaction) isExempt() (bool, error) {
	if i.MetaBits.IsTroll() {
		return true, nil
	}

	accountId, err := i.getAccountId()
	if err != nil {
		return false, err
	}

	account, err := ResetAccountCache(accountId)
	if err != nil {
		return false, err
	}

	if account == nil {
		return false, fmt.Errorf("account is nil, accountId:%d", i.AccountId)
	}

	if account.IsTroll {
		return true, nil
	}

	return false, nil
}

func (i *Interaction) getAccountId() (int64, error) {
	if i.AccountId != 0 {
		return i.AccountId, nil
	}

	if i.Id == 0 {
		return 0, fmt.Errorf("couldnt find accountId from content %+v", i)
	}

	ii := NewInteraction()
	if err := ii.ById(i.Id); err != nil {
		return 0, err
	}

	return ii.AccountId, nil
}

func (i *Interaction) Some(data interface{}, q *bongo.Query) error {
	return bongo.B.Some(i, data, q)
}

func (i *Interaction) Delete() error {
	selector := map[string]interface{}{
		"message_id": i.MessageId,
		"account_id": i.AccountId,
	}

	if err := i.One(bongo.NewQS(selector)); err != nil {
		return err
	}

	if err := bongo.B.Delete(i); err != nil {
		return err
	}

	return nil
}

func (i *Interaction) List(query *request.Query) ([]int64, error) {
	var interactions []int64

	if i.MessageId == 0 {
		return interactions, errors.New("Message is not set")
	}

	return i.FetchInteractorIds(query)
}

func (i *Interaction) FetchInteractorIds(query *request.Query) ([]int64, error) {
	interactorIds := make([]int64, 0)
	q := &bongo.Query{
		Selector: map[string]interface{}{
			"message_id":    i.MessageId,
			"type_constant": query.Type,
		},
		Pagination: *bongo.NewPagination(query.Limit, query.Skip),
		Pluck:      "account_id",
		Sort: map[string]string{
			"created_at": "desc",
		},
	}

	q.AddScope(RemoveTrollContent(i, query.ShowExempt))

	if err := i.Some(&interactorIds, q); err != nil {
		// TODO log this error
		return make([]int64, 0), nil
	}

	return interactorIds, nil
}

func (c *Interaction) Count(q *request.Query) (int, error) {
	if c.MessageId == 0 {
		return 0, errors.New("messageId is not set")
	}

	if q.Type == "" {
		return 0, errors.New("query type is not set")
	}

	query := &bongo.Query{
		Selector: map[string]interface{}{
			"message_id":    c.MessageId,
			"type_constant": q.Type,
		},
	}

	query.AddScope(RemoveTrollContent(
		c, q.ShowExempt,
	))

	return c.CountWithQuery(query)
}

func (c *Interaction) CountWithQuery(q *bongo.Query) (int, error) {
	return bongo.B.CountWithQuery(c, q)
}

func (c *Interaction) FetchAll(interactionType string) ([]Interaction, error) {
	var interactions []Interaction

	if c.MessageId == 0 {
		return interactions, errors.New("ChannelId is not set")
	}

	selector := map[string]interface{}{
		"message_id":    c.MessageId,
		"type_constant": interactionType,
	}

	err := c.Some(&interactions, bongo.NewQS(selector))
	if err != nil {
		return interactions, err
	}

	return interactions, nil
}

func (i *Interaction) IsInteracted(accountId int64) (bool, error) {
	if i.MessageId == 0 {
		return false, errors.New("Message Id is not set")
	}

	selector := map[string]interface{}{
		"message_id": i.MessageId,
		"account_id": accountId,
	}

	// do not set
	err := NewInteraction().One(bongo.NewQS(selector))
	if err == nil {
		return true, nil
	}

	if err == bongo.RecordNotFound {
		return false, nil
	}

	return false, err
}

func (i *Interaction) FetchInteractorCount() (int, error) {
	return bongo.B.Count(i, "message_id = ?", i.MessageId)
}
<|endoftext|>


<commit_before>// Copyright 2016, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"math/rand"
	"sync"
	"testing"
	"time"

	"github.com/youtube/vitess/go/vt/discovery"
	"github.com/youtube/vitess/go/vt/logutil"
	"github.com/youtube/vitess/go/vt/servenv"
	"github.com/youtube/vitess/go/vt/tabletmanager/tmclient"
	"github.com/youtube/vitess/go/vt/tabletserver/grpcqueryservice"
	"github.com/youtube/vitess/go/vt/tabletserver/queryservice/fakes"
	"github.com/youtube/vitess/go/vt/throttler"
	"github.com/youtube/vitess/go/vt/vttest/fakesqldb"
	"github.com/youtube/vitess/go/vt/wrangler"
	"github.com/youtube/vitess/go/vt/wrangler/testlib"
	"github.com/youtube/vitess/go/vt/zktopo/zktestserver"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"

	log "github.com/golang/glog"
)

// This file contains a demo binary that demonstrates how the resharding
// throttler adapts its throttling rate to the replication lag.
//
// The throttler is necessary because replicas apply transactions at a slower
// rate than masters and fall behind at high write throughput.
// (Mostly they fall behind because MySQL replication is single threaded but
//  the write throughput on the master does not have to.)
//
// This demo simulates a client (writer), a master and a replica.
// The client writes to the master which in turn replicas everything to the
// replica.
// The replica measures its replication lag via the timestamp which is part of
// each message.
// While the master has no rate limit, the replica is limited to
// --rate (see below) transactions/second. The client runs the resharding
// throttler which tries to throttle the client based on the observed
// replication lag.

var (
	rate                     = flag.Int64("rate", 1000, "maximum rate of the throttled demo server at the start")
	duration                 = flag.Duration("duration", 600*time.Second, "total duration the demo runs")
	lagUpdateInterval        = flag.Duration("lag_update_interval", 1*time.Second, "interval at which the current replication lag will be broadcasted to the throttler")
	replicaDegrationInterval = flag.Duration("replica_degration_interval", 0*time.Second, "simulate a throughput degration of the replica every X interval (i.e. the replica applies transactions at a slower rate for -reparent_duration and the replication lag might go up)")
	replicaDegrationDuration = flag.Duration("replica_degration_duration", 10*time.Second, "duration a simulated degration should take")
)

// master simulates an *unthrottled* MySQL master which replicates every
// received "execute" call to a known "replica".
type master struct {
	replica *replica
}

// execute is the simulated RPC which is called by the client.
func (m *master) execute(msg time.Time) {
	m.replica.replicate(msg)
}

// replica simulates a *throttled* MySQL replica.
// If it cannot keep up with applying the master writes, it will report a
// replication lag > 0 seconds.
type replica struct {
	fakeTablet *testlib.FakeTablet
	qs         *fakes.StreamHealthQueryService

	// replicationStream is the incoming stream of messages from the master.
	replicationStream chan time.Time

	// throttler is used to enforce the maximum rate at which replica applies
	// transactions. It must not be confused with the client's throttler.
	throttler         *throttler.Throttler
	lastHealthUpdate  time.Time
	lagUpdateInterval time.Duration

	degrationInterval   time.Duration
	degrationDuration   time.Duration
	nextDegration       time.Time
	currentDegrationEnd time.Time

	stopChan chan struct{}
	wg       sync.WaitGroup
}

func newReplica(lagUpdateInterval, degrationInterval, degrationDuration time.Duration) *replica {
	t := &testing.T{}
	ts := zktestserver.New(t, []string{"cell1"})
	wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
	db := fakesqldb.Register()
	fakeTablet := testlib.NewFakeTablet(t, wr, "cell1", 0,
		topodatapb.TabletType_REPLICA, db, testlib.TabletKeyspaceShard(t, "ks", "-80"))
	fakeTablet.StartActionLoop(t, wr)

	target := querypb.Target{
		Keyspace:   "ks",
		Shard:      "-80",
		TabletType: topodatapb.TabletType_REPLICA,
	}
	qs := fakes.NewStreamHealthQueryService(target)
	grpcqueryservice.Register(fakeTablet.RPCServer, qs)

	throttler, err := throttler.NewThrottler("replica", "TPS", 1, *rate, throttler.ReplicationLagModuleDisabled)
	if err != nil {
		log.Fatal(err)
	}

	var nextDegration time.Time
	if degrationInterval != time.Duration(0) {
		nextDegration = time.Now().Add(degrationInterval)
	}
	r := &replica{
		fakeTablet:        fakeTablet,
		qs:                qs,
		throttler:         throttler,
		replicationStream: make(chan time.Time, 1*1024*1024),
		lagUpdateInterval: lagUpdateInterval,
		degrationInterval: degrationInterval,
		degrationDuration: degrationDuration,
		nextDegration:     nextDegration,
		stopChan:          make(chan struct{}),
	}
	r.wg.Add(1)
	go r.processReplicationStream()
	return r
}

func (r *replica) replicate(msg time.Time) {
	r.replicationStream <- msg
}

func (r *replica) processReplicationStream() {
	defer r.wg.Done()

	// actualRate counts the number of requests per r.lagUpdateInterval.
	actualRate := 0
	for msg := range r.replicationStream {
		select {
		case <-r.stopChan:
			return
		default:
		}

		now := time.Now()
		if now.Sub(r.lastHealthUpdate) > r.lagUpdateInterval {
			// Broadcast current lag every "lagUpdateInterval".
			//
			// Use integer values to calculate the lag. In consequence, the reported
			// lag will constantly vary between the floor and ceil value e.g.
			// an actual lag of 0.5s could be reported as 0s or 1s based on the
			// truncation of the two times.
			lagTruncated := uint32(now.Unix() - msg.Unix())
			// Display lag with a higher precision as well.
			lag := now.Sub(msg).Seconds()
			log.Infof("current lag: %1ds (%1.1fs) replica rate: % 7.1f chan len: % 6d", lagTruncated, lag, float64(actualRate)/r.lagUpdateInterval.Seconds(), len(r.replicationStream))
			r.qs.AddHealthResponseWithSecondsBehindMaster(lagTruncated)
			r.lastHealthUpdate = now
			actualRate = 0
		}
		if !r.nextDegration.IsZero() && time.Now().After(r.nextDegration) && r.currentDegrationEnd.IsZero() {
			degradedRate := rand.Int63n(*rate)
			log.Infof("degrading the replica for %.f seconds from %v TPS to %v", r.degrationDuration.Seconds(), *rate, degradedRate)
			r.throttler.SetMaxRate(degradedRate)
			r.currentDegrationEnd = time.Now().Add(r.degrationDuration)
		}
		if !r.currentDegrationEnd.IsZero() && time.Now().After(r.currentDegrationEnd) {
			log.Infof("degrading the replica stopped. Restoring TPS to: %v", *rate)
			r.throttler.SetMaxRate(*rate)
			r.currentDegrationEnd = time.Time{}
			r.nextDegration = time.Now().Add(r.degrationInterval)
		}

		for {
			backoff := r.throttler.Throttle(0 /* threadID */)
			if backoff == throttler.NotThrottled {
				break
			}
			time.Sleep(backoff)
		}
		actualRate++
	}
}

func (r *replica) stop() {
	close(r.replicationStream)
	close(r.stopChan)
	log.Info("Triggered replica shutdown. Waiting for it to stop.")
	r.wg.Wait()
	r.fakeTablet.StopActionLoop(&testing.T{})
}

// client simulates a client which should throttle itself based on the
// replication lag of all replicas.
type client struct {
	master *master

	healthCheck discovery.HealthCheck
	throttler   *throttler.Throttler

	stopChan chan struct{}
	wg       sync.WaitGroup
}

func newClient(master *master, replica *replica) *client {
	t, err := throttler.NewThrottler("client", "TPS", 1, throttler.MaxRateModuleDisabled, 5 /* seconds */)
	if err != nil {
		log.Fatal(err)
	}

	healthCheck := discovery.NewHealthCheck(1*time.Minute, 5*time.Second, 1*time.Minute)
	c := &client{
		master:      master,
		healthCheck: healthCheck,
		throttler:   t,
		stopChan:    make(chan struct{}),
	}
	c.healthCheck.SetListener(c, false /* sendDownEvents */)
	c.healthCheck.AddTablet(replica.fakeTablet.Tablet, "name")
	return c
}

func (c *client) run() {
	c.wg.Add(1)
	go c.loop()
}

func (c *client) loop() {
	defer c.wg.Done()

	for {
		select {
		case <-c.stopChan:
			return
		default:
		}

		for {
			backoff := c.throttler.Throttle(0 /* threadID */)
			if backoff == throttler.NotThrottled {
				break
			}
			time.Sleep(backoff)
		}

		c.master.execute(time.Now())
	}
}

func (c *client) stop() {
	close(c.stopChan)
	c.wg.Wait()

	c.healthCheck.Close()
	c.throttler.Close()
}

// StatsUpdate implements discovery.HealthCheckStatsListener.
// It gets called by the healthCheck instance every time a tablet broadcasts
// a health update.
func (c *client) StatsUpdate(ts *discovery.TabletStats) {
	c.throttler.RecordReplicationLag(time.Now(), ts)
}

func main() {
	flag.Parse()

	go servenv.RunDefault()

	log.Infof("start rate set to: %v", *rate)
	replica := newReplica(*lagUpdateInterval, *replicaDegrationInterval, *replicaDegrationDuration)
	master := &master{replica: replica}
	client := newClient(master, replica)
	client.run()

	time.Sleep(*duration)
	client.stop()
	replica.stop()
}

func init() {
	servenv.RegisterDefaultFlags()
}
<commit_msg>throttler: demo: Increase the lag broadcast interval from 1s to 5s.<commit_after>// Copyright 2016, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"math/rand"
	"sync"
	"testing"
	"time"

	"github.com/youtube/vitess/go/vt/discovery"
	"github.com/youtube/vitess/go/vt/logutil"
	"github.com/youtube/vitess/go/vt/servenv"
	"github.com/youtube/vitess/go/vt/tabletmanager/tmclient"
	"github.com/youtube/vitess/go/vt/tabletserver/grpcqueryservice"
	"github.com/youtube/vitess/go/vt/tabletserver/queryservice/fakes"
	"github.com/youtube/vitess/go/vt/throttler"
	"github.com/youtube/vitess/go/vt/vttest/fakesqldb"
	"github.com/youtube/vitess/go/vt/wrangler"
	"github.com/youtube/vitess/go/vt/wrangler/testlib"
	"github.com/youtube/vitess/go/vt/zktopo/zktestserver"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"

	log "github.com/golang/glog"
)

// This file contains a demo binary that demonstrates how the resharding
// throttler adapts its throttling rate to the replication lag.
//
// The throttler is necessary because replicas apply transactions at a slower
// rate than masters and fall behind at high write throughput.
// (Mostly they fall behind because MySQL replication is single threaded but
//  the write throughput on the master does not have to.)
//
// This demo simulates a client (writer), a master and a replica.
// The client writes to the master which in turn replicas everything to the
// replica.
// The replica measures its replication lag via the timestamp which is part of
// each message.
// While the master has no rate limit, the replica is limited to
// --rate (see below) transactions/second. The client runs the resharding
// throttler which tries to throttle the client based on the observed
// replication lag.

var (
	rate                     = flag.Int64("rate", 1000, "maximum rate of the throttled demo server at the start")
	duration                 = flag.Duration("duration", 600*time.Second, "total duration the demo runs")
	lagUpdateInterval        = flag.Duration("lag_update_interval", 5*time.Second, "interval at which the current replication lag will be broadcasted to the throttler")
	replicaDegrationInterval = flag.Duration("replica_degration_interval", 0*time.Second, "simulate a throughput degration of the replica every X interval (i.e. the replica applies transactions at a slower rate for -reparent_duration and the replication lag might go up)")
	replicaDegrationDuration = flag.Duration("replica_degration_duration", 10*time.Second, "duration a simulated degration should take")
)

// master simulates an *unthrottled* MySQL master which replicates every
// received "execute" call to a known "replica".
type master struct {
	replica *replica
}

// execute is the simulated RPC which is called by the client.
func (m *master) execute(msg time.Time) {
	m.replica.replicate(msg)
}

// replica simulates a *throttled* MySQL replica.
// If it cannot keep up with applying the master writes, it will report a
// replication lag > 0 seconds.
type replica struct {
	fakeTablet *testlib.FakeTablet
	qs         *fakes.StreamHealthQueryService

	// replicationStream is the incoming stream of messages from the master.
	replicationStream chan time.Time

	// throttler is used to enforce the maximum rate at which replica applies
	// transactions. It must not be confused with the client's throttler.
	throttler         *throttler.Throttler
	lastHealthUpdate  time.Time
	lagUpdateInterval time.Duration

	degrationInterval   time.Duration
	degrationDuration   time.Duration
	nextDegration       time.Time
	currentDegrationEnd time.Time

	stopChan chan struct{}
	wg       sync.WaitGroup
}

func newReplica(lagUpdateInterval, degrationInterval, degrationDuration time.Duration) *replica {
	t := &testing.T{}
	ts := zktestserver.New(t, []string{"cell1"})
	wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())
	db := fakesqldb.Register()
	fakeTablet := testlib.NewFakeTablet(t, wr, "cell1", 0,
		topodatapb.TabletType_REPLICA, db, testlib.TabletKeyspaceShard(t, "ks", "-80"))
	fakeTablet.StartActionLoop(t, wr)

	target := querypb.Target{
		Keyspace:   "ks",
		Shard:      "-80",
		TabletType: topodatapb.TabletType_REPLICA,
	}
	qs := fakes.NewStreamHealthQueryService(target)
	grpcqueryservice.Register(fakeTablet.RPCServer, qs)

	throttler, err := throttler.NewThrottler("replica", "TPS", 1, *rate, throttler.ReplicationLagModuleDisabled)
	if err != nil {
		log.Fatal(err)
	}

	var nextDegration time.Time
	if degrationInterval != time.Duration(0) {
		nextDegration = time.Now().Add(degrationInterval)
	}
	r := &replica{
		fakeTablet:        fakeTablet,
		qs:                qs,
		throttler:         throttler,
		replicationStream: make(chan time.Time, 1*1024*1024),
		lagUpdateInterval: lagUpdateInterval,
		degrationInterval: degrationInterval,
		degrationDuration: degrationDuration,
		nextDegration:     nextDegration,
		stopChan:          make(chan struct{}),
	}
	r.wg.Add(1)
	go r.processReplicationStream()
	return r
}

func (r *replica) replicate(msg time.Time) {
	r.replicationStream <- msg
}

func (r *replica) processReplicationStream() {
	defer r.wg.Done()

	// actualRate counts the number of requests per r.lagUpdateInterval.
	actualRate := 0
	for msg := range r.replicationStream {
		select {
		case <-r.stopChan:
			return
		default:
		}

		now := time.Now()
		if now.Sub(r.lastHealthUpdate) > r.lagUpdateInterval {
			// Broadcast current lag every "lagUpdateInterval".
			//
			// Use integer values to calculate the lag. In consequence, the reported
			// lag will constantly vary between the floor and ceil value e.g.
			// an actual lag of 0.5s could be reported as 0s or 1s based on the
			// truncation of the two times.
			lagTruncated := uint32(now.Unix() - msg.Unix())
			// Display lag with a higher precision as well.
			lag := now.Sub(msg).Seconds()
			log.Infof("current lag: %1ds (%1.1fs) replica rate: % 7.1f chan len: % 6d", lagTruncated, lag, float64(actualRate)/r.lagUpdateInterval.Seconds(), len(r.replicationStream))
			r.qs.AddHealthResponseWithSecondsBehindMaster(lagTruncated)
			r.lastHealthUpdate = now
			actualRate = 0
		}
		if !r.nextDegration.IsZero() && time.Now().After(r.nextDegration) && r.currentDegrationEnd.IsZero() {
			degradedRate := rand.Int63n(*rate)
			log.Infof("degrading the replica for %.f seconds from %v TPS to %v", r.degrationDuration.Seconds(), *rate, degradedRate)
			r.throttler.SetMaxRate(degradedRate)
			r.currentDegrationEnd = time.Now().Add(r.degrationDuration)
		}
		if !r.currentDegrationEnd.IsZero() && time.Now().After(r.currentDegrationEnd) {
			log.Infof("degrading the replica stopped. Restoring TPS to: %v", *rate)
			r.throttler.SetMaxRate(*rate)
			r.currentDegrationEnd = time.Time{}
			r.nextDegration = time.Now().Add(r.degrationInterval)
		}

		for {
			backoff := r.throttler.Throttle(0 /* threadID */)
			if backoff == throttler.NotThrottled {
				break
			}
			time.Sleep(backoff)
		}
		actualRate++
	}
}

func (r *replica) stop() {
	close(r.replicationStream)
	close(r.stopChan)
	log.Info("Triggered replica shutdown. Waiting for it to stop.")
	r.wg.Wait()
	r.fakeTablet.StopActionLoop(&testing.T{})
}

// client simulates a client which should throttle itself based on the
// replication lag of all replicas.
type client struct {
	master *master

	healthCheck discovery.HealthCheck
	throttler   *throttler.Throttler

	stopChan chan struct{}
	wg       sync.WaitGroup
}

func newClient(master *master, replica *replica) *client {
	t, err := throttler.NewThrottler("client", "TPS", 1, throttler.MaxRateModuleDisabled, 5 /* seconds */)
	if err != nil {
		log.Fatal(err)
	}

	healthCheck := discovery.NewHealthCheck(1*time.Minute, 5*time.Second, 1*time.Minute)
	c := &client{
		master:      master,
		healthCheck: healthCheck,
		throttler:   t,
		stopChan:    make(chan struct{}),
	}
	c.healthCheck.SetListener(c, false /* sendDownEvents */)
	c.healthCheck.AddTablet(replica.fakeTablet.Tablet, "name")
	return c
}

func (c *client) run() {
	c.wg.Add(1)
	go c.loop()
}

func (c *client) loop() {
	defer c.wg.Done()

	for {
		select {
		case <-c.stopChan:
			return
		default:
		}

		for {
			backoff := c.throttler.Throttle(0 /* threadID */)
			if backoff == throttler.NotThrottled {
				break
			}
			time.Sleep(backoff)
		}

		c.master.execute(time.Now())
	}
}

func (c *client) stop() {
	close(c.stopChan)
	c.wg.Wait()

	c.healthCheck.Close()
	c.throttler.Close()
}

// StatsUpdate implements discovery.HealthCheckStatsListener.
// It gets called by the healthCheck instance every time a tablet broadcasts
// a health update.
func (c *client) StatsUpdate(ts *discovery.TabletStats) {
	c.throttler.RecordReplicationLag(time.Now(), ts)
}

func main() {
	flag.Parse()

	go servenv.RunDefault()

	log.Infof("start rate set to: %v", *rate)
	replica := newReplica(*lagUpdateInterval, *replicaDegrationInterval, *replicaDegrationDuration)
	master := &master{replica: replica}
	client := newClient(master, replica)
	client.run()

	time.Sleep(*duration)
	client.stop()
	replica.stop()
}

func init() {
	servenv.RegisterDefaultFlags()
}
<|endoftext|>


<commit_before>package sam3

import (
	"bufio"
	"bytes"
	"errors"
  "io"
	"net"
	"strings"
)

// Represents a streaming session.
type StreamSession struct {
  samAddr     string             // address to the sam bridge (ipv4:port)
	id          string             // tunnel name
	conn        net.Conn           // connection to sam bridge
	keys        I2PKeys            // i2p destination keys
}

// Returns the local tunnel name of the I2P tunnel used for the stream session
func (ss StreamSession) ID() string {
	return ss.id
}

func (ss *StreamSession) Close() error {
  return ss.conn.Close()
}

// Returns the I2P destination (the address) of the stream session
func (ss StreamSession) Addr() I2PAddr {
	return ss.keys.Addr()
}

// Returns the keys associated with the stream session
func (ss StreamSession) Keys() I2PKeys {
	return ss.keys
}

// Creates a new StreamSession with the I2CP- and streaminglib options as
// specified. See the I2P documentation for a full list of options.
func (sam *SAM) NewStreamSession(id string, keys I2PKeys, options []string) (*StreamSession, error) {
	conn, err := sam.newGenericSession("STREAM", id, keys, options, []string{})
	if err != nil {
		return nil, err
	}
	return &StreamSession{sam.address, id, conn, keys}, nil
}

// implement net.Dialer
func (s *StreamSession) Dial(n, addr string) (c net.Conn, err error) {

  var i2paddr I2PAddr
  var host string
  host, _, err = net.SplitHostPort(addr)
  if err == nil {
    // check for name
    if strings.HasSuffix(host, ".b32.i2p") || strings.HasSuffix(host, ".i2p") {
      // name lookup
      var sam *SAM
      sam, err = NewSAM(s.samAddr)
      if err == nil {
        i2paddr, err = sam.Lookup(host)
        sam.Close()
      }
    } else {
      // probably a destination
      i2paddr = I2PAddr(host)
    }
    return s.DialI2P(i2paddr)
  }
  return
}

// Dials to an I2P destination and returns a SAMConn, which implements a net.Conn.
func (s *StreamSession) DialI2P(addr I2PAddr) (*SAMConn, error) {
	sam, err := NewSAM(s.samAddr)
	if err != nil {
		return nil, err
	}
	conn := sam.conn
	_,err = conn.Write([]byte("STREAM CONNECT ID=" + s.id + " DESTINATION=" + addr.Base64() + " SILENT=false\n"))
	if err != nil {
		return nil, err
	}
	buf := make([]byte, 4096)
	n, err := conn.Read(buf)
	if err != nil {
		return nil, err
	}
	scanner := bufio.NewScanner(bytes.NewReader(buf[:n]))
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		switch scanner.Text() {
		case "STREAM" :
			continue
		case "STATUS" :
			continue
		case "RESULT=OK" :
			return &SAMConn{s.keys.addr, addr, conn}, nil
		case "RESULT=CANT_REACH_PEER" :
      sam.Close()
			return nil, errors.New("Can not reach peer")
		case "RESULT=I2P_ERROR" :
      sam.Close()
			return nil, errors.New("I2P internal error")
		case "RESULT=INVALID_KEY" :
      sam.Close()
			return nil, errors.New("Invalid key")
		case "RESULT=INVALID_ID" :
      sam.Close()
			return nil, errors.New("Invalid tunnel ID")
		case "RESULT=TIMEOUT" :
      sam.Close()
			return nil, errors.New("Timeout")
    default:
      sam.Close()
			return nil, errors.New("Unknown error: " + scanner.Text() + " : " + string(buf[:n]))
		}
	}
	panic("sam3 go library error in StreamSession.DialI2P()")
}

// create a new stream listener to accept inbound connections
func (s *StreamSession) Listen() (*StreamListener, error) {
  return &StreamListener{
    session: s,
    id: s.id,
    laddr: s.keys.Addr(),
  }, nil
}

type StreamListener struct {
  // parent stream session
  session *StreamSession
  // our session id
  id string
  // our local address for this sam socket
  laddr I2PAddr
}

// get our address
// implements net.Listener
func (l *StreamListener) Addr() net.Addr {
  return l.laddr
}

// implements net.Listener
func (l *StreamListener) Close() error {
  return l.session.Close()
}

// accept a new inbound connection
// implements net.Listener
func (l *StreamListener) Accept() (net.Conn, error) {
  s, err := NewSAM(l.session.samAddr)
  if err == nil {
    // we connected to sam
    // send accept() command
    _, err = io.WriteString(s.conn, "STREAM ACCEPT ID="+l.id+" SILENT=false\n")
    // read reply
    rd := bufio.NewReader(s.conn)
    // read first line
    line, err := rd.ReadString(10)
    if err == nil {
      if strings.HasPrefix(line, "STREAM STATUS RESULT=OK") {
        // we gud read destination line
        dest, err := rd.ReadString(10)
        if err == nil {
          // return wrapped connection
          dest = strings.Trim(dest, "\n")
          return &SAMConn{
            laddr: l.laddr,
            raddr: I2PAddr(dest),
            conn: s.conn,
          }, nil
        } else {
          s.Close()
          return nil, err
        }
      } else {
        s.Close()
        return nil, errors.New("invalid sam line: "+line)
      }
    } else {
      s.Close()
      return nil, err
    }
  }
  return nil, err
}
<commit_msg>fixups in sam3<commit_after>package sam3

import (
	"bufio"
	"bytes"
	"errors"
  "io"
	"net"
	"strings"
)

// Represents a streaming session.
type StreamSession struct {
  samAddr     string             // address to the sam bridge (ipv4:port)
	id          string             // tunnel name
	conn        net.Conn           // connection to sam bridge
	keys        I2PKeys            // i2p destination keys
}

// Returns the local tunnel name of the I2P tunnel used for the stream session
func (ss StreamSession) ID() string {
	return ss.id
}

func (ss *StreamSession) Close() error {
  return ss.conn.Close()
}

// Returns the I2P destination (the address) of the stream session
func (ss StreamSession) Addr() I2PAddr {
	return ss.keys.Addr()
}

// Returns the keys associated with the stream session
func (ss StreamSession) Keys() I2PKeys {
	return ss.keys
}

// Creates a new StreamSession with the I2CP- and streaminglib options as
// specified. See the I2P documentation for a full list of options.
func (sam *SAM) NewStreamSession(id string, keys I2PKeys, options []string) (*StreamSession, error) {
	conn, err := sam.newGenericSession("STREAM", id, keys, options, []string{})
	if err != nil {
		return nil, err
	}
	return &StreamSession{sam.address, id, conn, keys}, nil
}

// implement net.Dialer
func (s *StreamSession) Dial(n, addr string) (c net.Conn, err error) {

  var i2paddr I2PAddr
  var host string
  host, _, err = net.SplitHostPort(addr)
  if err == nil {
    // check for name
    if strings.HasSuffix(host, ".b32.i2p") || strings.HasSuffix(host, ".i2p") {
      // name lookup
      var sam *SAM
      sam, err = NewSAM(s.samAddr)
      if err == nil {
        i2paddr, err = sam.Lookup(host)
        sam.Close()
      }
    } else {
      // probably a destination
      i2paddr = I2PAddr(host)
    }
    return s.DialI2P(i2paddr)
  }
  return
}

// Dials to an I2P destination and returns a SAMConn, which implements a net.Conn.
func (s *StreamSession) DialI2P(addr I2PAddr) (*SAMConn, error) {
	sam, err := NewSAM(s.samAddr)
	if err != nil {
		return nil, err
	}
	conn := sam.conn
	_,err = conn.Write([]byte("STREAM CONNECT ID=" + s.id + " DESTINATION=" + addr.Base64() + " SILENT=false\n"))
	if err != nil {
		return nil, err
	}
	buf := make([]byte, 4096)
	n, err := conn.Read(buf)
	if err != nil {
		return nil, err
	}
	scanner := bufio.NewScanner(bytes.NewReader(buf[:n]))
	scanner.Split(bufio.ScanWords)
	for scanner.Scan() {
		switch scanner.Text() {
		case "STREAM" :
			continue
		case "STATUS" :
			continue
		case "RESULT=OK" :
			return &SAMConn{s.keys.addr, addr, conn}, nil
		case "RESULT=CANT_REACH_PEER" :
      sam.Close()
			return nil, errors.New("Can not reach peer")
		case "RESULT=I2P_ERROR" :
      sam.Close()
			return nil, errors.New("I2P internal error")
		case "RESULT=INVALID_KEY" :
      sam.Close()
			return nil, errors.New("Invalid key")
		case "RESULT=INVALID_ID" :
      sam.Close()
			return nil, errors.New("Invalid tunnel ID")
		case "RESULT=TIMEOUT" :
      sam.Close()
			return nil, errors.New("Timeout")
    default:
      sam.Close()
			return nil, errors.New("Unknown error: " + scanner.Text() + " : " + string(buf[:n]))
		}
	}
	panic("sam3 go library error in StreamSession.DialI2P()")
}

// create a new stream listener to accept inbound connections
func (s *StreamSession) Listen() (*StreamListener, error) {
  return &StreamListener{
    session: s,
    id: s.id,
    laddr: s.keys.Addr(),
  }, nil
}

type StreamListener struct {
  // parent stream session
  session *StreamSession
  // our session id
  id string
  // our local address for this sam socket
  laddr I2PAddr
}

// get our address
// implements net.Listener
func (l *StreamListener) Addr() net.Addr {
  return l.laddr
}

// implements net.Listener
func (l *StreamListener) Close() error {
  return l.session.Close()
}

// implements net.Listener
func (l *StreamListener) Accept() (net.Conn, error) {
  return l.AcceptI2P()
}

// accept a new inbound connection
func (l *StreamListener) AcceptI2P() (*SAMConn, error) {
  s, err := NewSAM(l.session.samAddr)
  if err == nil {
    // we connected to sam
    // send accept() command
    _, err = io.WriteString(s.conn, "STREAM ACCEPT ID="+l.id+" SILENT=false\n")
    // read reply
    rd := bufio.NewReader(s.conn)
    // read first line
    line, err := rd.ReadString(10)
    if err == nil {
      if strings.HasPrefix(line, "STREAM STATUS RESULT=OK") {
        // we gud read destination line
        dest, err := rd.ReadString(10)
        if err == nil {
          // return wrapped connection
          dest = strings.Trim(dest, "\n")
          return &SAMConn{
            laddr: l.laddr,
            raddr: I2PAddr(dest),
            conn: s.conn,
          }, nil
        } else {
          s.Close()
          return nil, err
        }
      } else {
        s.Close()
        return nil, errors.New("invalid sam line: "+line)
      }
    } else {
      s.Close()
      return nil, err
    }
  }
  return nil, err
}
<|endoftext|>


<commit_before>package docker

import (
	"archive/tar"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"github.com/hashicorp/go-version"
	"github.com/hashicorp/packer/packer"
)

type Communicator struct {
	ContainerID   string
	HostDir       string
	ContainerDir  string
	Version       *version.Version
	Config        *Config
	ContainerUser string
	lock          sync.Mutex
	EntryPoint    []string
}

var _ packer.Communicator = new(Communicator)

func (c *Communicator) Start(ctx context.Context, remote *packer.RemoteCmd) error {
	dockerArgs := []string{
		"exec",
		"-i",
		c.ContainerID,
	}
	dockerArgs = append(dockerArgs, c.EntryPoint...)
	dockerArgs = append(dockerArgs, fmt.Sprintf("(%s)", remote.Command))

	if c.Config.Pty {
		dockerArgs = append(dockerArgs[:2], append([]string{"-t"}, dockerArgs[2:]...)...)
	}

	if c.Config.ExecUser != "" {
		dockerArgs = append(dockerArgs[:2],
			append([]string{"-u", c.Config.ExecUser}, dockerArgs[2:]...)...)
	}

	cmd := exec.Command("docker", dockerArgs...)

	var (
		stdin_w io.WriteCloser
		err     error
	)

	stdin_w, err = cmd.StdinPipe()
	if err != nil {
		return err
	}

	stderr_r, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	stdout_r, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	// Run the actual command in a goroutine so that Start doesn't block
	go c.run(cmd, remote, stdin_w, stdout_r, stderr_r)

	return nil
}

// Upload uploads a file to the docker container
func (c *Communicator) Upload(dst string, src io.Reader, fi *os.FileInfo) error {
	if fi == nil {
		return c.uploadReader(dst, src)
	}
	return c.uploadFile(dst, src, fi)
}

// uploadReader writes an io.Reader to a temporary file before uploading
func (c *Communicator) uploadReader(dst string, src io.Reader) error {
	// Create a temporary file to store the upload
	tempfile, err := ioutil.TempFile(c.HostDir, "upload")
	if err != nil {
		return fmt.Errorf("Failed to open temp file for writing: %s", err)
	}
	defer os.Remove(tempfile.Name())
	defer tempfile.Close()

	if _, err := io.Copy(tempfile, src); err != nil {
		return fmt.Errorf("Failed to copy upload file to tempfile: %s", err)
	}
	tempfile.Seek(0, 0)
	fi, err := tempfile.Stat()
	if err != nil {
		return fmt.Errorf("Error getting tempfile info: %s", err)
	}
	return c.uploadFile(dst, tempfile, &fi)
}

// uploadFile uses docker cp to copy the file from the host to the container
func (c *Communicator) uploadFile(dst string, src io.Reader, fi *os.FileInfo) error {
	// command format: docker cp /path/to/infile containerid:/path/to/outfile
	log.Printf("Copying to %s on container %s.", dst, c.ContainerID)

	localCmd := exec.Command("docker", "cp", "-",
		fmt.Sprintf("%s:%s", c.ContainerID, filepath.Dir(dst)))

	stderrP, err := localCmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("Failed to open pipe: %s", err)
	}

	stdin, err := localCmd.StdinPipe()
	if err != nil {
		return fmt.Errorf("Failed to open pipe: %s", err)
	}

	if err := localCmd.Start(); err != nil {
		return err
	}

	archive := tar.NewWriter(stdin)
	header, err := tar.FileInfoHeader(*fi, "")
	if err != nil {
		return err
	}
	header.Name = filepath.Base(dst)
	archive.WriteHeader(header)
	numBytes, err := io.Copy(archive, src)
	if err != nil {
		return fmt.Errorf("Failed to pipe upload: %s", err)
	}
	log.Printf("Copied %d bytes for %s", numBytes, dst)

	if err := archive.Close(); err != nil {
		return fmt.Errorf("Failed to close archive: %s", err)
	}
	if err := stdin.Close(); err != nil {
		return fmt.Errorf("Failed to close stdin: %s", err)
	}

	stderrOut, err := ioutil.ReadAll(stderrP)
	if err != nil {
		return err
	}

	if err := localCmd.Wait(); err != nil {
		return fmt.Errorf("Failed to upload to '%s' in container: %s. %s.", dst, stderrOut, err)
	}

	if err := c.fixDestinationOwner(dst); err != nil {
		return err
	}

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	/*
		from https://docs.docker.com/engine/reference/commandline/cp/#extended-description
		SRC_PATH specifies a directory
			DEST_PATH does not exist
				DEST_PATH is created as a directory and the contents of the source directory are copied into this directory
			DEST_PATH exists and is a file
				Error condition: cannot copy a directory to a file
			DEST_PATH exists and is a directory
				SRC_PATH does not end with /. (that is: slash followed by dot)
					the source directory is copied into this directory
				SRC_PATH does end with /. (that is: slash followed by dot)
					the content of the source directory is copied into this directory

		translating that in to our semantics:

		if source ends in /
			docker cp src. dest
		otherwise, cp source dest

	*/

	var dockerSource string

	if src[len(src)-1] == '/' {
		dockerSource = fmt.Sprintf("%s.", src)
	} else {
		dockerSource = fmt.Sprintf("%s", src)
	}

	// Make the directory, then copy into it
	localCmd := exec.Command("docker", "cp", dockerSource, fmt.Sprintf("%s:%s", c.ContainerID, dst))

	stderrP, err := localCmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("Failed to open pipe: %s", err)
	}
	if err := localCmd.Start(); err != nil {
		return fmt.Errorf("Failed to copy: %s", err)
	}
	stderrOut, err := ioutil.ReadAll(stderrP)
	if err != nil {
		return err
	}

	// Wait for the copy to complete
	if err := localCmd.Wait(); err != nil {
		return fmt.Errorf("Failed to upload to '%s' in container: %s. %s.", dst, stderrOut, err)
	}

	if err := c.fixDestinationOwner(dst); err != nil {
		return err
	}

	return nil
}

// Download pulls a file out of a container using `docker cp`. We have a source
// path and want to write to an io.Writer, not a file. We use - to make docker
// cp to write to stdout, and then copy the stream to our destination io.Writer.
func (c *Communicator) Download(src string, dst io.Writer) error {
	log.Printf("Downloading file from container: %s:%s", c.ContainerID, src)
	localCmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", c.ContainerID, src), "-")

	pipe, err := localCmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("Failed to open pipe: %s", err)
	}

	if err = localCmd.Start(); err != nil {
		return fmt.Errorf("Failed to start download: %s", err)
	}

	// When you use - to send docker cp to stdout it is streamed as a tar; this
	// enables it to work with directories. We don't actually support
	// directories in Download() but we still need to handle the tar format.
	archive := tar.NewReader(pipe)
	_, err = archive.Next()
	if err != nil {
		return fmt.Errorf("Failed to read header from tar stream: %s", err)
	}

	numBytes, err := io.Copy(dst, archive)
	if err != nil {
		return fmt.Errorf("Failed to pipe download: %s", err)
	}
	log.Printf("Copied %d bytes for %s", numBytes, src)

	if err = localCmd.Wait(); err != nil {
		return fmt.Errorf("Failed to download '%s' from container: %s", src, err)
	}

	return nil
}

func (c *Communicator) DownloadDir(src string, dst string, exclude []string) error {
	return fmt.Errorf("DownloadDir is not implemented for docker")
}

// Runs the given command and blocks until completion
func (c *Communicator) run(cmd *exec.Cmd, remote *packer.RemoteCmd, stdin io.WriteCloser, stdout, stderr io.ReadCloser) {
	// For Docker, remote communication must be serialized since it
	// only supports single execution.
	c.lock.Lock()
	defer c.lock.Unlock()

	wg := sync.WaitGroup{}
	repeat := func(w io.Writer, r io.ReadCloser) {
		io.Copy(w, r)
		r.Close()
		wg.Done()
	}

	if remote.Stdout != nil {
		wg.Add(1)
		go repeat(remote.Stdout, stdout)
	}

	if remote.Stderr != nil {
		wg.Add(1)
		go repeat(remote.Stderr, stderr)
	}

	// Start the command
	log.Printf("Executing %s:", strings.Join(cmd.Args, " "))
	if err := cmd.Start(); err != nil {
		log.Printf("Error executing: %s", err)
		remote.SetExited(254)
		return
	}

	var exitStatus int

	if remote.Stdin != nil {
		go func() {
			io.Copy(stdin, remote.Stdin)
			// close stdin to support commands that wait for stdin to be closed before exiting.
			stdin.Close()
		}()
	}

	wg.Wait()
	err := cmd.Wait()

	if exitErr, ok := err.(*exec.ExitError); ok {
		exitStatus = 1

		// There is no process-independent way to get the REAL
		// exit status so we just try to go deeper.
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		}
	}

	// Set the exit status which triggers waiters
	remote.SetExited(exitStatus)
}

// TODO Workaround for #5307. Remove once #5409 is fixed.
func (c *Communicator) fixDestinationOwner(destination string) error {
	if !c.Config.FixUploadOwner {
		return nil
	}

	owner := c.ContainerUser
	if owner == "" {
		owner = "root"
	}

	chownArgs := []string{
		"docker", "exec", "--user", "root", c.ContainerID, "/bin/sh", "-c",
		fmt.Sprintf("chown -R %s %s", owner, destination),
	}
	if output, err := exec.Command(chownArgs[0], chownArgs[1:]...).CombinedOutput(); err != nil {
		return fmt.Errorf("Failed to set owner of the uploaded file: %s, %s", err, output)
	}

	return nil
}
<commit_msg>read stderr before we try to read the artchive.<commit_after>package docker

import (
	"archive/tar"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"github.com/hashicorp/go-version"
	"github.com/hashicorp/packer/packer"
)

type Communicator struct {
	ContainerID   string
	HostDir       string
	ContainerDir  string
	Version       *version.Version
	Config        *Config
	ContainerUser string
	lock          sync.Mutex
	EntryPoint    []string
}

var _ packer.Communicator = new(Communicator)

func (c *Communicator) Start(ctx context.Context, remote *packer.RemoteCmd) error {
	dockerArgs := []string{
		"exec",
		"-i",
		c.ContainerID,
	}
	dockerArgs = append(dockerArgs, c.EntryPoint...)
	dockerArgs = append(dockerArgs, fmt.Sprintf("(%s)", remote.Command))

	if c.Config.Pty {
		dockerArgs = append(dockerArgs[:2], append([]string{"-t"}, dockerArgs[2:]...)...)
	}

	if c.Config.ExecUser != "" {
		dockerArgs = append(dockerArgs[:2],
			append([]string{"-u", c.Config.ExecUser}, dockerArgs[2:]...)...)
	}

	cmd := exec.Command("docker", dockerArgs...)

	var (
		stdin_w io.WriteCloser
		err     error
	)

	stdin_w, err = cmd.StdinPipe()
	if err != nil {
		return err
	}

	stderr_r, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	stdout_r, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	// Run the actual command in a goroutine so that Start doesn't block
	go c.run(cmd, remote, stdin_w, stdout_r, stderr_r)

	return nil
}

// Upload uploads a file to the docker container
func (c *Communicator) Upload(dst string, src io.Reader, fi *os.FileInfo) error {
	if fi == nil {
		return c.uploadReader(dst, src)
	}
	return c.uploadFile(dst, src, fi)
}

// uploadReader writes an io.Reader to a temporary file before uploading
func (c *Communicator) uploadReader(dst string, src io.Reader) error {
	// Create a temporary file to store the upload
	tempfile, err := ioutil.TempFile(c.HostDir, "upload")
	if err != nil {
		return fmt.Errorf("Failed to open temp file for writing: %s", err)
	}
	defer os.Remove(tempfile.Name())
	defer tempfile.Close()

	if _, err := io.Copy(tempfile, src); err != nil {
		return fmt.Errorf("Failed to copy upload file to tempfile: %s", err)
	}
	tempfile.Seek(0, 0)
	fi, err := tempfile.Stat()
	if err != nil {
		return fmt.Errorf("Error getting tempfile info: %s", err)
	}
	return c.uploadFile(dst, tempfile, &fi)
}

// uploadFile uses docker cp to copy the file from the host to the container
func (c *Communicator) uploadFile(dst string, src io.Reader, fi *os.FileInfo) error {
	// command format: docker cp /path/to/infile containerid:/path/to/outfile
	log.Printf("Copying to %s on container %s.", dst, c.ContainerID)

	localCmd := exec.Command("docker", "cp", "-",
		fmt.Sprintf("%s:%s", c.ContainerID, filepath.Dir(dst)))

	stderrP, err := localCmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("Failed to open pipe: %s", err)
	}

	stdin, err := localCmd.StdinPipe()
	if err != nil {
		return fmt.Errorf("Failed to open pipe: %s", err)
	}

	if err := localCmd.Start(); err != nil {
		return err
	}

	archive := tar.NewWriter(stdin)
	header, err := tar.FileInfoHeader(*fi, "")
	if err != nil {
		return err
	}
	header.Name = filepath.Base(dst)
	archive.WriteHeader(header)
	numBytes, err := io.Copy(archive, src)
	if err != nil {
		return fmt.Errorf("Failed to pipe upload: %s", err)
	}
	log.Printf("Copied %d bytes for %s", numBytes, dst)

	if err := archive.Close(); err != nil {
		return fmt.Errorf("Failed to close archive: %s", err)
	}
	if err := stdin.Close(); err != nil {
		return fmt.Errorf("Failed to close stdin: %s", err)
	}

	stderrOut, err := ioutil.ReadAll(stderrP)
	if err != nil {
		return err
	}

	if err := localCmd.Wait(); err != nil {
		return fmt.Errorf("Failed to upload to '%s' in container: %s. %s.", dst, stderrOut, err)
	}

	if err := c.fixDestinationOwner(dst); err != nil {
		return err
	}

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	/*
		from https://docs.docker.com/engine/reference/commandline/cp/#extended-description
		SRC_PATH specifies a directory
			DEST_PATH does not exist
				DEST_PATH is created as a directory and the contents of the source directory are copied into this directory
			DEST_PATH exists and is a file
				Error condition: cannot copy a directory to a file
			DEST_PATH exists and is a directory
				SRC_PATH does not end with /. (that is: slash followed by dot)
					the source directory is copied into this directory
				SRC_PATH does end with /. (that is: slash followed by dot)
					the content of the source directory is copied into this directory

		translating that in to our semantics:

		if source ends in /
			docker cp src. dest
		otherwise, cp source dest

	*/

	var dockerSource string

	if src[len(src)-1] == '/' {
		dockerSource = fmt.Sprintf("%s.", src)
	} else {
		dockerSource = fmt.Sprintf("%s", src)
	}

	// Make the directory, then copy into it
	localCmd := exec.Command("docker", "cp", dockerSource, fmt.Sprintf("%s:%s", c.ContainerID, dst))

	stderrP, err := localCmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("Failed to open pipe: %s", err)
	}
	if err := localCmd.Start(); err != nil {
		return fmt.Errorf("Failed to copy: %s", err)
	}
	stderrOut, err := ioutil.ReadAll(stderrP)
	if err != nil {
		return err
	}

	// Wait for the copy to complete
	if err := localCmd.Wait(); err != nil {
		return fmt.Errorf("Failed to upload to '%s' in container: %s. %s.", dst, stderrOut, err)
	}

	if err := c.fixDestinationOwner(dst); err != nil {
		return err
	}

	return nil
}

// Download pulls a file out of a container using `docker cp`. We have a source
// path and want to write to an io.Writer, not a file. We use - to make docker
// cp to write to stdout, and then copy the stream to our destination io.Writer.
func (c *Communicator) Download(src string, dst io.Writer) error {
	log.Printf("Downloading file from container: %s:%s", c.ContainerID, src)
	localCmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", c.ContainerID, src), "-")

	pipe, err := localCmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("Failed to open pipe: %s", err)
	}

	stderrP, err := localCmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("Failed to open stderr pipe: %s", err)
	}

	if err = localCmd.Start(); err != nil {
		return fmt.Errorf("Failed to start download: %s", err)
	}

	// When you use - to send docker cp to stdout it is streamed as a tar; this
	// enables it to work with directories. We don't actually support
	// directories in Download() but we still need to handle the tar format.

	stderrOut, err := ioutil.ReadAll(stderrP)
	if err != nil {
		return err
	}

	if string(stderrOut) != "" {
		return fmt.Errorf("Error downloading file: %s", string(stderrOut))
	}

	archive := tar.NewReader(pipe)
	_, err = archive.Next()
	if err != nil {
		return fmt.Errorf("Failed to read header from tar stream: %s", err)
	}

	numBytes, err := io.Copy(dst, archive)
	if err != nil {
		return fmt.Errorf("Failed to pipe download: %s", err)
	}
	log.Printf("Copied %d bytes for %s", numBytes, src)

	if err = localCmd.Wait(); err != nil {
		return fmt.Errorf("Failed to download '%s' from container: %s", src, err)
	}

	return nil
}

func (c *Communicator) DownloadDir(src string, dst string, exclude []string) error {
	return fmt.Errorf("DownloadDir is not implemented for docker")
}

// Runs the given command and blocks until completion
func (c *Communicator) run(cmd *exec.Cmd, remote *packer.RemoteCmd, stdin io.WriteCloser, stdout, stderr io.ReadCloser) {
	// For Docker, remote communication must be serialized since it
	// only supports single execution.
	c.lock.Lock()
	defer c.lock.Unlock()

	wg := sync.WaitGroup{}
	repeat := func(w io.Writer, r io.ReadCloser) {
		io.Copy(w, r)
		r.Close()
		wg.Done()
	}

	if remote.Stdout != nil {
		wg.Add(1)
		go repeat(remote.Stdout, stdout)
	}

	if remote.Stderr != nil {
		wg.Add(1)
		go repeat(remote.Stderr, stderr)
	}

	// Start the command
	log.Printf("Executing %s:", strings.Join(cmd.Args, " "))
	if err := cmd.Start(); err != nil {
		log.Printf("Error executing: %s", err)
		remote.SetExited(254)
		return
	}

	var exitStatus int

	if remote.Stdin != nil {
		go func() {
			io.Copy(stdin, remote.Stdin)
			// close stdin to support commands that wait for stdin to be closed before exiting.
			stdin.Close()
		}()
	}

	wg.Wait()
	err := cmd.Wait()

	if exitErr, ok := err.(*exec.ExitError); ok {
		exitStatus = 1

		// There is no process-independent way to get the REAL
		// exit status so we just try to go deeper.
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		}
	}

	// Set the exit status which triggers waiters
	remote.SetExited(exitStatus)
}

// TODO Workaround for #5307. Remove once #5409 is fixed.
func (c *Communicator) fixDestinationOwner(destination string) error {
	if !c.Config.FixUploadOwner {
		return nil
	}

	owner := c.ContainerUser
	if owner == "" {
		owner = "root"
	}

	chownArgs := []string{
		"docker", "exec", "--user", "root", c.ContainerID, "/bin/sh", "-c",
		fmt.Sprintf("chown -R %s %s", owner, destination),
	}
	if output, err := exec.Command(chownArgs[0], chownArgs[1:]...).CombinedOutput(); err != nil {
		return fmt.Errorf("Failed to set owner of the uploaded file: %s, %s", err, output)
	}

	return nil
}
<|endoftext|>