<commit_before>package main

import "math"

var sampleFreq float32 = 512.0
var q0, q1, q2, q3 float64 = 1.0, 0.0, 0.0, 0.0
var attitudeX, attitudeY, attitudeZ float32

func GetCurrentAttitude()
{
    attitudeX = math.Atan2(2(q0*q1 + q2*q3), 1-2((q1*q1) + (q2*q2)))
    attitudeY = math.Asin(2(q0*q2 - q3*q1))
    attitudeZ = math.Atan2(2(q0*q3 + q1*q2), 1-2((q2*q2) + (q3*q3)))

    return attitudeX, attitudeY, attitudeZ
}

func AHRSupdate(gx, gy, gz, ax, ay, az, mx, my, mz float32)
{
    recipNorm float64
	s0, s1, s2, s3 float64
	qDot1, qDot2, qDot3, qDot4 float64
	hx, hy float64
	_2q0mx, _2q0my, _2q0mz, _2q1mx, _2bx, _2bz, _4bx, _4bz, _2q0, _2q1, _2q2, _2q3, _2q0q2, _2q2q3, q0q0, q0q1, q0q2, q0q3, q1q1, q1q2, q1q3, q2q2, q2q3, q3q3 float64

    // Use IMU algorithm if magnetometer measurement invalid (avoids NaN in magnetometer normalisation)
	if((mx == 0.0) && (my == 0.0) && (mz == 0.0)) {
		AHRSupdateIMU(gx, gy, gz, ax, ay, az)
		return;
	}

    // Rate of change of quaternion from gyroscope
	qDot1 = 0.5 * (-q1 * gx - q2 * gy - q3 * gz)
	qDot2 = 0.5 * (q0 * gx + q2 * gz - q3 * gy)
	qDot3 = 0.5 * (q0 * gy - q1 * gz + q3 * gx)
	qDot4 = 0.5 * (q0 * gz + q1 * gy - q2 * gx)

    // Compute feedback only if accelerometer measurement valid (avoids NaN in accelerometer normalisation)
	if(!((ax == 0.0) && (ay == 0.0) && (az == 0.0))) {

		// Normalise accelerometer measurement
		recipNorm = invSqrt(ax * ax + ay * ay + az * az)
		ax *= recipNorm
		ay *= recipNorm
		az *= recipNorm

		// Normalise magnetometer measurement
		recipNorm = invSqrt(mx * mx + my * my + mz * mz)
		mx *= recipNorm
		my *= recipNorm
		mz *= recipNorm

		// Auxiliary variables to avoid repeated arithmetic
		_2q0mx = 2.0 * q0 * mx
		_2q0my = 2.0 * q0 * my
		_2q0mz = 2.0 * q0 * mz
		_2q1mx = 2.0 * q1 * mx
		_2q0 = 2.0 * q0
		_2q1 = 2.0 * q1
		_2q2 = 2.0 * q2
		_2q3 = 2.0 * q3
		_2q0q2 = 2.0 * q0 * q2
		_2q2q3 = 2.0 * q2 * q3
		q0q0 = q0 * q0
		q0q1 = q0 * q1
		q0q2 = q0 * q2
		q0q3 = q0 * q3
		q1q1 = q1 * q1
		q1q2 = q1 * q2
		q1q3 = q1 * q3
		q2q2 = q2 * q2
		q2q3 = q2 * q3
		q3q3 = q3 * q3

		// Reference direction of Earth's magnetic field
		hx = mx * q0q0 - _2q0my * q3 + _2q0mz * q2 + mx * q1q1 + _2q1 * my * q2 + _2q1 * mz * q3 - mx * q2q2 - mx * q3q3
		hy = _2q0mx * q3 + my * q0q0 - _2q0mz * q1 + _2q1mx * q2 - my * q1q1 + my * q2q2 + _2q2 * mz * q3 - my * q3q3
		_2bx = sqrt(hx * hx + hy * hy)
		_2bz = -_2q0mx * q2 + _2q0my * q1 + mz * q0q0 + _2q1mx * q3 - mz * q1q1 + _2q2 * my * q3 - mz * q2q2 + mz * q3q3
		_4bx = 2.0 * _2bx
		_4bz = 2.0 * _2bz

		// Gradient decent algorithm corrective step
		s0 = -_2q2 * (2.0 * q1q3 - _2q0q2 - ax) + _2q1 * (2.0 * q0q1 + _2q2q3 - ay) - _2bz * q2 * (_2bx * (0.5 - q2q2 - q3q3) + _2bz * (q1q3 - q0q2) - mx) + (-_2bx * q3 + _2bz * q1) * (_2bx * (q1q2 - q0q3) + _2bz * (q0q1 + q2q3) - my) + _2bx * q2 * (_2bx * (q0q2 + q1q3) + _2bz * (0.5 - q1q1 - q2q2) - mz)
		s1 = _2q3 * (2.0 * q1q3 - _2q0q2 - ax) + _2q0 * (2.0 * q0q1 + _2q2q3 - ay) - 4.0 * q1 * (1 - 2.0 * q1q1 - 2.0 * q2q2 - az) + _2bz * q3 * (_2bx * (0.5 - q2q2 - q3q3) + _2bz * (q1q3 - q0q2) - mx) + (_2bx * q2 + _2bz * q0) * (_2bx * (q1q2 - q0q3) + _2bz * (q0q1 + q2q3) - my) + (_2bx * q3 - _4bz * q1) * (_2bx * (q0q2 + q1q3) + _2bz * (0.5 - q1q1 - q2q2) - mz)
		s2 = -_2q0 * (2.0 * q1q3 - _2q0q2 - ax) + _2q3 * (2.0 * q0q1 + _2q2q3 - ay) - 4.0 * q2 * (1 - 2.0 * q1q1 - 2.0 * q2q2 - az) + (-_4bx * q2 - _2bz * q0) * (_2bx * (0.5 - q2q2 - q3q3) + _2bz * (q1q3 - q0q2) - mx) + (_2bx * q1 + _2bz * q3) * (_2bx * (q1q2 - q0q3) + _2bz * (q0q1 + q2q3) - my) + (_2bx * q0 - _4bz * q2) * (_2bx * (q0q2 + q1q3) + _2bz * (0.5 - q1q1 - q2q2) - mz)
		s3 = _2q1 * (2.0 * q1q3 - _2q0q2 - ax) + _2q2 * (2.0 * q0q1 + _2q2q3 - ay) + (-_4bx * q3 + _2bz * q1) * (_2bx * (0.5 - q2q2 - q3q3) + _2bz * (q1q3 - q0q2) - mx) + (-_2bx * q0 + _2bz * q2) * (_2bx * (q1q2 - q0q3) + _2bz * (q0q1 + q2q3) - my) + _2bx * q1 * (_2bx * (q0q2 + q1q3) + _2bz * (0.5 - q1q1 - q2q2) - mz)
		recipNorm = invSqrt(s0 * s0 + s1 * s1 + s2 * s2 + s3 * s3) // normalise step magnitude
		s0 *= recipNorm
		s1 *= recipNorm
		s2 *= recipNorm
		s3 *= recipNorm

		// Apply feedback step
		qDot1 -= beta * s0
		qDot2 -= beta * s1
		qDot3 -= beta * s2
		qDot4 -= beta * s3
	}

    // Integrate rate of change of quaternion to yield quaternion
	q0 += qDot1 * (1.0 / sampleFreq)
	q1 += qDot2 * (1.0 / sampleFreq)
	q2 += qDot3 * (1.0 / sampleFreq)
	q3 += qDot4 * (1.0 / sampleFreq)

	// Normalise quaternion
	recipNorm = invSqrt(q0 * q0 + q1 * q1 + q2 * q2 + q3 * q3)
	q0 *= recipNorm
	q1 *= recipNorm
	q2 *= recipNorm
	q3 *= recipNorm
}

func AHRSupdateIMU(gx, gy, gz, ax, ay, az float32)
{
    recipNorm float32
	s0, s1, s2, s3 float32
	qDot1, qDot2, qDot3, qDot4 float32
	_2q0, _2q1, _2q2, _2q3, _4q0, _4q1, _4q2 ,_8q1, _8q2, q0q0, q1q1, q2q2, q3q3 float32

	// Rate of change of quaternion from gyroscope
	qDot1 = 0.5 * (-q1 * gx - q2 * gy - q3 * gz)
	qDot2 = 0.5 * (q0 * gx + q2 * gz - q3 * gy)
	qDot3 = 0.5 * (q0 * gy - q1 * gz + q3 * gx)
	qDot4 = 0.5 * (q0 * gz + q1 * gy - q2 * gx)

	// Compute feedback only if accelerometer measurement valid (avoids NaN in accelerometer normalisation)
	if(!((ax == 0.0) && (ay == 0.0) && (az == 0.0))) {

		// Normalise accelerometer measurement
		recipNorm = invSqrt(ax * ax + ay * ay + az * az)
		ax *= recipNorm
		ay *= recipNorm
		az *= recipNorm

		// Auxiliary variables to avoid repeated arithmetic
		_2q0 = 2.0 * q0
		_2q1 = 2.0 * q1
		_2q2 = 2.0 * q2
		_2q3 = 2.0 * q3
		_4q0 = 4.0 * q0
		_4q1 = 4.0 * q1
		_4q2 = 4.0 * q2
		_8q1 = 8.0 * q1
		_8q2 = 8.0 * q2
		q0q0 = q0 * q0
		q1q1 = q1 * q1
		q2q2 = q2 * q2
		q3q3 = q3 * q3

		// Gradient decent algorithm corrective step
		s0 = _4q0 * q2q2 + _2q2 * ax + _4q0 * q1q1 - _2q1 * ay
		s1 = _4q1 * q3q3 - _2q3 * ax + 4.0 * q0q0 * q1 - _2q0 * ay - _4q1 + _8q1 * q1q1 + _8q1 * q2q2 + _4q1 * az
		s2 = 4.0 * q0q0 * q2 + _2q0 * ax + _4q2 * q3q3 - _2q3 * ay - _4q2 + _8q2 * q1q1 + _8q2 * q2q2 + _4q2 * az
		s3 = 4.0 * q1q1 * q3 - _2q1 * ax + 4.0 * q2q2 * q3 - _2q2 * ay
		recipNorm = invSqrt(s0 * s0 + s1 * s1 + s2 * s2 + s3 * s3) // normalise step magnitude
		s0 *= recipNorm
		s1 *= recipNorm
		s2 *= recipNorm
		s3 *= recipNorm

		// Apply feedback step
		qDot1 -= beta * s0
		qDot2 -= beta * s1
		qDot3 -= beta * s2
		qDot4 -= beta * s3
	}

	// Integrate rate of change of quaternion to yield quaternion
	q0 += qDot1 * (1.0 / sampleFreq)
	q1 += qDot2 * (1.0 / sampleFreq)
	q2 += qDot3 * (1.0 / sampleFreq)
	q3 += qDot4 * (1.0 / sampleFreq)

	// Normalise quaternion
	recipNorm = invSqrt(q0 * q0 + q1 * q1 + q2 * q2 + q3 * q3)
	q0 *= recipNorm
	q1 *= recipNorm
	q2 *= recipNorm
	q3 *= recipNorm
}

func invSqrt(x float64) float64 {
	halfx float64 = 0.5 * x
	y float64 = x
	i int64 = *(int64*)&y
	i = 0x5f3759df - (i>>1)
	y = *(float64*)&i
	y = y * (1.5 - (halfx * y * y))
	return y
}<commit_msg>Added Alt Implementation for Inverse Sq Rt.<commit_after>package main

import "math"

var sampleFreq float32 = 512.0
var beta float32 = 0.1
var q0, q1, q2, q3 float64 = 1.0, 0.0, 0.0, 0.0
var attitudeX, attitudeY, attitudeZ float32

func GetCurrentAttitude()
{
    attitudeX = math.Atan2(2(q0*q1 + q2*q3), 1-2((q1*q1) + (q2*q2)))
    attitudeY = math.Asin(2(q0*q2 - q3*q1))
    attitudeZ = math.Atan2(2(q0*q3 + q1*q2), 1-2((q2*q2) + (q3*q3)))

    return attitudeX, attitudeY, attitudeZ
}

func AHRSupdate(gx, gy, gz, ax, ay, az, mx, my, mz float32)
{
    recipNorm float64
	s0, s1, s2, s3 float64
	qDot1, qDot2, qDot3, qDot4 float64
	hx, hy float64
	_2q0mx, _2q0my, _2q0mz, _2q1mx, _2bx, _2bz, _4bx, _4bz, _2q0, _2q1, _2q2, _2q3, _2q0q2, _2q2q3, q0q0, q0q1, q0q2, q0q3, q1q1, q1q2, q1q3, q2q2, q2q3, q3q3 float64

    // Use IMU algorithm if magnetometer measurement invalid (avoids NaN in magnetometer normalisation)
	if((mx == 0.0) && (my == 0.0) && (mz == 0.0)) {
		AHRSupdateIMU(gx, gy, gz, ax, ay, az)
		return;
	}

    // Rate of change of quaternion from gyroscope
	qDot1 = 0.5 * (-q1 * gx - q2 * gy - q3 * gz)
	qDot2 = 0.5 * (q0 * gx + q2 * gz - q3 * gy)
	qDot3 = 0.5 * (q0 * gy - q1 * gz + q3 * gx)
	qDot4 = 0.5 * (q0 * gz + q1 * gy - q2 * gx)

    // Compute feedback only if accelerometer measurement valid (avoids NaN in accelerometer normalisation)
	if(!((ax == 0.0) && (ay == 0.0) && (az == 0.0))) {

		// Normalise accelerometer measurement
		recipNorm = invSqrt(ax * ax + ay * ay + az * az)
		ax *= recipNorm
		ay *= recipNorm
		az *= recipNorm

		// Normalise magnetometer measurement
		recipNorm = invSqrt(mx * mx + my * my + mz * mz)
		mx *= recipNorm
		my *= recipNorm
		mz *= recipNorm

		// Auxiliary variables to avoid repeated arithmetic
		_2q0mx = 2.0 * q0 * mx
		_2q0my = 2.0 * q0 * my
		_2q0mz = 2.0 * q0 * mz
		_2q1mx = 2.0 * q1 * mx
		_2q0 = 2.0 * q0
		_2q1 = 2.0 * q1
		_2q2 = 2.0 * q2
		_2q3 = 2.0 * q3
		_2q0q2 = 2.0 * q0 * q2
		_2q2q3 = 2.0 * q2 * q3
		q0q0 = q0 * q0
		q0q1 = q0 * q1
		q0q2 = q0 * q2
		q0q3 = q0 * q3
		q1q1 = q1 * q1
		q1q2 = q1 * q2
		q1q3 = q1 * q3
		q2q2 = q2 * q2
		q2q3 = q2 * q3
		q3q3 = q3 * q3

		// Reference direction of Earth's magnetic field
		hx = mx * q0q0 - _2q0my * q3 + _2q0mz * q2 + mx * q1q1 + _2q1 * my * q2 + _2q1 * mz * q3 - mx * q2q2 - mx * q3q3
		hy = _2q0mx * q3 + my * q0q0 - _2q0mz * q1 + _2q1mx * q2 - my * q1q1 + my * q2q2 + _2q2 * mz * q3 - my * q3q3
		_2bx = sqrt(hx * hx + hy * hy)
		_2bz = -_2q0mx * q2 + _2q0my * q1 + mz * q0q0 + _2q1mx * q3 - mz * q1q1 + _2q2 * my * q3 - mz * q2q2 + mz * q3q3
		_4bx = 2.0 * _2bx
		_4bz = 2.0 * _2bz

		// Gradient decent algorithm corrective step
		s0 = -_2q2 * (2.0 * q1q3 - _2q0q2 - ax) + _2q1 * (2.0 * q0q1 + _2q2q3 - ay) - _2bz * q2 * (_2bx * (0.5 - q2q2 - q3q3) + _2bz * (q1q3 - q0q2) - mx) + (-_2bx * q3 + _2bz * q1) * (_2bx * (q1q2 - q0q3) + _2bz * (q0q1 + q2q3) - my) + _2bx * q2 * (_2bx * (q0q2 + q1q3) + _2bz * (0.5 - q1q1 - q2q2) - mz)
		s1 = _2q3 * (2.0 * q1q3 - _2q0q2 - ax) + _2q0 * (2.0 * q0q1 + _2q2q3 - ay) - 4.0 * q1 * (1 - 2.0 * q1q1 - 2.0 * q2q2 - az) + _2bz * q3 * (_2bx * (0.5 - q2q2 - q3q3) + _2bz * (q1q3 - q0q2) - mx) + (_2bx * q2 + _2bz * q0) * (_2bx * (q1q2 - q0q3) + _2bz * (q0q1 + q2q3) - my) + (_2bx * q3 - _4bz * q1) * (_2bx * (q0q2 + q1q3) + _2bz * (0.5 - q1q1 - q2q2) - mz)
		s2 = -_2q0 * (2.0 * q1q3 - _2q0q2 - ax) + _2q3 * (2.0 * q0q1 + _2q2q3 - ay) - 4.0 * q2 * (1 - 2.0 * q1q1 - 2.0 * q2q2 - az) + (-_4bx * q2 - _2bz * q0) * (_2bx * (0.5 - q2q2 - q3q3) + _2bz * (q1q3 - q0q2) - mx) + (_2bx * q1 + _2bz * q3) * (_2bx * (q1q2 - q0q3) + _2bz * (q0q1 + q2q3) - my) + (_2bx * q0 - _4bz * q2) * (_2bx * (q0q2 + q1q3) + _2bz * (0.5 - q1q1 - q2q2) - mz)
		s3 = _2q1 * (2.0 * q1q3 - _2q0q2 - ax) + _2q2 * (2.0 * q0q1 + _2q2q3 - ay) + (-_4bx * q3 + _2bz * q1) * (_2bx * (0.5 - q2q2 - q3q3) + _2bz * (q1q3 - q0q2) - mx) + (-_2bx * q0 + _2bz * q2) * (_2bx * (q1q2 - q0q3) + _2bz * (q0q1 + q2q3) - my) + _2bx * q1 * (_2bx * (q0q2 + q1q3) + _2bz * (0.5 - q1q1 - q2q2) - mz)
		recipNorm = invSqrt(s0 * s0 + s1 * s1 + s2 * s2 + s3 * s3) // normalise step magnitude
		s0 *= recipNorm
		s1 *= recipNorm
		s2 *= recipNorm
		s3 *= recipNorm

		// Apply feedback step
		qDot1 -= beta * s0
		qDot2 -= beta * s1
		qDot3 -= beta * s2
		qDot4 -= beta * s3
	}

    // Integrate rate of change of quaternion to yield quaternion
	q0 += qDot1 * (1.0 / sampleFreq)
	q1 += qDot2 * (1.0 / sampleFreq)
	q2 += qDot3 * (1.0 / sampleFreq)
	q3 += qDot4 * (1.0 / sampleFreq)

	// Normalise quaternion
	recipNorm = invSqrt(q0 * q0 + q1 * q1 + q2 * q2 + q3 * q3)
	q0 *= recipNorm
	q1 *= recipNorm
	q2 *= recipNorm
	q3 *= recipNorm
}

func AHRSupdateIMU(gx, gy, gz, ax, ay, az float32)
{
    recipNorm float32
	s0, s1, s2, s3 float32
	qDot1, qDot2, qDot3, qDot4 float32
	_2q0, _2q1, _2q2, _2q3, _4q0, _4q1, _4q2 ,_8q1, _8q2, q0q0, q1q1, q2q2, q3q3 float32

	// Rate of change of quaternion from gyroscope
	qDot1 = 0.5 * (-q1 * gx - q2 * gy - q3 * gz)
	qDot2 = 0.5 * (q0 * gx + q2 * gz - q3 * gy)
	qDot3 = 0.5 * (q0 * gy - q1 * gz + q3 * gx)
	qDot4 = 0.5 * (q0 * gz + q1 * gy - q2 * gx)

	// Compute feedback only if accelerometer measurement valid (avoids NaN in accelerometer normalisation)
	if(!((ax == 0.0) && (ay == 0.0) && (az == 0.0))) {

		// Normalise accelerometer measurement
		recipNorm = invSqrt(ax * ax + ay * ay + az * az)
		ax *= recipNorm
		ay *= recipNorm
		az *= recipNorm

		// Auxiliary variables to avoid repeated arithmetic
		_2q0 = 2.0 * q0
		_2q1 = 2.0 * q1
		_2q2 = 2.0 * q2
		_2q3 = 2.0 * q3
		_4q0 = 4.0 * q0
		_4q1 = 4.0 * q1
		_4q2 = 4.0 * q2
		_8q1 = 8.0 * q1
		_8q2 = 8.0 * q2
		q0q0 = q0 * q0
		q1q1 = q1 * q1
		q2q2 = q2 * q2
		q3q3 = q3 * q3

		// Gradient decent algorithm corrective step
		s0 = _4q0 * q2q2 + _2q2 * ax + _4q0 * q1q1 - _2q1 * ay
		s1 = _4q1 * q3q3 - _2q3 * ax + 4.0 * q0q0 * q1 - _2q0 * ay - _4q1 + _8q1 * q1q1 + _8q1 * q2q2 + _4q1 * az
		s2 = 4.0 * q0q0 * q2 + _2q0 * ax + _4q2 * q3q3 - _2q3 * ay - _4q2 + _8q2 * q1q1 + _8q2 * q2q2 + _4q2 * az
		s3 = 4.0 * q1q1 * q3 - _2q1 * ax + 4.0 * q2q2 * q3 - _2q2 * ay
		recipNorm = invSqrt(s0 * s0 + s1 * s1 + s2 * s2 + s3 * s3) // normalise step magnitude
		s0 *= recipNorm
		s1 *= recipNorm
		s2 *= recipNorm
		s3 *= recipNorm

		// Apply feedback step
		qDot1 -= beta * s0
		qDot2 -= beta * s1
		qDot3 -= beta * s2
		qDot4 -= beta * s3
	}

	// Integrate rate of change of quaternion to yield quaternion
	q0 += qDot1 * (1.0 / sampleFreq)
	q1 += qDot2 * (1.0 / sampleFreq)
	q2 += qDot3 * (1.0 / sampleFreq)
	q3 += qDot4 * (1.0 / sampleFreq)

	// Normalise quaternion
	recipNorm = invSqrt(q0 * q0 + q1 * q1 + q2 * q2 + q3 * q3)
	q0 *= recipNorm
	q1 *= recipNorm
	q2 *= recipNorm
	q3 *= recipNorm
}

func invSqrt(x float64) float64 {
    xhalf := float32(0.5) * x
	i := math.Float32bits(x)
	i = 0x5f3759df - i>>1
	x = math.Float32frombits(i)
	x = x * (1.5 - (xhalf * x * x))
	return x

    // the following line replaces the above. It may be faster, but it
    // also may be more or less accurate. Need to test. At the time the
    // above was written, CPUs did not have the instruction set built
    // into hardware. Now they do, but I'm not sure that applies to the
    // CPU that the RasPi is using. It appears that it does, but that is
    // not a guarantee of performance. There may also be a difference
    // between RasPi 2 and 3.
    //return 1.0 / Math.sqrt (x)
}<|endoftext|>


<commit_before>package lfs

import (
	"sync"
	"sync/atomic"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errutil"
	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/progress"
	"github.com/github/git-lfs/transfer"
	"github.com/rubyist/tracerx"
)

const (
	batchSize = 100
)

type Transferable interface {
	Oid() string
	Size() int64
	Name() string
	Path() string
	Object() *api.ObjectResource
	SetObject(*api.ObjectResource)
	// Legacy API check - TODO remove this and only support batch
	LegacyCheck() (*api.ObjectResource, error)
}

// TransferQueue organises the wider process of uploading and downloading,
// including calling the API, passing the actual transfer request to transfer
// adapters, and dealing with progress, errors and retries
type TransferQueue struct {
	direction         transfer.Direction
	adapter           transfer.TransferAdapter
	adapterInProgress bool
	adapterResultChan chan transfer.TransferResult
	adapterInitMutex  sync.Mutex
	dryRun            bool
	retrying          uint32
	meter             *progress.ProgressMeter
	errors            []error
	transferables     map[string]Transferable
	retries           []Transferable
	batcher           *Batcher
	apic              chan Transferable // Channel for processing individual API requests
	retriesc          chan Transferable // Channel for processing retries
	errorc            chan error        // Channel for processing errors
	watchers          []chan string
	trMutex           *sync.Mutex
	errorwait         sync.WaitGroup
	retrywait         sync.WaitGroup
	wait              sync.WaitGroup // Incremented on Add(), decremented on transfer complete or skip
	oldApiWorkers     int            // Number of non-batch API workers to spawn (deprecated)
	manifest          *transfer.Manifest
}

// newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction) *TransferQueue {
	logPath, _ := config.Config.Os.Get("GIT_LFS_PROGRESS")

	q := &TransferQueue{
		direction:     dir,
		dryRun:        dryRun,
		meter:         progress.NewProgressMeter(files, size, dryRun, logPath),
		apic:          make(chan Transferable, batchSize),
		retriesc:      make(chan Transferable, batchSize),
		errorc:        make(chan error),
		oldApiWorkers: config.Config.ConcurrentTransfers(),
		transferables: make(map[string]Transferable),
		trMutex:       &sync.Mutex{},
		manifest:      transfer.ConfigureManifest(transfer.NewManifest(), config.Config),
	}

	q.errorwait.Add(1)
	q.retrywait.Add(1)

	q.run()

	return q
}

// Add adds a Transferable to the transfer queue.
func (q *TransferQueue) Add(t Transferable) {
	q.trMutex.Lock()
	if _, ok := q.transferables[t.Oid()]; ok {
		q.trMutex.Unlock()
		return
	}
	q.transferables[t.Oid()] = t
	q.trMutex.Unlock()
	q.wait.Add(1)

	if q.batcher != nil {
		q.batcher.Add(t)
		return
	}

	q.apic <- t
}

func (q *TransferQueue) useAdapter(name string) {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapter != nil {
		if q.adapter.Name() == name {
			// re-use, this is the normal path
			return
		}
		// If the adapter we're using isn't the same as the one we've been
		// told to use now, must wait for the current one to finish then switch
		// This will probably never happen but is just in case server starts
		// changing adapter support in between batches
		q.finishAdapter()
	}
	q.adapter = q.manifest.NewAdapterOrDefault(name, q.direction)
}

func (q *TransferQueue) finishAdapter() {
	if q.adapterInProgress {
		q.adapter.End()
		q.adapterInProgress = false
		q.adapter = nil
	}
}

func (q *TransferQueue) addToAdapter(t Transferable) {
	tr := transfer.NewTransfer(t.Name(), t.Object(), t.Path())

	if q.dryRun {
		// Don't actually transfer
		res := transfer.TransferResult{tr, nil}
		q.handleTransferResult(res)
		return
	}
	err := q.ensureAdapterBegun()
	if err != nil {
		q.errorc <- err
		q.Skip(t.Size())
		q.wait.Done()
		return
	}
	q.adapter.Add(tr)
}

func (q *TransferQueue) Skip(size int64) {
	q.meter.Skip(size)
}

func (q *TransferQueue) transferKind() string {
	if q.direction == transfer.Download {
		return "download"
	} else {
		return "upload"
	}
}

func (q *TransferQueue) ensureAdapterBegun() error {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapterInProgress {
		return nil
	}

	adapterResultChan := make(chan transfer.TransferResult, 20)

	// Progress callback - receives byte updates
	cb := func(name string, total, read int64, current int) error {
		q.meter.TransferBytes(q.transferKind(), name, read, total, current)
		return nil
	}

	tracerx.Printf("tq: starting transfer adapter %q", q.adapter.Name())
	err := q.adapter.Begin(config.Config.ConcurrentTransfers(), cb, adapterResultChan)
	if err != nil {
		return err
	}
	q.adapterInProgress = true

	// Collector for completed transfers
	// q.wait.Done() in handleTransferResult is enough to know when this is complete for all transfers
	go func() {
		for res := range adapterResultChan {
			q.handleTransferResult(res)
		}
	}()

	return nil
}

func (q *TransferQueue) handleTransferResult(res transfer.TransferResult) {
	if res.Error != nil {
		if q.canRetry(res.Error) {
			tracerx.Printf("tq: retrying object %s", res.Transfer.Object.Oid)
			q.trMutex.Lock()
			t, ok := q.transferables[res.Transfer.Object.Oid]
			q.trMutex.Unlock()
			if ok {
				q.retry(t)
			} else {
				q.errorc <- res.Error
			}
		} else {
			q.errorc <- res.Error
		}
	} else {
		oid := res.Transfer.Object.Oid
		for _, c := range q.watchers {
			c <- oid
		}

		q.meter.FinishTransfer(res.Transfer.Name)
	}

	q.wait.Done()

}

// Wait waits for the queue to finish processing all transfers. Once Wait is
// called, Add will no longer add transferables to the queue. Any failed
// transfers will be automatically retried once.
func (q *TransferQueue) Wait() {
	if q.batcher != nil {
		q.batcher.Exit()
	}

	q.wait.Wait()

	// Handle any retries
	close(q.retriesc)
	q.retrywait.Wait()
	atomic.StoreUint32(&q.retrying, 1)

	if len(q.retries) > 0 {
		tracerx.Printf("tq: retrying %d failed transfers", len(q.retries))
		for _, t := range q.retries {
			q.Add(t)
		}
		if q.batcher != nil {
			q.batcher.Exit()
		}
		q.wait.Wait()
	}

	atomic.StoreUint32(&q.retrying, 0)

	close(q.apic)
	q.finishAdapter()
	close(q.errorc)

	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.meter.Finish()
	q.errorwait.Wait()
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, batchSize)
	q.watchers = append(q.watchers, c)
	return c
}

// individualApiRoutine processes the queue of transfers one at a time by making
// a POST call for each object, feeding the results to the transfer workers.
// If configured, the object transfers can still happen concurrently, the
// sequential nature here is only for the meta POST calls.
// TODO LEGACY API: remove when legacy API removed
func (q *TransferQueue) individualApiRoutine(apiWaiter chan interface{}) {
	for t := range q.apic {
		obj, err := t.LegacyCheck()
		if err != nil {
			if q.canRetry(err) {
				q.retry(t)
			} else {
				q.errorc <- err
			}
			q.wait.Done()
			continue
		}

		if apiWaiter != nil { // Signal to launch more individual api workers
			q.meter.Start()
			select {
			case apiWaiter <- 1:
			default:
			}
		}

		// Legacy API has no support for anything but basic transfer adapter
		q.useAdapter(transfer.BasicAdapterName)
		if obj != nil {
			t.SetObject(obj)
			q.meter.Add(t.Name())
			q.addToAdapter(t)
		} else {
			q.Skip(t.Size())
			q.wait.Done()
		}
	}
}

// legacyFallback is used when a batch request is made to a server that does
// not support the batch endpoint. When this happens, the Transferables are
// fed from the batcher into apic to be processed individually.
// TODO LEGACY API: remove when legacy API removed
func (q *TransferQueue) legacyFallback(failedBatch []interface{}) {
	tracerx.Printf("tq: batch api not implemented, falling back to individual")

	q.launchIndividualApiRoutines()

	for _, t := range failedBatch {
		q.apic <- t.(Transferable)
	}

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		for _, t := range batch {
			q.apic <- t.(Transferable)
		}
	}
}

// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
	var startProgress sync.Once

	transferAdapterNames := q.manifest.GetAdapterNames(q.direction)

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		tracerx.Printf("tq: sending batch of size %d", len(batch))

		transfers := make([]*api.ObjectResource, 0, len(batch))
		for _, i := range batch {
			t := i.(Transferable)
			transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
		}

		if len(transfers) == 0 {
			continue
		}

		objs, adapterName, err := api.Batch(config.Config, transfers, q.transferKind(), transferAdapterNames)
		if err != nil {
			if errutil.IsNotImplementedError(err) {
				git.Config.SetLocal("", "lfs.batch", "false")

				go q.legacyFallback(batch)
				return
			}

			if q.canRetry(err) {
				for _, t := range batch {
					q.retry(t.(Transferable))
				}
			} else {
				q.errorc <- err
			}

			q.wait.Add(-len(transfers))
			continue
		}

		q.useAdapter(adapterName)
		startProgress.Do(q.meter.Start)

		for _, o := range objs {
			if o.Error != nil {
				q.errorc <- errutil.Errorf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
				q.Skip(o.Size)
				q.wait.Done()
				continue
			}

			if _, ok := o.Rel(q.transferKind()); ok {
				// This object needs to be transferred
				q.trMutex.Lock()
				transfer, ok := q.transferables[o.Oid]
				q.trMutex.Unlock()

				if ok {
					transfer.SetObject(o)
					q.meter.Add(transfer.Name())
					q.addToAdapter(transfer)
				} else {
					q.Skip(transfer.Size())
					q.wait.Done()
				}
			} else {
				q.Skip(o.Size)
				q.wait.Done()
			}
		}
	}
}

// This goroutine collects errors returned from transfers
func (q *TransferQueue) errorCollector() {
	for err := range q.errorc {
		q.errors = append(q.errors, err)
	}
	q.errorwait.Done()
}

func (q *TransferQueue) retryCollector() {
	for t := range q.retriesc {
		q.retries = append(q.retries, t)
	}
	q.retrywait.Done()
}

// launchIndividualApiRoutines first launches a single api worker. When it
// receives the first successful api request it launches workers - 1 more
// workers. This prevents being prompted for credentials multiple times at once
// when they're needed.
func (q *TransferQueue) launchIndividualApiRoutines() {
	go func() {
		apiWaiter := make(chan interface{})
		go q.individualApiRoutine(apiWaiter)

		<-apiWaiter

		for i := 0; i < q.oldApiWorkers-1; i++ {
			go q.individualApiRoutine(nil)
		}
	}()
}

// run starts the transfer queue, doing individual or batch transfers depending
// on the Config.BatchTransfer() value. run will transfer files sequentially or
// concurrently depending on the Config.ConcurrentTransfers() value.
func (q *TransferQueue) run() {
	go q.errorCollector()
	go q.retryCollector()

	if config.Config.BatchTransfer() {
		tracerx.Printf("tq: running as batched queue, batch size of %d", batchSize)
		q.batcher = NewBatcher(batchSize)
		go q.batchApiRoutine()
	} else {
		tracerx.Printf("tq: running as individual queue")
		q.launchIndividualApiRoutines()
	}
}

func (q *TransferQueue) retry(t Transferable) {
	q.retriesc <- t
}

func (q *TransferQueue) canRetry(err error) bool {
	if !errutil.IsRetriableError(err) || atomic.LoadUint32(&q.retrying) == 1 {
		return false
	}

	return true
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []error {
	return q.errors
}
<commit_msg>revert last change<commit_after>package lfs

import (
	"sync"
	"sync/atomic"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/config"
	"github.com/github/git-lfs/errutil"
	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/progress"
	"github.com/github/git-lfs/transfer"
	"github.com/rubyist/tracerx"
)

const (
	batchSize = 100
)

type Transferable interface {
	Oid() string
	Size() int64
	Name() string
	Path() string
	Object() *api.ObjectResource
	SetObject(*api.ObjectResource)
	// Legacy API check - TODO remove this and only support batch
	LegacyCheck() (*api.ObjectResource, error)
}

// TransferQueue organises the wider process of uploading and downloading,
// including calling the API, passing the actual transfer request to transfer
// adapters, and dealing with progress, errors and retries
type TransferQueue struct {
	direction         transfer.Direction
	adapter           transfer.TransferAdapter
	adapterInProgress bool
	adapterResultChan chan transfer.TransferResult
	adapterInitMutex  sync.Mutex
	dryRun            bool
	retrying          uint32
	meter             *progress.ProgressMeter
	errors            []error
	transferables     map[string]Transferable
	retries           []Transferable
	batcher           *Batcher
	apic              chan Transferable // Channel for processing individual API requests
	retriesc          chan Transferable // Channel for processing retries
	errorc            chan error        // Channel for processing errors
	watchers          []chan string
	trMutex           *sync.Mutex
	errorwait         sync.WaitGroup
	retrywait         sync.WaitGroup
	wait              sync.WaitGroup // Incremented on Add(), decremented on transfer complete or skip
	oldApiWorkers     int            // Number of non-batch API workers to spawn (deprecated)
	manifest          *transfer.Manifest
}

// newTransferQueue builds a TransferQueue, direction and underlying mechanism determined by adapter
func newTransferQueue(files int, size int64, dryRun bool, dir transfer.Direction) *TransferQueue {
	logPath, _ := config.Config.Os.Get("GIT_LFS_PROGRESS")

	q := &TransferQueue{
		direction:     dir,
		dryRun:        dryRun,
		meter:         progress.NewProgressMeter(files, size, dryRun, logPath),
		apic:          make(chan Transferable, batchSize),
		retriesc:      make(chan Transferable, batchSize),
		errorc:        make(chan error),
		oldApiWorkers: config.Config.ConcurrentTransfers(),
		transferables: make(map[string]Transferable),
		trMutex:       &sync.Mutex{},
		manifest:      transfer.ConfigureManifest(transfer.NewManifest(), config.Config),
	}

	q.errorwait.Add(1)
	q.retrywait.Add(1)

	q.run()

	return q
}

// Add adds a Transferable to the transfer queue.
func (q *TransferQueue) Add(t Transferable) {
	q.wait.Add(1)
	q.trMutex.Lock()
	q.transferables[t.Oid()] = t
	q.trMutex.Unlock()

	if q.batcher != nil {
		q.batcher.Add(t)
		return
	}

	q.apic <- t
}

func (q *TransferQueue) useAdapter(name string) {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapter != nil {
		if q.adapter.Name() == name {
			// re-use, this is the normal path
			return
		}
		// If the adapter we're using isn't the same as the one we've been
		// told to use now, must wait for the current one to finish then switch
		// This will probably never happen but is just in case server starts
		// changing adapter support in between batches
		q.finishAdapter()
	}
	q.adapter = q.manifest.NewAdapterOrDefault(name, q.direction)
}

func (q *TransferQueue) finishAdapter() {
	if q.adapterInProgress {
		q.adapter.End()
		q.adapterInProgress = false
		q.adapter = nil
	}
}

func (q *TransferQueue) addToAdapter(t Transferable) {
	tr := transfer.NewTransfer(t.Name(), t.Object(), t.Path())

	if q.dryRun {
		// Don't actually transfer
		res := transfer.TransferResult{tr, nil}
		q.handleTransferResult(res)
		return
	}
	err := q.ensureAdapterBegun()
	if err != nil {
		q.errorc <- err
		q.Skip(t.Size())
		q.wait.Done()
		return
	}
	q.adapter.Add(tr)
}

func (q *TransferQueue) Skip(size int64) {
	q.meter.Skip(size)
}

func (q *TransferQueue) transferKind() string {
	if q.direction == transfer.Download {
		return "download"
	} else {
		return "upload"
	}
}

func (q *TransferQueue) ensureAdapterBegun() error {
	q.adapterInitMutex.Lock()
	defer q.adapterInitMutex.Unlock()

	if q.adapterInProgress {
		return nil
	}

	adapterResultChan := make(chan transfer.TransferResult, 20)

	// Progress callback - receives byte updates
	cb := func(name string, total, read int64, current int) error {
		q.meter.TransferBytes(q.transferKind(), name, read, total, current)
		return nil
	}

	tracerx.Printf("tq: starting transfer adapter %q", q.adapter.Name())
	err := q.adapter.Begin(config.Config.ConcurrentTransfers(), cb, adapterResultChan)
	if err != nil {
		return err
	}
	q.adapterInProgress = true

	// Collector for completed transfers
	// q.wait.Done() in handleTransferResult is enough to know when this is complete for all transfers
	go func() {
		for res := range adapterResultChan {
			q.handleTransferResult(res)
		}
	}()

	return nil
}

func (q *TransferQueue) handleTransferResult(res transfer.TransferResult) {
	if res.Error != nil {
		if q.canRetry(res.Error) {
			tracerx.Printf("tq: retrying object %s", res.Transfer.Object.Oid)
			q.trMutex.Lock()
			t, ok := q.transferables[res.Transfer.Object.Oid]
			q.trMutex.Unlock()
			if ok {
				q.retry(t)
			} else {
				q.errorc <- res.Error
			}
		} else {
			q.errorc <- res.Error
		}
	} else {
		oid := res.Transfer.Object.Oid
		for _, c := range q.watchers {
			c <- oid
		}

		q.meter.FinishTransfer(res.Transfer.Name)
	}

	q.wait.Done()

}

// Wait waits for the queue to finish processing all transfers. Once Wait is
// called, Add will no longer add transferables to the queue. Any failed
// transfers will be automatically retried once.
func (q *TransferQueue) Wait() {
	if q.batcher != nil {
		q.batcher.Exit()
	}

	q.wait.Wait()

	// Handle any retries
	close(q.retriesc)
	q.retrywait.Wait()
	atomic.StoreUint32(&q.retrying, 1)

	if len(q.retries) > 0 {
		tracerx.Printf("tq: retrying %d failed transfers", len(q.retries))
		for _, t := range q.retries {
			q.Add(t)
		}
		if q.batcher != nil {
			q.batcher.Exit()
		}
		q.wait.Wait()
	}

	atomic.StoreUint32(&q.retrying, 0)

	close(q.apic)
	q.finishAdapter()
	close(q.errorc)

	for _, watcher := range q.watchers {
		close(watcher)
	}

	q.meter.Finish()
	q.errorwait.Wait()
}

// Watch returns a channel where the queue will write the OID of each transfer
// as it completes. The channel will be closed when the queue finishes processing.
func (q *TransferQueue) Watch() chan string {
	c := make(chan string, batchSize)
	q.watchers = append(q.watchers, c)
	return c
}

// individualApiRoutine processes the queue of transfers one at a time by making
// a POST call for each object, feeding the results to the transfer workers.
// If configured, the object transfers can still happen concurrently, the
// sequential nature here is only for the meta POST calls.
// TODO LEGACY API: remove when legacy API removed
func (q *TransferQueue) individualApiRoutine(apiWaiter chan interface{}) {
	for t := range q.apic {
		obj, err := t.LegacyCheck()
		if err != nil {
			if q.canRetry(err) {
				q.retry(t)
			} else {
				q.errorc <- err
			}
			q.wait.Done()
			continue
		}

		if apiWaiter != nil { // Signal to launch more individual api workers
			q.meter.Start()
			select {
			case apiWaiter <- 1:
			default:
			}
		}

		// Legacy API has no support for anything but basic transfer adapter
		q.useAdapter(transfer.BasicAdapterName)
		if obj != nil {
			t.SetObject(obj)
			q.meter.Add(t.Name())
			q.addToAdapter(t)
		} else {
			q.Skip(t.Size())
			q.wait.Done()
		}
	}
}

// legacyFallback is used when a batch request is made to a server that does
// not support the batch endpoint. When this happens, the Transferables are
// fed from the batcher into apic to be processed individually.
// TODO LEGACY API: remove when legacy API removed
func (q *TransferQueue) legacyFallback(failedBatch []interface{}) {
	tracerx.Printf("tq: batch api not implemented, falling back to individual")

	q.launchIndividualApiRoutines()

	for _, t := range failedBatch {
		q.apic <- t.(Transferable)
	}

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		for _, t := range batch {
			q.apic <- t.(Transferable)
		}
	}
}

// batchApiRoutine processes the queue of transfers using the batch endpoint,
// making only one POST call for all objects. The results are then handed
// off to the transfer workers.
func (q *TransferQueue) batchApiRoutine() {
	var startProgress sync.Once

	transferAdapterNames := q.manifest.GetAdapterNames(q.direction)

	for {
		batch := q.batcher.Next()
		if batch == nil {
			break
		}

		tracerx.Printf("tq: sending batch of size %d", len(batch))

		transfers := make([]*api.ObjectResource, 0, len(batch))
		for _, i := range batch {
			t := i.(Transferable)
			transfers = append(transfers, &api.ObjectResource{Oid: t.Oid(), Size: t.Size()})
		}

		if len(transfers) == 0 {
			continue
		}

		objs, adapterName, err := api.Batch(config.Config, transfers, q.transferKind(), transferAdapterNames)
		if err != nil {
			if errutil.IsNotImplementedError(err) {
				git.Config.SetLocal("", "lfs.batch", "false")

				go q.legacyFallback(batch)
				return
			}

			if q.canRetry(err) {
				for _, t := range batch {
					q.retry(t.(Transferable))
				}
			} else {
				q.errorc <- err
			}

			q.wait.Add(-len(transfers))
			continue
		}

		q.useAdapter(adapterName)
		startProgress.Do(q.meter.Start)

		for _, o := range objs {
			if o.Error != nil {
				q.errorc <- errutil.Errorf(o.Error, "[%v] %v", o.Oid, o.Error.Message)
				q.Skip(o.Size)
				q.wait.Done()
				continue
			}

			if _, ok := o.Rel(q.transferKind()); ok {
				// This object needs to be transferred
				q.trMutex.Lock()
				transfer, ok := q.transferables[o.Oid]
				q.trMutex.Unlock()

				if ok {
					transfer.SetObject(o)
					q.meter.Add(transfer.Name())
					q.addToAdapter(transfer)
				} else {
					q.Skip(transfer.Size())
					q.wait.Done()
				}
			} else {
				q.Skip(o.Size)
				q.wait.Done()
			}
		}
	}
}

// This goroutine collects errors returned from transfers
func (q *TransferQueue) errorCollector() {
	for err := range q.errorc {
		q.errors = append(q.errors, err)
	}
	q.errorwait.Done()
}

func (q *TransferQueue) retryCollector() {
	for t := range q.retriesc {
		q.retries = append(q.retries, t)
	}
	q.retrywait.Done()
}

// launchIndividualApiRoutines first launches a single api worker. When it
// receives the first successful api request it launches workers - 1 more
// workers. This prevents being prompted for credentials multiple times at once
// when they're needed.
func (q *TransferQueue) launchIndividualApiRoutines() {
	go func() {
		apiWaiter := make(chan interface{})
		go q.individualApiRoutine(apiWaiter)

		<-apiWaiter

		for i := 0; i < q.oldApiWorkers-1; i++ {
			go q.individualApiRoutine(nil)
		}
	}()
}

// run starts the transfer queue, doing individual or batch transfers depending
// on the Config.BatchTransfer() value. run will transfer files sequentially or
// concurrently depending on the Config.ConcurrentTransfers() value.
func (q *TransferQueue) run() {
	go q.errorCollector()
	go q.retryCollector()

	if config.Config.BatchTransfer() {
		tracerx.Printf("tq: running as batched queue, batch size of %d", batchSize)
		q.batcher = NewBatcher(batchSize)
		go q.batchApiRoutine()
	} else {
		tracerx.Printf("tq: running as individual queue")
		q.launchIndividualApiRoutines()
	}
}

func (q *TransferQueue) retry(t Transferable) {
	q.retriesc <- t
}

func (q *TransferQueue) canRetry(err error) bool {
	if !errutil.IsRetriableError(err) || atomic.LoadUint32(&q.retrying) == 1 {
		return false
	}

	return true
}

// Errors returns any errors encountered during transfer.
func (q *TransferQueue) Errors() []error {
	return q.errors
}
<|endoftext|>


<commit_before>package shellwords

import (
	"errors"
	"os"
	"regexp"
)

var (
	ParseEnv      bool = false
	ParseBacktick bool = false
)

var envRe = regexp.MustCompile(`\$({[a-zA-Z0-9_]+}|[a-zA-Z0-9_]+)`)

func isSpace(r rune) bool {
	switch r {
	case ' ', '\t', '\r', '\n':
		return true
	}
	return false
}

func replaceEnv(s string) string {
	return envRe.ReplaceAllStringFunc(s, func(s string) string {
		s = s[1:]
		if s[0] == '{' {
			s = s[1 : len(s)-1]
		}
		return os.Getenv(s)
	})
}

type Parser struct {
	ParseEnv      bool
	ParseBacktick bool
	Position      int
}

func NewParser() *Parser {
	return &Parser{ParseEnv, ParseBacktick, 0}
}

func (p *Parser) Parse(line string) ([]string, error) {
	args := []string{}
	buf := ""
	var escaped, doubleQuoted, singleQuoted, backQuote, dollarQuote bool
	backtick := ""

	pos := -1
	got := false

loop:
	for i, r := range line {
		if escaped {
			buf += string(r)
			escaped = false
			continue
		}

		if r == '\\' {
			if singleQuoted {
				buf += string(r)
			} else {
				escaped = true
			}
			continue
		}

		if isSpace(r) {
			if singleQuoted || doubleQuoted || backQuote || dollarQuote {
				buf += string(r)
				backtick += string(r)
			} else if got {
				if p.ParseEnv {
					buf = replaceEnv(buf)
				}
				args = append(args, buf)
				buf = ""
				got = false
			}
			continue
		}

		switch r {
		case '`':
			if !singleQuoted && !doubleQuoted && !dollarQuote {
				if p.ParseBacktick {
					if backQuote {
						out, err := shellRun(backtick)
						if err != nil {
							return nil, err
						}
						buf = out
					}
					backtick = ""
					backQuote = !backQuote
					continue
				}
				backtick = ""
				backQuote = !backQuote
			}
		case ')':
			if !singleQuoted && !doubleQuoted && !backQuote {
				if p.ParseBacktick {
					if dollarQuote {
						out, err := shellRun(backtick)
						if err != nil {
							return nil, err
						}
						buf = out
					}
					backtick = ""
					dollarQuote = !dollarQuote
					continue
				}
				backtick = ""
				dollarQuote = !dollarQuote
			}
		case '(':
			if !singleQuoted && !doubleQuoted && !backQuote && !dollarQuote && len(buf) > 0 && buf == "$" {
				dollarQuote = true
				buf += "("
				continue
			}
		case '"':
			if !singleQuoted && !dollarQuote {
				doubleQuoted = !doubleQuoted
				continue
			}
		case '\'':
			if !doubleQuoted && !dollarQuote {
				singleQuoted = !singleQuoted
				continue
			}
		case ';', '&', '|', '<', '>':
			if !(escaped || singleQuoted || doubleQuoted || backQuote) {
				pos = i
				break loop
			}
		}

		got = true
		buf += string(r)
		if backQuote || dollarQuote {
			backtick += string(r)
		}
	}

	if got {
		if p.ParseEnv {
			buf = replaceEnv(buf)
		}
		args = append(args, buf)
	}

	if escaped || singleQuoted || doubleQuoted || backQuote || dollarQuote {
		return nil, errors.New("invalid command line string")
	}

	p.Position = pos

	return args, nil
}

func Parse(line string) ([]string, error) {
	return NewParser().Parse(line)
}
<commit_msg>paren should be an error<commit_after>package shellwords

import (
	"errors"
	"os"
	"regexp"
)

var (
	ParseEnv      bool = false
	ParseBacktick bool = false
)

var envRe = regexp.MustCompile(`\$({[a-zA-Z0-9_]+}|[a-zA-Z0-9_]+)`)

func isSpace(r rune) bool {
	switch r {
	case ' ', '\t', '\r', '\n':
		return true
	}
	return false
}

func replaceEnv(s string) string {
	return envRe.ReplaceAllStringFunc(s, func(s string) string {
		s = s[1:]
		if s[0] == '{' {
			s = s[1 : len(s)-1]
		}
		return os.Getenv(s)
	})
}

type Parser struct {
	ParseEnv      bool
	ParseBacktick bool
	Position      int
}

func NewParser() *Parser {
	return &Parser{ParseEnv, ParseBacktick, 0}
}

func (p *Parser) Parse(line string) ([]string, error) {
	args := []string{}
	buf := ""
	var escaped, doubleQuoted, singleQuoted, backQuote, dollarQuote bool
	backtick := ""

	pos := -1
	got := false

loop:
	for i, r := range line {
		if escaped {
			buf += string(r)
			escaped = false
			continue
		}

		if r == '\\' {
			if singleQuoted {
				buf += string(r)
			} else {
				escaped = true
			}
			continue
		}

		if isSpace(r) {
			if singleQuoted || doubleQuoted || backQuote || dollarQuote {
				buf += string(r)
				backtick += string(r)
			} else if got {
				if p.ParseEnv {
					buf = replaceEnv(buf)
				}
				args = append(args, buf)
				buf = ""
				got = false
			}
			continue
		}

		switch r {
		case '`':
			if !singleQuoted && !doubleQuoted && !dollarQuote {
				if p.ParseBacktick {
					if backQuote {
						out, err := shellRun(backtick)
						if err != nil {
							return nil, err
						}
						buf = out
					}
					backtick = ""
					backQuote = !backQuote
					continue
				}
				backtick = ""
				backQuote = !backQuote
			}
		case ')':
			if !singleQuoted && !doubleQuoted && !backQuote {
				if p.ParseBacktick {
					if dollarQuote {
						out, err := shellRun(backtick)
						if err != nil {
							return nil, err
						}
						buf = out
					}
					backtick = ""
					dollarQuote = !dollarQuote
					continue
				}
				backtick = ""
				dollarQuote = !dollarQuote
			}
		case '(':
			if !singleQuoted && !doubleQuoted && !backQuote {
				if !dollarQuote && len(buf) > 0 && buf == "$" {
					dollarQuote = true
					buf += "("
					continue
				} else {
					return nil, errors.New("invalid command line string")
				}
			}
		case '"':
			if !singleQuoted && !dollarQuote {
				doubleQuoted = !doubleQuoted
				continue
			}
		case '\'':
			if !doubleQuoted && !dollarQuote {
				singleQuoted = !singleQuoted
				continue
			}
		case ';', '&', '|', '<', '>':
			if !(escaped || singleQuoted || doubleQuoted || backQuote) {
				pos = i
				break loop
			}
		}

		got = true
		buf += string(r)
		if backQuote || dollarQuote {
			backtick += string(r)
		}
	}

	if got {
		if p.ParseEnv {
			buf = replaceEnv(buf)
		}
		args = append(args, buf)
	}

	if escaped || singleQuoted || doubleQuoted || backQuote || dollarQuote {
		return nil, errors.New("invalid command line string")
	}

	p.Position = pos

	return args, nil
}

func Parse(line string) ([]string, error) {
	return NewParser().Parse(line)
}
<|endoftext|>


<commit_before>package lib

import "fmt"

// Plan on Vultr
type Plan struct {
	ID        int    `json:"VPSPLANID,string"`
	Name      string `json:"name"`
	VCpus     int    `json:"vcpu_count,string"`
	RAM       string `json:"ram"`
	Disk      string `json:"disk"`
	Bandwidth string `json:"bandwidth"`
	Price     string `json:"price_per_month"`
	Regions   []int  `json:"available_locations"`
}

// Add sort ability
type SortByID []Plan

func (b SortByID) Len() int           { return len(b) }
func (b SortByID) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
func (b SortByID) Less(i, j int) bool { return b[i].ID < b[j].ID }

func (c *Client) GetPlans() ([]Plan, error) {
	var planMap map[string]Plan
	if err := c.get(`plans/list`, &planMap); err != nil {
		return nil, err
	}

	var planList []Plan
	for _, plan := range planMap {
		planList = append(planList, plan)
	}
	return planList, nil
}

func (c *Client) GetVC2Plans() ([]Plan, error) {
	var planMap map[string]Plan
	if err := c.get(`plans/list_vc2`, &planMap); err != nil {
		return nil, err
	}

	var planList []Plan
	for _, plan := range planMap {
		planList = append(planList, plan)
	}
	return planList, nil
}

func (c *Client) GetAvailablePlansForRegion(id int) (planIDs []int, err error) {
	if err := c.get(fmt.Sprintf(`regions/availability?DCID=%v`, id), &planIDs); err != nil {
		return nil, err
	}
	return
}
<commit_msg>remove custom sort ability<commit_after>package lib

import "fmt"

// Plan on Vultr
type Plan struct {
	ID        int    `json:"VPSPLANID,string"`
	Name      string `json:"name"`
	VCpus     int    `json:"vcpu_count,string"`
	RAM       string `json:"ram"`
	Disk      string `json:"disk"`
	Bandwidth string `json:"bandwidth"`
	Price     string `json:"price_per_month"`
	Regions   []int  `json:"available_locations"`
}

func (c *Client) GetPlans() ([]Plan, error) {
	var planMap map[string]Plan
	if err := c.get(`plans/list`, &planMap); err != nil {
		return nil, err
	}

	var planList []Plan
	for _, plan := range planMap {
		planList = append(planList, plan)
	}
	return planList, nil
}

func (c *Client) GetVC2Plans() ([]Plan, error) {
	var planMap map[string]Plan
	if err := c.get(`plans/list_vc2`, &planMap); err != nil {
		return nil, err
	}

	var planList []Plan
	for _, plan := range planMap {
		planList = append(planList, plan)
	}
	return planList, nil
}

func (c *Client) GetAvailablePlansForRegion(id int) (planIDs []int, err error) {
	if err := c.get(fmt.Sprintf(`regions/availability?DCID=%v`, id), &planIDs); err != nil {
		return nil, err
	}
	return
}
<|endoftext|>


<commit_before>package proxy_test

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/tendermint/tendermint/lite"
	"github.com/tendermint/tendermint/lite/proxy"
	"github.com/tendermint/tendermint/types"
)

var (
	deabBeefTxs           = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")}
	deadBeefRipEmd160Hash = deabBeefTxs.Hash()
)

func TestValidateBlock(t *testing.T) {
	tests := []struct {
		block   *types.Block
		commit  lite.Commit
		wantErr string
	}{
		{
			block: nil, wantErr: "non-nil Block",
		},
		{
			block: &types.Block{}, wantErr: "nil Header",
		},
		{
			block: &types.Block{Header: new(types.Header)},
		},

		// Start Header.Height mismatch test
		{
			block:   &types.Block{Header: &types.Header{Height: 10}},
			commit:  lite.Commit{Header: &types.Header{Height: 11}},
			wantErr: "don't match - 10 vs 11",
		},

		{
			block:  &types.Block{Header: &types.Header{Height: 11}},
			commit: lite.Commit{Header: &types.Header{Height: 11}},
		},
		// End Header.Height mismatch test

		// Start Header.Hash mismatch test
		{
			block: &types.Block{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
			commit:  lite.Commit{Header: &types.Header{Height: 11}},
			wantErr: "Headers don't match",
		},

		{
			block: &types.Block{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
		},
		// End Header.Hash mismatch test

		// Start Header.Data hash mismatch test
		{
			block: &types.Block{
				Header: &types.Header{Height: 11},
				Data:   &types.Data{Txs: []types.Tx{[]byte("0xDE"), []byte("AD")}},
			},
			commit: lite.Commit{
				Header: &types.Header{Height: 11},
				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("0xDEADBEEF")}},
			},
			wantErr: "Data hash doesn't match header",
		},
		{
			block: &types.Block{
				Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash},
				Data:   &types.Data{Txs: deabBeefTxs},
			},
			commit: lite.Commit{
				Header: &types.Header{Height: 11},
				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}},
			},
		},
		// End Header.Data hash mismatch test
	}

	assert := assert.New(t)

	for i, tt := range tests {
		err := proxy.ValidateBlock(tt.block, tt.commit)
		if tt.wantErr != "" {
			if err == nil {
				assert.FailNowf("Unexpectedly passed", "#%d", i)
			} else {
				assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i)
			}
			continue
		}

		assert.Nil(err, "#%d: expecting a nil error", i)
	}
}

func TestValidateBlockMeta(t *testing.T) {
	tests := []struct {
		meta    *types.BlockMeta
		commit  lite.Commit
		wantErr string
	}{
		{
			meta: nil, wantErr: "non-nil BlockMeta",
		},
		{
			meta: &types.BlockMeta{}, wantErr: "non-nil Header",
		},
		{
			meta: &types.BlockMeta{Header: new(types.Header)},
		},

		// Start Header.Height mismatch test
		{
			meta:    &types.BlockMeta{Header: &types.Header{Height: 10}},
			commit:  lite.Commit{Header: &types.Header{Height: 11}},
			wantErr: "don't match - 10 vs 11",
		},

		{
			meta:   &types.BlockMeta{Header: &types.Header{Height: 11}},
			commit: lite.Commit{Header: &types.Header{Height: 11}},
		},
		// End Header.Height mismatch test

		// Start Headers don't match test
		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
			commit:  lite.Commit{Header: &types.Header{Height: 11}},
			wantErr: "Headers don't match",
		},

		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
		},

		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height: 11,
					// TODO: (@odeke-em) inquire why ValidatorsHash has to be non-blank
					// for the Header to be hashed. Perhaps this is a security hole because
					// an aggressor could perhaps pass in headers that don't have
					// ValidatorsHash set and we won't be able to validate blocks.
					ValidatorsHash: []byte("lite-test"),
					// TODO: (@odeke-em) file an issue with Tendermint to get them to update
					// to the latest go-wire, then no more need for this value fill to avoid
					// the time zero value of less than 1970.
					Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash},
			},
			wantErr: "Headers don't match",
		},

		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height: 11, DataHash: deadBeefRipEmd160Hash,
					ValidatorsHash: []byte("Tendermint"),
					Time:           time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{
					Height: 11, DataHash: deadBeefRipEmd160Hash,
					ValidatorsHash: []byte("Tendermint"),
					Time:           time.Date(2017, 1, 2, 2, 1, 1, 1, time.UTC),
				},
				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}},
			},
			wantErr: "Headers don't match",
		},

		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height: 11, DataHash: deadBeefRipEmd160Hash,
					ValidatorsHash: []byte("Tendermint"),
					Time:           time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{
					Height: 11, DataHash: deadBeefRipEmd160Hash,
					ValidatorsHash: []byte("Tendermint-x"),
					Time:           time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC),
				},
				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}},
			},
			wantErr: "Headers don't match",
		},
		// End Headers don't match test
	}

	assert := assert.New(t)

	for i, tt := range tests {
		err := proxy.ValidateBlockMeta(tt.meta, tt.commit)
		if tt.wantErr != "" {
			if err == nil {
				assert.FailNowf("Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr)
			} else {
				assert.Contains(err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i)
			}
			continue
		}

		assert.Nil(err, "#%d: expecting a nil error", i)
	}
}
<commit_msg>Review feedback from @melekes<commit_after>package proxy_test

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/tendermint/tendermint/lite"
	"github.com/tendermint/tendermint/lite/proxy"
	"github.com/tendermint/tendermint/types"
)

var (
	deadBeefTxs           = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")}
	deadBeefRipEmd160Hash = deadBeefTxs.Hash()
)

func TestValidateBlock(t *testing.T) {
	tests := []struct {
		block   *types.Block
		commit  lite.Commit
		wantErr string
	}{
		{
			block: nil, wantErr: "non-nil Block",
		},
		{
			block: &types.Block{}, wantErr: "nil Header",
		},
		{
			block: &types.Block{Header: new(types.Header)},
		},

		// Start Header.Height mismatch test
		{
			block:   &types.Block{Header: &types.Header{Height: 10}},
			commit:  lite.Commit{Header: &types.Header{Height: 11}},
			wantErr: "don't match - 10 vs 11",
		},

		{
			block:  &types.Block{Header: &types.Header{Height: 11}},
			commit: lite.Commit{Header: &types.Header{Height: 11}},
		},
		// End Header.Height mismatch test

		// Start Header.Hash mismatch test
		{
			block: &types.Block{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
			commit:  lite.Commit{Header: &types.Header{Height: 11}},
			wantErr: "Headers don't match",
		},

		{
			block: &types.Block{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
		},
		// End Header.Hash mismatch test

		// Start Header.Data hash mismatch test
		{
			block: &types.Block{
				Header: &types.Header{Height: 11},
				Data:   &types.Data{Txs: []types.Tx{[]byte("0xDE"), []byte("AD")}},
			},
			commit: lite.Commit{
				Header: &types.Header{Height: 11},
				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("0xDEADBEEF")}},
			},
			wantErr: "Data hash doesn't match header",
		},
		{
			block: &types.Block{
				Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash},
				Data:   &types.Data{Txs: deadBeefTxs},
			},
			commit: lite.Commit{
				Header: &types.Header{Height: 11},
				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}},
			},
		},
		// End Header.Data hash mismatch test
	}

	for i, tt := range tests {
		err := proxy.ValidateBlock(tt.block, tt.commit)
		if tt.wantErr != "" {
			if err == nil {
				assert.FailNowf(t, "Unexpectedly passed", "#%d", i)
			} else {
				assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i)
			}
			continue
		}

		assert.Nil(t, err, "#%d: expecting a nil error", i)
	}
}

func TestValidateBlockMeta(t *testing.T) {
	tests := []struct {
		meta    *types.BlockMeta
		commit  lite.Commit
		wantErr string
	}{
		{
			meta: nil, wantErr: "non-nil BlockMeta",
		},
		{
			meta: &types.BlockMeta{}, wantErr: "non-nil Header",
		},
		{
			meta: &types.BlockMeta{Header: new(types.Header)},
		},

		// Start Header.Height mismatch test
		{
			meta:    &types.BlockMeta{Header: &types.Header{Height: 10}},
			commit:  lite.Commit{Header: &types.Header{Height: 11}},
			wantErr: "don't match - 10 vs 11",
		},

		{
			meta:   &types.BlockMeta{Header: &types.Header{Height: 11}},
			commit: lite.Commit{Header: &types.Header{Height: 11}},
		},
		// End Header.Height mismatch test

		// Start Headers don't match test
		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
			commit:  lite.Commit{Header: &types.Header{Height: 11}},
			wantErr: "Headers don't match",
		},

		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{
					Height:         11,
					Time:           time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
					ValidatorsHash: []byte("Tendermint"),
				},
			},
		},

		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height: 11,
					// TODO: (@odeke-em) inquire why ValidatorsHash has to be non-blank
					// for the Header to be hashed. Perhaps this is a security hole because
					// an aggressor could perhaps pass in headers that don't have
					// ValidatorsHash set and we won't be able to validate blocks.
					ValidatorsHash: []byte("lite-test"),
					// TODO: (@odeke-em) file an issue with Tendermint to get them to update
					// to the latest go-wire, then no more need for this value fill to avoid
					// the time zero value of less than 1970.
					Time: time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{Height: 11, DataHash: deadBeefRipEmd160Hash},
			},
			wantErr: "Headers don't match",
		},

		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height: 11, DataHash: deadBeefRipEmd160Hash,
					ValidatorsHash: []byte("Tendermint"),
					Time:           time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{
					Height: 11, DataHash: deadBeefRipEmd160Hash,
					ValidatorsHash: []byte("Tendermint"),
					Time:           time.Date(2017, 1, 2, 2, 1, 1, 1, time.UTC),
				},
				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}},
			},
			wantErr: "Headers don't match",
		},

		{
			meta: &types.BlockMeta{
				Header: &types.Header{
					Height: 11, DataHash: deadBeefRipEmd160Hash,
					ValidatorsHash: []byte("Tendermint"),
					Time:           time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC),
				},
			},
			commit: lite.Commit{
				Header: &types.Header{
					Height: 11, DataHash: deadBeefRipEmd160Hash,
					ValidatorsHash: []byte("Tendermint-x"),
					Time:           time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC),
				},
				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}},
			},
			wantErr: "Headers don't match",
		},
		// End Headers don't match test
	}

	for i, tt := range tests {
		err := proxy.ValidateBlockMeta(tt.meta, tt.commit)
		if tt.wantErr != "" {
			if err == nil {
				assert.FailNowf(t, "Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr)
			} else {
				assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i)
			}
			continue
		}

		assert.Nil(t, err, "#%d: expecting a nil error", i)
	}
}
<|endoftext|>


<commit_before>package physical

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	log "github.com/mgutz/logxi/v1"

	"github.com/armon/go-metrics"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/hashicorp/errwrap"
	cleanhttp "github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/vault/helper/awsutil"
	"github.com/hashicorp/vault/helper/consts"
)

// S3Backend is a physical backend that stores data
// within an S3 bucket.
type S3Backend struct {
	bucket     string
	client     *s3.S3
	logger     log.Logger
	permitPool *PermitPool
}

// newS3Backend constructs a S3 backend using a pre-existing
// bucket. Credentials can be provided to the backend, sourced
// from the environment, AWS credential files or by IAM role.
func newS3Backend(conf map[string]string, logger log.Logger) (Backend, error) {

	bucket := os.Getenv("AWS_S3_BUCKET")
	if bucket == "" {
		bucket = conf["bucket"]
		if bucket == "" {
			return nil, fmt.Errorf("'bucket' must be set")
		}
	}

	accessKey, ok := conf["access_key"]
	if !ok {
		accessKey = ""
	}
	secretKey, ok := conf["secret_key"]
	if !ok {
		secretKey = ""
	}
	sessionToken, ok := conf["session_token"]
	if !ok {
		sessionToken = ""
	}
	endpoint := os.Getenv("AWS_S3_ENDPOINT")
	if endpoint == "" {
		endpoint = conf["endpoint"]
	}
	region := os.Getenv("AWS_DEFAULT_REGION")
	if region == "" {
		region = conf["region"]
		if region == "" {
			region = "us-east-1"
		}
	}

	credsConfig := &awsutil.CredentialsConfig{
		AccessKey:    accessKey,
		SecretKey:    secretKey,
		SessionToken: sessionToken,
	}
	creds, err := credsConfig.GenerateCredentialChain()
	if err != nil {
		return nil, err
	}

	pooledTransport := cleanhttp.DefaultPooledTransport()
	pooledTransport.MaxIdleConnsPerHost = consts.ExpirationRestoreWorkerCount

	s3conn := s3.New(session.New(&aws.Config{
		Credentials: creds,
		HTTPClient: &http.Client{
			Transport: pooledTransport,
		},
		Endpoint: aws.String(endpoint),
		Region:   aws.String(region),
	}))

	_, err = s3conn.HeadBucket(&s3.HeadBucketInput{Bucket: &bucket})
	if err != nil {
		return nil, fmt.Errorf("unable to access bucket '%s': %v", bucket, err)
	}

	maxParStr, ok := conf["max_parallel"]
	var maxParInt int
	if ok {
		maxParInt, err = strconv.Atoi(maxParStr)
		if err != nil {
			return nil, errwrap.Wrapf("failed parsing max_parallel parameter: {{err}}", err)
		}
		if logger.IsDebug() {
			logger.Debug("s3: max_parallel set", "max_parallel", maxParInt)
		}
	}

	s := &S3Backend{
		client:     s3conn,
		bucket:     bucket,
		logger:     logger,
		permitPool: NewPermitPool(maxParInt),
	}
	return s, nil
}

// Put is used to insert or update an entry
func (s *S3Backend) Put(entry *Entry) error {
	defer metrics.MeasureSince([]string{"s3", "put"}, time.Now())

	s.permitPool.Acquire()
	defer s.permitPool.Release()

	_, err := s.client.PutObject(&s3.PutObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(entry.Key),
		Body:   bytes.NewReader(entry.Value),
	})

	if err != nil {
		return err
	}

	return nil
}

// Get is used to fetch an entry
func (s *S3Backend) Get(key string) (*Entry, error) {
	defer metrics.MeasureSince([]string{"s3", "get"}, time.Now())

	s.permitPool.Acquire()
	defer s.permitPool.Release()

	resp, err := s.client.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	})
	if awsErr, ok := err.(awserr.RequestFailure); ok {
		// Return nil on 404s, error on anything else
		if awsErr.StatusCode() == 404 {
			return nil, nil
		}
		return nil, err
	}
	if err != nil {
		return nil, err
	}
	if resp == nil {
		return nil, fmt.Errorf("got nil response from S3 but no error")
	}

	data := make([]byte, *resp.ContentLength)
	_, err = io.ReadFull(resp.Body, data)
	if err != nil {
		return nil, err
	}

	ent := &Entry{
		Key:   key,
		Value: data,
	}

	return ent, nil
}

// Delete is used to permanently delete an entry
func (s *S3Backend) Delete(key string) error {
	defer metrics.MeasureSince([]string{"s3", "delete"}, time.Now())

	s.permitPool.Acquire()
	defer s.permitPool.Release()

	_, err := s.client.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	})

	if err != nil {
		return err
	}

	return nil
}

// List is used to list all the keys under a given
// prefix, up to the next prefix.
func (s *S3Backend) List(prefix string) ([]string, error) {
	defer metrics.MeasureSince([]string{"s3", "list"}, time.Now())

	s.permitPool.Acquire()
	defer s.permitPool.Release()

	params := &s3.ListObjectsV2Input{
		Bucket: aws.String(s.bucket),
		Prefix: aws.String(prefix),
	}

	keys := []string{}

	err := s.client.ListObjectsV2Pages(params,
		func(page *s3.ListObjectsV2Output, lastPage bool) bool {
			for _, key := range page.Contents {
				key := strings.TrimPrefix(*key.Key, prefix)

				if i := strings.Index(key, "/"); i == -1 {
					// Add objects only from the current 'folder'
					keys = append(keys, key)
				} else if i != -1 {
					// Add truncated 'folder' paths
					keys = appendIfMissing(keys, key[:i+1])
				}
			}
			return true
		})

	if err != nil {
		return nil, err
	}

	sort.Strings(keys)

	return keys, nil
}

func appendIfMissing(slice []string, i string) []string {
	for _, ele := range slice {
		if ele == i {
			return slice
		}
	}
	return append(slice, i)
}
<commit_msg>Avoid panic in s3 list operation (#2785)<commit_after>package physical

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	log "github.com/mgutz/logxi/v1"

	"github.com/armon/go-metrics"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/hashicorp/errwrap"
	cleanhttp "github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/vault/helper/awsutil"
	"github.com/hashicorp/vault/helper/consts"
)

// S3Backend is a physical backend that stores data
// within an S3 bucket.
type S3Backend struct {
	bucket     string
	client     *s3.S3
	logger     log.Logger
	permitPool *PermitPool
}

// newS3Backend constructs a S3 backend using a pre-existing
// bucket. Credentials can be provided to the backend, sourced
// from the environment, AWS credential files or by IAM role.
func newS3Backend(conf map[string]string, logger log.Logger) (Backend, error) {

	bucket := os.Getenv("AWS_S3_BUCKET")
	if bucket == "" {
		bucket = conf["bucket"]
		if bucket == "" {
			return nil, fmt.Errorf("'bucket' must be set")
		}
	}

	accessKey, ok := conf["access_key"]
	if !ok {
		accessKey = ""
	}
	secretKey, ok := conf["secret_key"]
	if !ok {
		secretKey = ""
	}
	sessionToken, ok := conf["session_token"]
	if !ok {
		sessionToken = ""
	}
	endpoint := os.Getenv("AWS_S3_ENDPOINT")
	if endpoint == "" {
		endpoint = conf["endpoint"]
	}
	region := os.Getenv("AWS_DEFAULT_REGION")
	if region == "" {
		region = conf["region"]
		if region == "" {
			region = "us-east-1"
		}
	}

	credsConfig := &awsutil.CredentialsConfig{
		AccessKey:    accessKey,
		SecretKey:    secretKey,
		SessionToken: sessionToken,
	}
	creds, err := credsConfig.GenerateCredentialChain()
	if err != nil {
		return nil, err
	}

	pooledTransport := cleanhttp.DefaultPooledTransport()
	pooledTransport.MaxIdleConnsPerHost = consts.ExpirationRestoreWorkerCount

	s3conn := s3.New(session.New(&aws.Config{
		Credentials: creds,
		HTTPClient: &http.Client{
			Transport: pooledTransport,
		},
		Endpoint: aws.String(endpoint),
		Region:   aws.String(region),
	}))

	_, err = s3conn.HeadBucket(&s3.HeadBucketInput{Bucket: &bucket})
	if err != nil {
		return nil, fmt.Errorf("unable to access bucket '%s': %v", bucket, err)
	}

	maxParStr, ok := conf["max_parallel"]
	var maxParInt int
	if ok {
		maxParInt, err = strconv.Atoi(maxParStr)
		if err != nil {
			return nil, errwrap.Wrapf("failed parsing max_parallel parameter: {{err}}", err)
		}
		if logger.IsDebug() {
			logger.Debug("s3: max_parallel set", "max_parallel", maxParInt)
		}
	}

	s := &S3Backend{
		client:     s3conn,
		bucket:     bucket,
		logger:     logger,
		permitPool: NewPermitPool(maxParInt),
	}
	return s, nil
}

// Put is used to insert or update an entry
func (s *S3Backend) Put(entry *Entry) error {
	defer metrics.MeasureSince([]string{"s3", "put"}, time.Now())

	s.permitPool.Acquire()
	defer s.permitPool.Release()

	_, err := s.client.PutObject(&s3.PutObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(entry.Key),
		Body:   bytes.NewReader(entry.Value),
	})

	if err != nil {
		return err
	}

	return nil
}

// Get is used to fetch an entry
func (s *S3Backend) Get(key string) (*Entry, error) {
	defer metrics.MeasureSince([]string{"s3", "get"}, time.Now())

	s.permitPool.Acquire()
	defer s.permitPool.Release()

	resp, err := s.client.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	})
	if awsErr, ok := err.(awserr.RequestFailure); ok {
		// Return nil on 404s, error on anything else
		if awsErr.StatusCode() == 404 {
			return nil, nil
		}
		return nil, err
	}
	if err != nil {
		return nil, err
	}
	if resp == nil {
		return nil, fmt.Errorf("got nil response from S3 but no error")
	}

	data := make([]byte, *resp.ContentLength)
	_, err = io.ReadFull(resp.Body, data)
	if err != nil {
		return nil, err
	}

	ent := &Entry{
		Key:   key,
		Value: data,
	}

	return ent, nil
}

// Delete is used to permanently delete an entry
func (s *S3Backend) Delete(key string) error {
	defer metrics.MeasureSince([]string{"s3", "delete"}, time.Now())

	s.permitPool.Acquire()
	defer s.permitPool.Release()

	_, err := s.client.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	})

	if err != nil {
		return err
	}

	return nil
}

// List is used to list all the keys under a given
// prefix, up to the next prefix.
func (s *S3Backend) List(prefix string) ([]string, error) {
	defer metrics.MeasureSince([]string{"s3", "list"}, time.Now())

	s.permitPool.Acquire()
	defer s.permitPool.Release()

	params := &s3.ListObjectsV2Input{
		Bucket: aws.String(s.bucket),
		Prefix: aws.String(prefix),
	}

	keys := []string{}

	err := s.client.ListObjectsV2Pages(params,
		func(page *s3.ListObjectsV2Output, lastPage bool) bool {
			for _, key := range page.Contents {
				// Avoid panic
				if key == nil {
					continue
				}

				key := strings.TrimPrefix(*key.Key, prefix)

				if i := strings.Index(key, "/"); i == -1 {
					// Add objects only from the current 'folder'
					keys = append(keys, key)
				} else if i != -1 {
					// Add truncated 'folder' paths
					keys = appendIfMissing(keys, key[:i+1])
				}
			}
			return true
		})

	if err != nil {
		return nil, err
	}

	sort.Strings(keys)

	return keys, nil
}

func appendIfMissing(slice []string, i string) []string {
	for _, ele := range slice {
		if ele == i {
			return slice
		}
	}
	return append(slice, i)
}
<|endoftext|>


<commit_before>/*
Driver file for N. Wirth's PICL compiler

Build with:
C:\> go install picl/piclc
Run:
C:\Data\Personal\go\bin> piclc file.asm
*/

package main

import (
	"bufio"
	"flag"
	"fmt"
   "io"
	"os"
	"picl-go/PICL"
)

const Ver = "PICL compiler v1.0-alpha-2"

var (
   dump bool
)

func init() {
	flag.BoolVar(&dump, "d", false, "Dump listing output to console")
}

// Output an Intel HEX-record file
// format has 6 fields, all ASCII characters (2 chars per byte):
// : ll aaaa tt dd dd dd .... cc
// ll = 1 byte length, count only data bytes
// aaaa = 2 byte address
// tt = 1 byte type field (00 for normal, 01 for last record)
// dd = data bytes
// cc = checksum
func hexfile(f io.Writer) {
   var byteH, byteL, checksum int
   var recs, lastrec, addr int

   recs = PICL.Pc / 8
   lastrec = PICL.Pc % 8

   // Full records of 16 bytes (note each instruction is 2 bytes!)
   for i := 0; i < recs; i += 1 {
      addr = i*16
      fmt.Fprintf(f, ":10%.4X00", addr)
      checksum = 0x10 + ((addr & 0xFF00) >> 8) + (addr & 0x00FF)
      for j := 0; j < 8; j += 1 {
         byteH = (PICL.Code[i*8+j] & 0xFF00) >> 8
         byteL = PICL.Code[i*8+j] & 0x00FF
         // Remember to byte swap!
         fmt.Fprintf(f, "%.2X%.2X", byteL, byteH)
         checksum = checksum + byteH + byteL
      }
      fmt.Fprintf(f, "%.2X\n", (^(checksum & 0x00FF) + 1) & 0x00FF)
   }

   // The last, partial record
   if lastrec > 0 {
      here := recs * 8
      addr = here * 2
      fmt.Fprintf(f, ":%.2X%.4X00", lastrec*2, addr)
      checksum = lastrec*2 + ((addr & 0xFF00) >> 8) + (addr & 0x00FF)
      for i := here; i < PICL.Pc; i+= 1 {
         byteH = (PICL.Code[i] & 0xFF00) >> 8
         byteL = PICL.Code[i] & 0x00FF
         // Remember to byte swap!
         fmt.Fprintf(f, "%.2X%.2X", byteL, byteH)
         checksum = checksum + byteH + byteL
      }
      fmt.Fprintf(f, "%.2X\n", (^(checksum & 0x00FF) + 1) & 0x00FF)
   }

   // Terminating record
   fmt.Fprintf(f, ":00000001FF\n")

}

func main() {

	fmt.Printf("%s\n\n", Ver)

	// Handle args
	flag.Parse()
	// Exit on error
	if !(len(flag.Args()) > 0) {
		fmt.Printf("Usage: piclc <flags> sourcefile.pcl\n")
		flag.PrintDefaults()
		return
	}

	// Compile source file
	filename := flag.Arg(0)
	file, err := os.Open(filename)
	if err != nil {
		fmt.Println(err)
		return
	} else {
		fmt.Printf("Compiling: %s\n", filename)
		PICL.Compile(bufio.NewReader(file))
	}

   // Handle options
   if dump {
		for addr := 0; addr < PICL.Pc; addr += 1 {
			fmt.Printf("%#.3x %#.4x\n", addr, PICL.Code[addr])
		}
	}

   // Output
   // ##TODO: derive the output file name from the input file name
   //         Or allow Stdout to be used via cmd line switch
   f, err := os.Create("out.hex")
   hexfile(f)
   f.Close()

}
<commit_msg>Only output HEX if successful compile<commit_after>/*
Driver file for N. Wirth's PICL compiler

Build with:
C:\> go install picl/piclc
Run:
C:\Data\Personal\go\bin> piclc file.asm
*/

package main

import (
	"bufio"
	"flag"
	"fmt"
   "io"
	"os"
	"picl-go/PICL"
)

const Ver = "PICL compiler v1.0-alpha-2"

var (
   dump bool
)

func init() {
	flag.BoolVar(&dump, "d", false, "Dump listing output to console")
}

// Output an Intel HEX-record file
// format has 6 fields, all ASCII characters (2 chars per byte):
// : ll aaaa tt dd dd dd .... cc
// ll = 1 byte length, count only data bytes
// aaaa = 2 byte address
// tt = 1 byte type field (00 for normal, 01 for last record)
// dd = data bytes
// cc = checksum
func hexfile(f io.Writer) {
   var byteH, byteL, checksum int
   var recs, lastrec, addr int

   recs = PICL.Pc / 8
   lastrec = PICL.Pc % 8

   // Full records of 16 bytes (note each instruction is 2 bytes!)
   for i := 0; i < recs; i += 1 {
      addr = i*16
      fmt.Fprintf(f, ":10%.4X00", addr)
      checksum = 0x10 + ((addr & 0xFF00) >> 8) + (addr & 0x00FF)
      for j := 0; j < 8; j += 1 {
         byteH = (PICL.Code[i*8+j] & 0xFF00) >> 8
         byteL = PICL.Code[i*8+j] & 0x00FF
         // Remember to byte swap!
         fmt.Fprintf(f, "%.2X%.2X", byteL, byteH)
         checksum = checksum + byteH + byteL
      }
      fmt.Fprintf(f, "%.2X\n", (^(checksum & 0x00FF) + 1) & 0x00FF)
   }

   // The last, partial record
   if lastrec > 0 {
      here := recs * 8
      addr = here * 2
      fmt.Fprintf(f, ":%.2X%.4X00", lastrec*2, addr)
      checksum = lastrec*2 + ((addr & 0xFF00) >> 8) + (addr & 0x00FF)
      for i := here; i < PICL.Pc; i+= 1 {
         byteH = (PICL.Code[i] & 0xFF00) >> 8
         byteL = PICL.Code[i] & 0x00FF
         // Remember to byte swap!
         fmt.Fprintf(f, "%.2X%.2X", byteL, byteH)
         checksum = checksum + byteH + byteL
      }
      fmt.Fprintf(f, "%.2X\n", (^(checksum & 0x00FF) + 1) & 0x00FF)
   }

   // Terminating record
   fmt.Fprintf(f, ":00000001FF\n")

}

func main() {

	fmt.Printf("%s\n\n", Ver)

	// Handle args
	flag.Parse()
	// Exit on error
	if !(len(flag.Args()) > 0) {
		fmt.Printf("Usage: piclc <flags> sourcefile.pcl\n")
		flag.PrintDefaults()
		return
	}

	// Compile source file
	filename := flag.Arg(0)
	file, err := os.Open(filename)
	if err != nil {
		fmt.Println(err)
		return
	} else {
		fmt.Printf("Compiling: %s\n", filename)
		PICL.Compile(bufio.NewReader(file))
	}

   // Handle options
   if dump {
		for addr := 0; addr < PICL.Pc; addr += 1 {
			fmt.Printf("%#.3x %#.4x\n", addr, PICL.Code[addr])
		}
	}

   // Output on successful compile
   // ##TODO: derive the output file name from the input file name
   //         Or allow Stdout to be used via cmd line switch
   if !PICL.Err {
      f, _ := os.Create("out.hex")
      hexfile(f)
      f.Close()
   }

}
<|endoftext|>


<commit_before>/*
	Guides the design of `tok.Token`:
	test whether it's important to yield pointers to the values of interest,
	versus simply putting the values in an `interface{}` slot.

	(It is.)

	Though syntactically irritating to use of pointers to primitives for all tokens,
	this bench demonstrates that doing so avoids a source of allocs,
	and thus has significant performance implications.
*/
package bench

import (
	"testing"
)

// Std:  Benchmark_CopyByValue-8         30000000                43.4 ns/op
// noGC: Benchmark_CopyByValue-8         30000000                34.0 ns/op
// mem:  Benchmark_CopyByValue-8         30000000                44.4 ns/op             8 B/op          1 allocs/op
func Benchmark_CopyByValue(b *testing.B) {
	type Alias interface{}
	var slot Alias
	type StructA struct {
		field int
	}
	type StructB struct {
		field int
	}
	valA := StructA{4}
	valB := StructB{}

	for i := 0; i < b.N; i++ {
		slot = valA.field
		valB.field = slot.(int)
	}
	if valB.field != 4 {
		b.Error("final value of valB wrong")
	}
}

// Std:  Benchmark_CopyByRef-8           2000000000               0.59 ns/op
// noGC: Benchmark_CopyByRef-8           2000000000               0.59 ns/op
// mem:  Benchmark_CopyByRef-8           2000000000               0.59 ns/op            0 B/op          0 allocs/op
func Benchmark_CopyByRef(b *testing.B) {
	type Alias interface{}
	var slot Alias
	type StructA struct {
		field int
	}
	type StructB struct {
		field int
	}
	valA := StructA{4}
	valB := StructB{}

	for i := 0; i < b.N; i++ {
		slot = &(valA.field)
		valB.field = *(slot.(*int))
	}
	if valB.field != 4 {
		b.Error("final value of valB wrong")
	}
}
<commit_msg>Continuing to pursue defeat-in-detail of this little alloc issue.<commit_after>/*
	Guides the design of `tok.Token`:
	test whether it's important to yield pointers to the values of interest,
	versus simply putting the values in an `interface{}` slot.

	(It is.)

	Though syntactically irritating to use of pointers to primitives for all tokens,
	this bench demonstrates that doing so avoids a source of allocs,
	and thus has significant performance implications.
*/
package bench

import (
	"testing"
)

// Std:  Benchmark_CopyByValue-8         30000000                43.4 ns/op
// noGC: Benchmark_CopyByValue-8         30000000                34.0 ns/op
// mem:  Benchmark_CopyByValue-8         30000000                44.4 ns/op             8 B/op          1 allocs/op
func Benchmark_CopyByValue(b *testing.B) {
	type Alias interface{}
	var slot Alias
	type StructA struct {
		field int
	}
	type StructB struct {
		field int
	}
	valA := StructA{4}
	valB := StructB{}

	for i := 0; i < b.N; i++ {
		slot = valA.field
		valB.field = slot.(int)
	}
	if valB.field != 4 {
		b.Error("final value of valB wrong")
	}
}

// Std:  Benchmark_CopyByRef-8           2000000000               0.59 ns/op
// noGC: Benchmark_CopyByRef-8           2000000000               0.59 ns/op
// mem:  Benchmark_CopyByRef-8           2000000000               0.59 ns/op            0 B/op          0 allocs/op
func Benchmark_CopyByRef(b *testing.B) {
	type Alias interface{}
	var slot Alias
	type StructA struct {
		field int
	}
	type StructB struct {
		field int
	}
	valA := StructA{4}
	valB := StructB{}

	for i := 0; i < b.N; i++ {
		slot = &(valA.field)
		valB.field = *(slot.(*int))
	}
	if valB.field != 4 {
		b.Error("final value of valB wrong")
	}
}

// Sanity check: strings are not noticably different:
//
//	Benchmark_CopyByValue-8                 30000000                45.5 ns/op
//	Benchmark_CopyByRef-8                   2000000000               0.59 ns/op
//	Benchmark_CopyByValue_String-8          20000000                72.3 ns/op
//	Benchmark_CopyByRef_String-8            2000000000               0.60 ns/op
func Benchmark_CopyByValue_String(b *testing.B) {
	type Alias interface{}
	var slot Alias
	type StructA struct {
		field string
	}
	type StructB struct {
		field string
	}
	valA := StructA{"alksjdlkjweoihgowihehgioijerg"}
	valB := StructB{}

	for i := 0; i < b.N; i++ {
		slot = valA.field
		valB.field = slot.(string)
	}
	if valB.field != valA.field {
		b.Error("final value of valB wrong")
	}
}

func Benchmark_CopyByRef_String(b *testing.B) {
	type Alias interface{}
	var slot Alias
	type StructA struct {
		field string
	}
	type StructB struct {
		field string
	}
	valA := StructA{"alksjdlkjweoihgowihehgioijerg"}
	valB := StructB{}

	for i := 0; i < b.N; i++ {
		slot = &(valA.field)
		valB.field = *(slot.(*string))
	}
	if valB.field != valA.field {
		b.Error("final value of valB wrong")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2018 Laurent Moussault. All rights reserved.
// Licensed under a simplified BSD license (see LICENSE file).

package pixel

import (
	"image"
	"image/color"
	"strings"
	"unsafe"

	"github.com/drakmaniso/glam/internal"
	"github.com/drakmaniso/glam/x/atlas"
	"github.com/drakmaniso/glam/x/gl"
)

//------------------------------------------------------------------------------

func init() {
	internal.PixelSetup = setup
	internal.PixelCleanup = cleanup
}

func setup() error {
	// Create the canvases

	for i := range canvases {
		Canvas(i).createBuffer()
	}

	// Create the paint pipeline

	pipeline = gl.NewPipeline(
		gl.VertexShader(strings.NewReader(vertexShader)),
		gl.FragmentShader(strings.NewReader(fragmentShader)),
		gl.CullFace(false, false),
		gl.Topology(gl.TriangleStrip),
		gl.DepthTest(true),
		gl.DepthWrite(true),
		gl.DepthComparison(gl.GreaterOrEqual),
	)

	screenUBO = gl.NewUniformBuffer(&screenUniforms, gl.DynamicStorage|gl.MapWrite)

	commandsICBO = gl.NewIndirectBuffer(
		uintptr(maxCommandCount)*unsafe.Sizeof(gl.DrawIndirectCommand{}),
		gl.DynamicStorage,
	)

	parametersTBO = gl.NewBufferTexture(
		uintptr(maxCommandCount*maxParamCount),
		gl.R16I,
		gl.DynamicStorage,
	)

	// Create the display pipeline

	blitPipeline = gl.NewPipeline(
		gl.VertexShader(strings.NewReader(blitVertexShader)),
		gl.FragmentShader(strings.NewReader(blitFragmentShader)),
		gl.Topology(gl.TriangleStrip),
		gl.DepthTest(false),
		gl.DepthWrite(false),
	)

	blitUBO = gl.NewUniformBuffer(&blitUniforms, gl.DynamicStorage|gl.MapWrite)

	// Create texture atlases for pictures and fonts

	pictAtlas = atlas.New(1024, 1024)
	fntAtlas = atlas.New(128, 128)

	err := loadAssets()
	if err != nil {
		return err
	}
	//TODO: handle the case when there is no pictures

	// Mappings Buffer
	pictureMapTBO = gl.NewBufferTexture(pictureMap, gl.R16I, gl.StaticStorage)
	if len(glyphMap) > 0 {
		glyphMapTBO = gl.NewBufferTexture(glyphMap, gl.R16I, gl.StaticStorage)
	}

	// Create the pictures texture array
	w, h := pictAtlas.BinSize()
	picturesTA = gl.NewTextureArray2D(1, gl.R8UI, int32(w), int32(h), int32(pictAtlas.BinCount()))
	for i := int16(0); i < pictAtlas.BinCount(); i++ {
		m := image.NewPaletted(image.Rectangle{
			Min: image.Point{0, 0},
			Max: image.Point{int(w), int(h)},
		},
			color.Palette{},
		)

		err := pictAtlas.Paint(i, m)
		if err != nil {
			return err
		}

		picturesTA.SubImage(0, 0, 0, int32(i), m)
	}

	// Create the font texture array
	w, h = fntAtlas.BinSize()
	glyphsTA = gl.NewTextureArray2D(1, gl.R8UI, int32(w), int32(h), int32(fntAtlas.BinCount()))
	for i := int16(0); i < fntAtlas.BinCount(); i++ {
		m := image.NewPaletted(image.Rectangle{
			Min: image.Point{0, 0},
			Max: image.Point{int(w), int(h)},
		},
			color.Palette{},
		)

		err := fntAtlas.Paint(i, m)
		if err != nil {
			return err
		}

		glyphsTA.SubImage(0, 0, 0, int32(i), m)

		// of, err := os.Create("testdata/fnt" + string('0'+i) + ".png")
		// if err != nil {
		// 	panic(err)
		// }
		// m.Palette = color.Palette{
		// 	color.RGBA{0, 0, 0, 255},
		// 	color.RGBA{255, 255, 255, 255},
		// 	color.RGBA{255, 0, 255, 255},
		// }
		// err = png.Encode(of, m)
		// if err != nil {
		// 	panic(err)
		// }
		// of.Close()
	}

	return gl.Err()
}

func cleanup() error {
	// Canvases
	for i := range canvases {
		s := &canvases[i]
		s.texture.Delete()
		s.depth.Delete()
		s.buffer.Delete()
	}

	// Display pipeline
	pipeline.Delete()
	pipeline = nil
	screenUBO.Delete()
	commandsICBO.Delete()
	parametersTBO.Delete()

	// Pictures
	pictAtlas = nil
	pictFiles = nil
	pictureMapTBO.Delete()
	picturesTA.Delete()

	// Fonts
	fntAtlas = nil
	fntFiles = nil
	glyphMapTBO.Delete()
	glyphsTA.Delete()

	return gl.Err()
}

//------------------------------------------------------------------------------

func init() {
	internal.PixelResize = func() {
		for i := range canvases {
			Canvas(i).autoresize()
		}
	}
}

//------------------------------------------------------------------------------
<commit_msg>Release pixel setup slices<commit_after>// Copyright (c) 2013-2018 Laurent Moussault. All rights reserved.
// Licensed under a simplified BSD license (see LICENSE file).

package pixel

import (
	"image"
	"image/color"
	"strings"
	"unsafe"

	"github.com/drakmaniso/glam/internal"
	"github.com/drakmaniso/glam/x/atlas"
	"github.com/drakmaniso/glam/x/gl"
)

//------------------------------------------------------------------------------

func init() {
	internal.PixelSetup = setup
	internal.PixelCleanup = cleanup
}

func setup() error {
	// Create the canvases

	for i := range canvases {
		Canvas(i).createBuffer()
	}

	// Create the paint pipeline

	pipeline = gl.NewPipeline(
		gl.VertexShader(strings.NewReader(vertexShader)),
		gl.FragmentShader(strings.NewReader(fragmentShader)),
		gl.CullFace(false, false),
		gl.Topology(gl.TriangleStrip),
		gl.DepthTest(true),
		gl.DepthWrite(true),
		gl.DepthComparison(gl.GreaterOrEqual),
	)

	screenUBO = gl.NewUniformBuffer(&screenUniforms, gl.DynamicStorage|gl.MapWrite)

	commandsICBO = gl.NewIndirectBuffer(
		uintptr(maxCommandCount)*unsafe.Sizeof(gl.DrawIndirectCommand{}),
		gl.DynamicStorage,
	)

	parametersTBO = gl.NewBufferTexture(
		uintptr(maxCommandCount*maxParamCount),
		gl.R16I,
		gl.DynamicStorage,
	)

	// Create the display pipeline

	blitPipeline = gl.NewPipeline(
		gl.VertexShader(strings.NewReader(blitVertexShader)),
		gl.FragmentShader(strings.NewReader(blitFragmentShader)),
		gl.Topology(gl.TriangleStrip),
		gl.DepthTest(false),
		gl.DepthWrite(false),
	)

	blitUBO = gl.NewUniformBuffer(&blitUniforms, gl.DynamicStorage|gl.MapWrite)

	// Create texture atlases for pictures and fonts

	pictAtlas = atlas.New(1024, 1024)
	fntAtlas = atlas.New(128, 128)

	err := loadAssets()
	if err != nil {
		return err
	}
	//TODO: handle the case when there is no pictures

	// Mappings Buffer
	pictureMapTBO = gl.NewBufferTexture(pictureMap, gl.R16I, gl.StaticStorage)
	if len(glyphMap) > 0 {
		glyphMapTBO = gl.NewBufferTexture(glyphMap, gl.R16I, gl.StaticStorage)
	}

	// Create the pictures texture array
	w, h := pictAtlas.BinSize()
	picturesTA = gl.NewTextureArray2D(1, gl.R8UI, int32(w), int32(h), int32(pictAtlas.BinCount()))
	for i := int16(0); i < pictAtlas.BinCount(); i++ {
		m := image.NewPaletted(image.Rectangle{
			Min: image.Point{0, 0},
			Max: image.Point{int(w), int(h)},
		},
			color.Palette{},
		)

		err := pictAtlas.Paint(i, m)
		if err != nil {
			return err
		}

		picturesTA.SubImage(0, 0, 0, int32(i), m)
	}

	pictFiles = pictFiles[:0]

	// Create the font texture array
	w, h = fntAtlas.BinSize()
	glyphsTA = gl.NewTextureArray2D(1, gl.R8UI, int32(w), int32(h), int32(fntAtlas.BinCount()))
	for i := int16(0); i < fntAtlas.BinCount(); i++ {
		m := image.NewPaletted(image.Rectangle{
			Min: image.Point{0, 0},
			Max: image.Point{int(w), int(h)},
		},
			color.Palette{},
		)

		err := fntAtlas.Paint(i, m)
		if err != nil {
			return err
		}

		glyphsTA.SubImage(0, 0, 0, int32(i), m)

		// of, err := os.Create("testdata/fnt" + string('0'+i) + ".png")
		// if err != nil {
		// 	panic(err)
		// }
		// m.Palette = color.Palette{
		// 	color.RGBA{0, 0, 0, 255},
		// 	color.RGBA{255, 255, 255, 255},
		// 	color.RGBA{255, 0, 255, 255},
		// }
		// err = png.Encode(of, m)
		// if err != nil {
		// 	panic(err)
		// }
		// of.Close()
	}

	fntFiles = fntFiles[:0]

	return gl.Err()
}

func cleanup() error {
	// Canvases
	for i := range canvases {
		s := &canvases[i]
		s.texture.Delete()
		s.depth.Delete()
		s.buffer.Delete()
	}

	// Display pipeline
	pipeline.Delete()
	pipeline = nil
	screenUBO.Delete()
	commandsICBO.Delete()
	parametersTBO.Delete()

	// Pictures
	pictAtlas = nil
	pictureMapTBO.Delete()
	picturesTA.Delete()

	// Fonts
	fntAtlas = nil
	glyphMapTBO.Delete()
	glyphsTA.Delete()

	return gl.Err()
}

//------------------------------------------------------------------------------

func init() {
	internal.PixelResize = func() {
		for i := range canvases {
			Canvas(i).autoresize()
		}
	}
}

//------------------------------------------------------------------------------
<|endoftext|>