<commit_before>package main

import (
	"errors"
	"fmt"
	"net"
	"syscall"
	//"golang.org/x/net/ipv4"
)

type IP_Conn struct {
	fd        int
    sockAddr  syscall.Sockaddr
	version   uint8
	dst, src  string
	headerLen uint16
	//len uint16
	//id uint16
	ttl      uint8
	protocol uint8
	//checksum int
	identifier uint16
}

func NewIP_Conn(dst string) (*IP_Conn, error) {
	//pc, err := net.ListenIP("ip4:17", &net.IPAddr{IP: net.ParseIP(dst)})
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW)
	if err != nil {
		fmt.Println("Failed to ListenIP")
		return nil, err
	}

    dstIPAddr, err := net.ResolveIPAddr("ip", dst)
    if err != nil {
        //fmt.Println(err)
        return nil, err
    }
    fmt.Println("Full Address: ", dstIPAddr)

    addr := &syscall.SockaddrInet4{
        Port: 20000,
        //Addr: [4]byte{127, 0, 0, 1},
        Addr: [4]byte{
            dstIPAddr.IP[12],
            dstIPAddr.IP[13],
            dstIPAddr.IP[14],
            dstIPAddr.IP[15],
        },
    }

    err = syscall.Connect(fd, addr)
    if err != nil {
        return nil, errors.New("Failed to connect.")
    }
    err = syscall.Bind(fd, addr)
    if err != nil {
        return nil, errors.New("Failed to bind to address.")
    }

	return &IP_Conn{
		fd:         fd,
        sockAddr:   addr,
		version:    4,
		headerLen:  20,
		dst:        dst,
		src:        "127.0.0.1",
		ttl:        8,
		protocol:   17,
		identifier: 20000,
	}, nil
}

func calcChecksum(head []byte, excludeChecksum bool) uint16 {
	totalSum := uint64(0)
	for ind, elem := range head {
		if (ind == 10 || ind == 11) && excludeChecksum { // Ignore the checksum in some situations
			continue
		}

		if ind%2 == 0 {
			totalSum += (uint64(elem) << 8)
		} else {
			totalSum += uint64(elem)
		}
	}
	fmt.Println("Checksum total: ", totalSum)

	for prefix := (totalSum >> 16); prefix != 0; prefix = (totalSum >> 16) {
		//        fmt.Println(prefix)
		//        fmt.Println(totalSum)
		//        fmt.Println(totalSum & 0xffff)
		totalSum = uint64(totalSum&0xffff) + prefix
	}
	fmt.Println("Checksum after carry: ", totalSum)

	carried := uint16(totalSum)

	return ^carried
}

func slicePacket(b []byte) (hrd, payload []byte) {
	hdrLen := int(b[0]&0x0f) * 4
	fmt.Println("HdrLen: ", hdrLen)
	return b[:hdrLen], b[hdrLen:]
}

func (ipc *IP_Conn) ReadFrom(b []byte) (payload []byte, e error) {
	//n, _, err := syscall.Recvfrom(ipc.fd, b, 0) //_ is src address
    n, _, _, _, err := syscall.Recvmsg(ipc.fd, b, make([]byte, 30000), 0)
	b = b[:n]
	fmt.Println("Read Length: ", n)
	fmt.Println("Full Read Data (after trim): ", b)
	hdr, p := slicePacket(b)

	// verify checksum
	if calcChecksum(hdr, false) != 0 {
		fmt.Println("Header checksum verification failed. Packet dropped.")
		fmt.Println("Wrong header: ", hdr)
		fmt.Println("Payload (dropped): ", p)
		return nil, errors.New("Header checksum incorrect, packet dropped")
	}

	return p, err
}

func (ipc *IP_Conn) WriteTo(p []byte) error {
	totalLen := uint16(ipc.headerLen) + uint16(len(p))
	fmt.Println("Total Len: ", totalLen)
	packet := make([]byte, ipc.headerLen)
	packet[0] = (byte)((ipc.version << 4) + (uint8)(ipc.headerLen/4)) // Version, IHL
	packet[1] = 0
	packet[2] = (byte)(totalLen >> 8) // Total Len
	packet[3] = (byte)(totalLen)

	id := ipc.identifier
	packet[4] = byte(id >> 8) // Identification
	packet[5] = byte(id)
	ipc.identifier++

	packet[6] = byte(1 << 6)         // Flags: Don't fragment
	packet[7] = 0                    // Fragment Offset
	packet[8] = (byte)(ipc.ttl)      // Time to Live
	packet[9] = (byte)(ipc.protocol) // Protocol

	// Src and Dst IPs
	srcIP := net.ParseIP(ipc.src)
	fmt.Println(srcIP)
	//    fmt.Println(srcIP[12])
	//    fmt.Println(srcIP[13])
	//    fmt.Println(srcIP[14])
	//    fmt.Println(srcIP[15])
	dstIP := net.ParseIP(ipc.dst)
	fmt.Println(dstIP)
	packet[12] = srcIP[12]
	packet[13] = srcIP[13]
	packet[14] = srcIP[14]
	packet[15] = srcIP[15]
	packet[16] = dstIP[12]
	packet[17] = dstIP[13]
	packet[18] = dstIP[14]
	packet[19] = dstIP[15]

	// IPv4 header test (before checksum)
	fmt.Println("Packet before checksum: ", packet)

	// Checksum
	checksum := calcChecksum(packet[:20], true)
	packet[10] = byte(checksum >> 8)
	packet[11] = byte(checksum)

	// Payload
	packet = append(packet, p...)
	fmt.Println("Full Packet:  ", packet)

	//ipc.pc.WriteMsgIP(packet, nil, dstIPAddr)

	return syscall.Sendto(ipc.fd, packet, 0, ipc.sockAddr)
}

func (ipc *IP_Conn) Close() error {
	return syscall.Close(ipc.fd)
}

/* h := &ipv4.Header{
	Version:  ipv4.Version,      // protocol version
	Len:      20,                // header length
	TOS:      0,                 // type-of-service (0 is everything normal)
	TotalLen: len(x) + 20,       // packet total length (octets)
	ID:       0,                 // identification
	Flags:    ipv4.DontFragment, // flags
	FragOff:  0,                 // fragment offset
	TTL:      8,                 // time-to-live (maximum lifespan in seconds)
	Protocol: 17,                // next protocol (17 is UDP)
	Checksum: 0,                 // checksum (apparently autocomputed)
	//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
	Dst: net.ParseIP(c.manager.ipAddress), // destination address
	//Options                         // options, extension headers
}
*/
<commit_msg>Removed connect call<commit_after>package main

import (
	"errors"
	"fmt"
	"net"
	"syscall"
	//"golang.org/x/net/ipv4"
)

type IP_Conn struct {
	fd        int
    sockAddr  syscall.Sockaddr
	version   uint8
	dst, src  string
	headerLen uint16
	//len uint16
	//id uint16
	ttl      uint8
	protocol uint8
	//checksum int
	identifier uint16
}

func NewIP_Conn(dst string) (*IP_Conn, error) {
	//pc, err := net.ListenIP("ip4:17", &net.IPAddr{IP: net.ParseIP(dst)})
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW)
	if err != nil {
		fmt.Println("Failed to ListenIP")
		return nil, err
	}

    dstIPAddr, err := net.ResolveIPAddr("ip", dst)
    if err != nil {
        //fmt.Println(err)
        return nil, err
    }
    fmt.Println("Full Address: ", dstIPAddr)

    addr := &syscall.SockaddrInet4{
        Port: 20000,
        //Addr: [4]byte{127, 0, 0, 1},
        Addr: [4]byte{
            dstIPAddr.IP[12],
            dstIPAddr.IP[13],
            dstIPAddr.IP[14],
            dstIPAddr.IP[15],
        },
    }

    /*err = syscall.Connect(fd, addr)
    if err != nil {
        return nil, errors.New("Failed to connect.")
    }*/
    err = syscall.Bind(fd, addr)
    if err != nil {
        return nil, errors.New("Failed to bind to address.")
    }

	return &IP_Conn{
		fd:         fd,
        sockAddr:   addr,
		version:    4,
		headerLen:  20,
		dst:        dst,
		src:        "127.0.0.1",
		ttl:        8,
		protocol:   17,
		identifier: 20000,
	}, nil
}

func calcChecksum(head []byte, excludeChecksum bool) uint16 {
	totalSum := uint64(0)
	for ind, elem := range head {
		if (ind == 10 || ind == 11) && excludeChecksum { // Ignore the checksum in some situations
			continue
		}

		if ind%2 == 0 {
			totalSum += (uint64(elem) << 8)
		} else {
			totalSum += uint64(elem)
		}
	}
	fmt.Println("Checksum total: ", totalSum)

	for prefix := (totalSum >> 16); prefix != 0; prefix = (totalSum >> 16) {
		//        fmt.Println(prefix)
		//        fmt.Println(totalSum)
		//        fmt.Println(totalSum & 0xffff)
		totalSum = uint64(totalSum&0xffff) + prefix
	}
	fmt.Println("Checksum after carry: ", totalSum)

	carried := uint16(totalSum)

	return ^carried
}

func slicePacket(b []byte) (hrd, payload []byte) {
	hdrLen := int(b[0]&0x0f) * 4
	fmt.Println("HdrLen: ", hdrLen)
	return b[:hdrLen], b[hdrLen:]
}

func (ipc *IP_Conn) ReadFrom(b []byte) (payload []byte, e error) {
	//n, _, err := syscall.Recvfrom(ipc.fd, b, 0) //_ is src address
    n, _, _, _, err := syscall.Recvmsg(ipc.fd, b, make([]byte, 30000), 0)
	b = b[:n]
	fmt.Println("Read Length: ", n)
	fmt.Println("Full Read Data (after trim): ", b)
	hdr, p := slicePacket(b)

	// verify checksum
	if calcChecksum(hdr, false) != 0 {
		fmt.Println("Header checksum verification failed. Packet dropped.")
		fmt.Println("Wrong header: ", hdr)
		fmt.Println("Payload (dropped): ", p)
		return nil, errors.New("Header checksum incorrect, packet dropped")
	}

	return p, err
}

func (ipc *IP_Conn) WriteTo(p []byte) error {
	totalLen := uint16(ipc.headerLen) + uint16(len(p))
	fmt.Println("Total Len: ", totalLen)
	packet := make([]byte, ipc.headerLen)
	packet[0] = (byte)((ipc.version << 4) + (uint8)(ipc.headerLen/4)) // Version, IHL
	packet[1] = 0
	packet[2] = (byte)(totalLen >> 8) // Total Len
	packet[3] = (byte)(totalLen)

	id := ipc.identifier
	packet[4] = byte(id >> 8) // Identification
	packet[5] = byte(id)
	ipc.identifier++

	packet[6] = byte(1 << 6)         // Flags: Don't fragment
	packet[7] = 0                    // Fragment Offset
	packet[8] = (byte)(ipc.ttl)      // Time to Live
	packet[9] = (byte)(ipc.protocol) // Protocol

	// Src and Dst IPs
	srcIP := net.ParseIP(ipc.src)
	fmt.Println(srcIP)
	//    fmt.Println(srcIP[12])
	//    fmt.Println(srcIP[13])
	//    fmt.Println(srcIP[14])
	//    fmt.Println(srcIP[15])
	dstIP := net.ParseIP(ipc.dst)
	fmt.Println(dstIP)
	packet[12] = srcIP[12]
	packet[13] = srcIP[13]
	packet[14] = srcIP[14]
	packet[15] = srcIP[15]
	packet[16] = dstIP[12]
	packet[17] = dstIP[13]
	packet[18] = dstIP[14]
	packet[19] = dstIP[15]

	// IPv4 header test (before checksum)
	fmt.Println("Packet before checksum: ", packet)

	// Checksum
	checksum := calcChecksum(packet[:20], true)
	packet[10] = byte(checksum >> 8)
	packet[11] = byte(checksum)

	// Payload
	packet = append(packet, p...)
	fmt.Println("Full Packet:  ", packet)

	//ipc.pc.WriteMsgIP(packet, nil, dstIPAddr)

	return syscall.Sendto(ipc.fd, packet, 0, ipc.sockAddr)
}

func (ipc *IP_Conn) Close() error {
	return syscall.Close(ipc.fd)
}

/* h := &ipv4.Header{
	Version:  ipv4.Version,      // protocol version
	Len:      20,                // header length
	TOS:      0,                 // type-of-service (0 is everything normal)
	TotalLen: len(x) + 20,       // packet total length (octets)
	ID:       0,                 // identification
	Flags:    ipv4.DontFragment, // flags
	FragOff:  0,                 // fragment offset
	TTL:      8,                 // time-to-live (maximum lifespan in seconds)
	Protocol: 17,                // next protocol (17 is UDP)
	Checksum: 0,                 // checksum (apparently autocomputed)
	//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
	Dst: net.ParseIP(c.manager.ipAddress), // destination address
	//Options                         // options, extension headers
}
*/
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file provides Go implementations of elementary multi-precision
// arithmetic operations on word vectors. Needed for platforms without
// assembly implementations of these routines.

package big

import "math/bits"

// A Word represents a single digit of a multi-precision unsigned integer.
type Word uint

const (
	_S = _W / 8 // word size in bytes

	_W = bits.UintSize // word size in bits
	_B = 1 << _W       // digit base
	_M = _B - 1        // digit mask

	_W2 = _W / 2   // half word size in bits
	_B2 = 1 << _W2 // half digit base
	_M2 = _B2 - 1  // half digit mask
)

// ----------------------------------------------------------------------------
// Elementary operations on words
//
// These operations are used by the vector operations below.

// z1<<_W + z0 = x+y+c, with c == 0 or 1
func addWW_g(x, y, c Word) (z1, z0 Word) {
	yc := y + c
	z0 = x + yc
	if z0 < x || yc < y {
		z1 = 1
	}
	return
}

// z1<<_W + z0 = x-y-c, with c == 0 or 1
func subWW_g(x, y, c Word) (z1, z0 Word) {
	yc := y + c
	z0 = x - yc
	if z0 > x || yc < y {
		z1 = 1
	}
	return
}

// z1<<_W + z0 = x*y
// Adapted from Warren, Hacker's Delight, p. 132.
func mulWW_g(x, y Word) (z1, z0 Word) {
	x0 := x & _M2
	x1 := x >> _W2
	y0 := y & _M2
	y1 := y >> _W2
	w0 := x0 * y0
	t := x1*y0 + w0>>_W2
	w1 := t & _M2
	w2 := t >> _W2
	w1 += x0 * y1
	z1 = x1*y1 + w2 + w1>>_W2
	z0 = x * y
	return
}

// z1<<_W + z0 = x*y + c
func mulAddWWW_g(x, y, c Word) (z1, z0 Word) {
	z1, zz0 := mulWW_g(x, y)
	if z0 = zz0 + c; z0 < zz0 {
		z1++
	}
	return
}

// nlz returns the number of leading zeros in x.
// Wraps bits.LeadingZeros call for convenience.
func nlz(x Word) uint {
	return uint(bits.LeadingZeros(uint(x)))
}

// q = (u1<<_W + u0 - r)/y
// Adapted from Warren, Hacker's Delight, p. 152.
func divWW_g(u1, u0, v Word) (q, r Word) {
	if u1 >= v {
		return 1<<_W - 1, 1<<_W - 1
	}

	s := nlz(v)
	v <<= s

	vn1 := v >> _W2
	vn0 := v & _M2
	un32 := u1<<s | u0>>(_W-s)
	un10 := u0 << s
	un1 := un10 >> _W2
	un0 := un10 & _M2
	q1 := un32 / vn1
	rhat := un32 - q1*vn1

	for q1 >= _B2 || q1*vn0 > _B2*rhat+un1 {
		q1--
		rhat += vn1
		if rhat >= _B2 {
			break
		}
	}

	un21 := un32*_B2 + un1 - q1*v
	q0 := un21 / vn1
	rhat = un21 - q0*vn1

	for q0 >= _B2 || q0*vn0 > _B2*rhat+un0 {
		q0--
		rhat += vn1
		if rhat >= _B2 {
			break
		}
	}

	return q1*_B2 + q0, (un21*_B2 + un0 - q0*v) >> s
}

// Keep for performance debugging.
// Using addWW_g is likely slower.
const use_addWW_g = false

// The resulting carry c is either 0 or 1.
func addVV_g(z, x, y []Word) (c Word) {
	if use_addWW_g {
		for i := range z {
			c, z[i] = addWW_g(x[i], y[i], c)
		}
		return
	}

	for i, xi := range x[:len(z)] {
		yi := y[i]
		zi := xi + yi + c
		z[i] = zi
		// see "Hacker's Delight", section 2-12 (overflow detection)
		c = (xi&yi | (xi|yi)&^zi) >> (_W - 1)
	}
	return
}

// The resulting carry c is either 0 or 1.
func subVV_g(z, x, y []Word) (c Word) {
	if use_addWW_g {
		for i := range z {
			c, z[i] = subWW_g(x[i], y[i], c)
		}
		return
	}

	for i, xi := range x[:len(z)] {
		yi := y[i]
		zi := xi - yi - c
		z[i] = zi
		// see "Hacker's Delight", section 2-12 (overflow detection)
		c = (yi&^xi | (yi|^xi)&zi) >> (_W - 1)
	}
	return
}

// The resulting carry c is either 0 or 1.
func addVW_g(z, x []Word, y Word) (c Word) {
	if use_addWW_g {
		c = y
		for i := range z {
			c, z[i] = addWW_g(x[i], c, 0)
		}
		return
	}

	c = y
	for i, xi := range x[:len(z)] {
		zi := xi + c
		z[i] = zi
		c = xi &^ zi >> (_W - 1)
	}
	return
}

func subVW_g(z, x []Word, y Word) (c Word) {
	if use_addWW_g {
		c = y
		for i := range z {
			c, z[i] = subWW_g(x[i], c, 0)
		}
		return
	}

	c = y
	for i, xi := range x[:len(z)] {
		zi := xi - c
		z[i] = zi
		c = (zi &^ xi) >> (_W - 1)
	}
	return
}

func shlVU_g(z, x []Word, s uint) (c Word) {
	if n := len(z); n > 0 {
		ŝ := _W - s
		w1 := x[n-1]
		c = w1 >> ŝ
		for i := n - 1; i > 0; i-- {
			w := w1
			w1 = x[i-1]
			z[i] = w<<s | w1>>ŝ
		}
		z[0] = w1 << s
	}
	return
}

func shrVU_g(z, x []Word, s uint) (c Word) {
	if n := len(z); n > 0 {
		ŝ := _W - s
		w1 := x[0]
		c = w1 << ŝ
		for i := 0; i < n-1; i++ {
			w := w1
			w1 = x[i+1]
			z[i] = w>>s | w1<<ŝ
		}
		z[n-1] = w1 >> s
	}
	return
}

func mulAddVWW_g(z, x []Word, y, r Word) (c Word) {
	c = r
	for i := range z {
		c, z[i] = mulAddWWW_g(x[i], y, c)
	}
	return
}

// TODO(gri) Remove use of addWW_g here and then we can remove addWW_g and subWW_g.
func addMulVVW_g(z, x []Word, y Word) (c Word) {
	for i := range z {
		z1, z0 := mulAddWWW_g(x[i], y, z[i])
		c, z[i] = addWW_g(z0, c, 0)
		c += z1
	}
	return
}

func divWVW_g(z []Word, xn Word, x []Word, y Word) (r Word) {
	r = xn
	for i := len(z) - 1; i >= 0; i-- {
		z[i], r = divWW_g(r, x[i], y)
	}
	return
}
<commit_msg>math/big: fix a formula used as documentation<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file provides Go implementations of elementary multi-precision
// arithmetic operations on word vectors. Needed for platforms without
// assembly implementations of these routines.

package big

import "math/bits"

// A Word represents a single digit of a multi-precision unsigned integer.
type Word uint

const (
	_S = _W / 8 // word size in bytes

	_W = bits.UintSize // word size in bits
	_B = 1 << _W       // digit base
	_M = _B - 1        // digit mask

	_W2 = _W / 2   // half word size in bits
	_B2 = 1 << _W2 // half digit base
	_M2 = _B2 - 1  // half digit mask
)

// ----------------------------------------------------------------------------
// Elementary operations on words
//
// These operations are used by the vector operations below.

// z1<<_W + z0 = x+y+c, with c == 0 or 1
func addWW_g(x, y, c Word) (z1, z0 Word) {
	yc := y + c
	z0 = x + yc
	if z0 < x || yc < y {
		z1 = 1
	}
	return
}

// z1<<_W + z0 = x-y-c, with c == 0 or 1
func subWW_g(x, y, c Word) (z1, z0 Word) {
	yc := y + c
	z0 = x - yc
	if z0 > x || yc < y {
		z1 = 1
	}
	return
}

// z1<<_W + z0 = x*y
// Adapted from Warren, Hacker's Delight, p. 132.
func mulWW_g(x, y Word) (z1, z0 Word) {
	x0 := x & _M2
	x1 := x >> _W2
	y0 := y & _M2
	y1 := y >> _W2
	w0 := x0 * y0
	t := x1*y0 + w0>>_W2
	w1 := t & _M2
	w2 := t >> _W2
	w1 += x0 * y1
	z1 = x1*y1 + w2 + w1>>_W2
	z0 = x * y
	return
}

// z1<<_W + z0 = x*y + c
func mulAddWWW_g(x, y, c Word) (z1, z0 Word) {
	z1, zz0 := mulWW_g(x, y)
	if z0 = zz0 + c; z0 < zz0 {
		z1++
	}
	return
}

// nlz returns the number of leading zeros in x.
// Wraps bits.LeadingZeros call for convenience.
func nlz(x Word) uint {
	return uint(bits.LeadingZeros(uint(x)))
}

// q = (u1<<_W + u0 - r)/v
// Adapted from Warren, Hacker's Delight, p. 152.
func divWW_g(u1, u0, v Word) (q, r Word) {
	if u1 >= v {
		return 1<<_W - 1, 1<<_W - 1
	}

	s := nlz(v)
	v <<= s

	vn1 := v >> _W2
	vn0 := v & _M2
	un32 := u1<<s | u0>>(_W-s)
	un10 := u0 << s
	un1 := un10 >> _W2
	un0 := un10 & _M2
	q1 := un32 / vn1
	rhat := un32 - q1*vn1

	for q1 >= _B2 || q1*vn0 > _B2*rhat+un1 {
		q1--
		rhat += vn1
		if rhat >= _B2 {
			break
		}
	}

	un21 := un32*_B2 + un1 - q1*v
	q0 := un21 / vn1
	rhat = un21 - q0*vn1

	for q0 >= _B2 || q0*vn0 > _B2*rhat+un0 {
		q0--
		rhat += vn1
		if rhat >= _B2 {
			break
		}
	}

	return q1*_B2 + q0, (un21*_B2 + un0 - q0*v) >> s
}

// Keep for performance debugging.
// Using addWW_g is likely slower.
const use_addWW_g = false

// The resulting carry c is either 0 or 1.
func addVV_g(z, x, y []Word) (c Word) {
	if use_addWW_g {
		for i := range z {
			c, z[i] = addWW_g(x[i], y[i], c)
		}
		return
	}

	for i, xi := range x[:len(z)] {
		yi := y[i]
		zi := xi + yi + c
		z[i] = zi
		// see "Hacker's Delight", section 2-12 (overflow detection)
		c = (xi&yi | (xi|yi)&^zi) >> (_W - 1)
	}
	return
}

// The resulting carry c is either 0 or 1.
func subVV_g(z, x, y []Word) (c Word) {
	if use_addWW_g {
		for i := range z {
			c, z[i] = subWW_g(x[i], y[i], c)
		}
		return
	}

	for i, xi := range x[:len(z)] {
		yi := y[i]
		zi := xi - yi - c
		z[i] = zi
		// see "Hacker's Delight", section 2-12 (overflow detection)
		c = (yi&^xi | (yi|^xi)&zi) >> (_W - 1)
	}
	return
}

// The resulting carry c is either 0 or 1.
func addVW_g(z, x []Word, y Word) (c Word) {
	if use_addWW_g {
		c = y
		for i := range z {
			c, z[i] = addWW_g(x[i], c, 0)
		}
		return
	}

	c = y
	for i, xi := range x[:len(z)] {
		zi := xi + c
		z[i] = zi
		c = xi &^ zi >> (_W - 1)
	}
	return
}

func subVW_g(z, x []Word, y Word) (c Word) {
	if use_addWW_g {
		c = y
		for i := range z {
			c, z[i] = subWW_g(x[i], c, 0)
		}
		return
	}

	c = y
	for i, xi := range x[:len(z)] {
		zi := xi - c
		z[i] = zi
		c = (zi &^ xi) >> (_W - 1)
	}
	return
}

func shlVU_g(z, x []Word, s uint) (c Word) {
	if n := len(z); n > 0 {
		ŝ := _W - s
		w1 := x[n-1]
		c = w1 >> ŝ
		for i := n - 1; i > 0; i-- {
			w := w1
			w1 = x[i-1]
			z[i] = w<<s | w1>>ŝ
		}
		z[0] = w1 << s
	}
	return
}

func shrVU_g(z, x []Word, s uint) (c Word) {
	if n := len(z); n > 0 {
		ŝ := _W - s
		w1 := x[0]
		c = w1 << ŝ
		for i := 0; i < n-1; i++ {
			w := w1
			w1 = x[i+1]
			z[i] = w>>s | w1<<ŝ
		}
		z[n-1] = w1 >> s
	}
	return
}

func mulAddVWW_g(z, x []Word, y, r Word) (c Word) {
	c = r
	for i := range z {
		c, z[i] = mulAddWWW_g(x[i], y, c)
	}
	return
}

// TODO(gri) Remove use of addWW_g here and then we can remove addWW_g and subWW_g.
func addMulVVW_g(z, x []Word, y Word) (c Word) {
	for i := range z {
		z1, z0 := mulAddWWW_g(x[i], y, z[i])
		c, z[i] = addWW_g(z0, c, 0)
		c += z1
	}
	return
}

func divWVW_g(z []Word, xn Word, x []Word, y Word) (r Word) {
	r = xn
	for i := len(z) - 1; i >= 0; i-- {
		z[i], r = divWW_g(r, x[i], y)
	}
	return
}
<|endoftext|>


<commit_before>package lib

import (
	"bytes"
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/streadway/amqp"
	"golang.org/x/net/context"
)

var (
	LogWriterTick = 500 * time.Millisecond

	// This is a bit of a magic number, calculated like this: The maximum
	// Pusher payload is 10 kB (or 10 KiB, who knows, but let's go with 10
	// kB since that is smaller). Looking at the travis-logs source, the
	// current message overhead (i.e. the part of the payload that isn't
	// the content of the log part) is 42 bytes + the length of the JSON-
	// encoded ID and the length of the JSON-encoded sequence number. A 64-
	// bit number is up to 20 digits long, so that means (assuming we don't
	// go over 64-bit numbers) the overhead is up to 82 bytes. That means
	// we can send up to 9918 bytes of content. However, the JSON-encoded
	// version of a string can be significantly longer than the raw bytes.
	// Worst case that I could find is "<", which with the Go JSON encoder
	// becomes "\u003c" (i.e. six bytes long). So, given a string of just
	// left angle brackets, the string would become six times as long,
	// meaning that the longest string we can take is 1653. We could still
	// get errors if we go over 64-bit numbers, but I find the likeliness
	// of that happening to both the sequence number, the ID, and us maxing
	// out the worst-case logs to be quite unlikely, so I'm willing to live
	// with that. --Henrik
	LogChunkSize = 1653
)

// A LogWriter is an io.WriteCloser that redirects to travis-logs
type LogWriter struct {
	ctx      context.Context
	amqpConn *amqp.Connection
	jobID    uint64

	closeChan chan struct{}

	bufferMutex   sync.Mutex
	buffer        *bytes.Buffer
	logPartNumber int

	amqpChanMutex sync.RWMutex
	amqpChan      *amqp.Channel
}

type logPart struct {
	JobID   uint64 `json:"id"`
	Content string `json:"log"`
	Number  int    `json:"number"`
	UUID    string `json:"uuid"`
	Final   bool   `json:"final"`
}

func NewLogWriter(ctx context.Context, conn *amqp.Connection, jobID uint64) (*LogWriter, error) {
	channel, err := conn.Channel()
	if err != nil {
		return nil, err
	}

	err = channel.ExchangeDeclare("reporting", "topic", true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	writer := &LogWriter{
		ctx:       contextFromComponent(ctx, "log_writer"),
		amqpConn:  conn,
		amqpChan:  channel,
		jobID:     jobID,
		closeChan: make(chan struct{}),
		buffer:    new(bytes.Buffer),
	}

	go writer.flushRegularly()

	return writer, nil
}

func (w *LogWriter) Write(p []byte) (int, error) {
	if w.closed() {
		return 0, fmt.Errorf("attempted write to closed log")
	}

	w.bufferMutex.Lock()
	defer w.bufferMutex.Unlock()
	return w.buffer.Write(p)
}

func (w *LogWriter) Close() error {
	if w.closed() {
		return nil
	}

	close(w.closeChan)
	w.flush()

	part := logPart{
		JobID:  w.jobID,
		Number: w.logPartNumber,
		Final:  true,
	}
	w.logPartNumber++

	return w.publishLogPart(part)
}

// WriteAndClose works like a Write followed by a Close, but ensures that no
// other Writes are allowed in between.
func (w *LogWriter) WriteAndClose(p []byte) (int, error) {
	if w.closed() {
		return 0, fmt.Errorf("log already closed")
	}

	close(w.closeChan)

	w.bufferMutex.Lock()
	n, err := w.buffer.Write(p)
	w.bufferMutex.Unlock()
	if err != nil {
		return n, err
	}

	w.flush()

	part := logPart{
		JobID:  w.jobID,
		Number: w.logPartNumber,
		Final:  true,
	}
	w.logPartNumber++

	return n, w.publishLogPart(part)
}

func (w *LogWriter) closed() bool {
	select {
	case <-w.closeChan:
		return true
	default:
		return false
	}
}

func (w *LogWriter) flushRegularly() {
	ticker := time.NewTicker(LogWriterTick)
	defer ticker.Stop()
	for {
		select {
		case <-w.closeChan:
			return
		case <-ticker.C:
			w.flush()
		}
	}
}

func (w *LogWriter) flush() {
	w.bufferMutex.Lock()
	defer w.bufferMutex.Unlock()

	if w.buffer.Len() <= 0 {
		return
	}

	buf := make([]byte, LogChunkSize)

	for w.buffer.Len() > 0 {
		n, err := w.buffer.Read(buf)
		if err != nil {
			// According to documentation, err should only be non-nil if
			// there's no data in the buffer. We've checked for this, so
			// this means that err should never be nil. Something is very
			// wrong if this happens, so let's abort!
			panic("non-empty buffer shouldn't return an error on Read")
		}

		part := logPart{
			JobID:   w.jobID,
			Content: string(buf[0:n]),
			Number:  w.logPartNumber,
		}
		w.logPartNumber++

		err = w.publishLogPart(part)
		if err != nil {
			switch err.(type) {
			case *amqp.Error:
				if w.reopenChannel() != nil {
					LoggerFromContext(w.ctx).WithField("err", err).Error("couldn't publish log part and couldn't reopen channel")
					// Close or something
					return
				}

				err = w.publishLogPart(part)
				LoggerFromContext(w.ctx).WithField("err", err).Error("couldn't publish log part, even after reopening channel")
			default:
				LoggerFromContext(w.ctx).WithField("err", err).Error("couldn't publish log part")
			}
		}
	}
}

func (w *LogWriter) publishLogPart(part logPart) error {
	part.UUID, _ = uuidFromContext(w.ctx)

	partBody, err := json.Marshal(part)
	if err != nil {
		return err
	}

	w.amqpChanMutex.RLock()
	err = w.amqpChan.Publish("reporting", "reporting.jobs.logs", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:log",
		Body:         partBody,
	})
	w.amqpChanMutex.RUnlock()

	return nil
}

func (w *LogWriter) reopenChannel() error {
	w.amqpChanMutex.Lock()
	defer w.amqpChanMutex.Unlock()

	amqpChan, err := w.amqpConn.Channel()
	if err != nil {
		return err
	}

	// reopenChannel() shouldn't be called if the channel isn't already closed.
	// but we're closing the channel again, just in case, to avoid leaking
	// channels.
	w.amqpChan.Close()

	w.amqpChan = amqpChan

	return nil
}
<commit_msg>log_writer: bind the reporting.jobs.logs queue to reporting exchange<commit_after>package lib

import (
	"bytes"
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/streadway/amqp"
	"golang.org/x/net/context"
)

var (
	LogWriterTick = 500 * time.Millisecond

	// This is a bit of a magic number, calculated like this: The maximum
	// Pusher payload is 10 kB (or 10 KiB, who knows, but let's go with 10
	// kB since that is smaller). Looking at the travis-logs source, the
	// current message overhead (i.e. the part of the payload that isn't
	// the content of the log part) is 42 bytes + the length of the JSON-
	// encoded ID and the length of the JSON-encoded sequence number. A 64-
	// bit number is up to 20 digits long, so that means (assuming we don't
	// go over 64-bit numbers) the overhead is up to 82 bytes. That means
	// we can send up to 9918 bytes of content. However, the JSON-encoded
	// version of a string can be significantly longer than the raw bytes.
	// Worst case that I could find is "<", which with the Go JSON encoder
	// becomes "\u003c" (i.e. six bytes long). So, given a string of just
	// left angle brackets, the string would become six times as long,
	// meaning that the longest string we can take is 1653. We could still
	// get errors if we go over 64-bit numbers, but I find the likeliness
	// of that happening to both the sequence number, the ID, and us maxing
	// out the worst-case logs to be quite unlikely, so I'm willing to live
	// with that. --Henrik
	LogChunkSize = 1653
)

// A LogWriter is an io.WriteCloser that redirects to travis-logs
type LogWriter struct {
	ctx      context.Context
	amqpConn *amqp.Connection
	jobID    uint64

	closeChan chan struct{}

	bufferMutex   sync.Mutex
	buffer        *bytes.Buffer
	logPartNumber int

	amqpChanMutex sync.RWMutex
	amqpChan      *amqp.Channel
}

type logPart struct {
	JobID   uint64 `json:"id"`
	Content string `json:"log"`
	Number  int    `json:"number"`
	UUID    string `json:"uuid"`
	Final   bool   `json:"final"`
}

func NewLogWriter(ctx context.Context, conn *amqp.Connection, jobID uint64) (*LogWriter, error) {
	channel, err := conn.Channel()
	if err != nil {
		return nil, err
	}

	err = channel.ExchangeDeclare("reporting", "topic", true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	_, err = channel.QueueDeclare("reporting.jobs.logs", true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	err = channel.QueueBind("reporting.jobs.logs", "reporting.jobs.logs", "reporting", false, nil)
	if err != nil {
		return nil, err
	}

	writer := &LogWriter{
		ctx:       contextFromComponent(ctx, "log_writer"),
		amqpConn:  conn,
		amqpChan:  channel,
		jobID:     jobID,
		closeChan: make(chan struct{}),
		buffer:    new(bytes.Buffer),
	}

	go writer.flushRegularly()

	return writer, nil
}

func (w *LogWriter) Write(p []byte) (int, error) {
	if w.closed() {
		return 0, fmt.Errorf("attempted write to closed log")
	}

	w.bufferMutex.Lock()
	defer w.bufferMutex.Unlock()
	return w.buffer.Write(p)
}

func (w *LogWriter) Close() error {
	if w.closed() {
		return nil
	}

	close(w.closeChan)
	w.flush()

	part := logPart{
		JobID:  w.jobID,
		Number: w.logPartNumber,
		Final:  true,
	}
	w.logPartNumber++

	return w.publishLogPart(part)
}

// WriteAndClose works like a Write followed by a Close, but ensures that no
// other Writes are allowed in between.
func (w *LogWriter) WriteAndClose(p []byte) (int, error) {
	if w.closed() {
		return 0, fmt.Errorf("log already closed")
	}

	close(w.closeChan)

	w.bufferMutex.Lock()
	n, err := w.buffer.Write(p)
	w.bufferMutex.Unlock()
	if err != nil {
		return n, err
	}

	w.flush()

	part := logPart{
		JobID:  w.jobID,
		Number: w.logPartNumber,
		Final:  true,
	}
	w.logPartNumber++

	return n, w.publishLogPart(part)
}

func (w *LogWriter) closed() bool {
	select {
	case <-w.closeChan:
		return true
	default:
		return false
	}
}

func (w *LogWriter) flushRegularly() {
	ticker := time.NewTicker(LogWriterTick)
	defer ticker.Stop()
	for {
		select {
		case <-w.closeChan:
			return
		case <-ticker.C:
			w.flush()
		}
	}
}

func (w *LogWriter) flush() {
	w.bufferMutex.Lock()
	defer w.bufferMutex.Unlock()

	if w.buffer.Len() <= 0 {
		return
	}

	buf := make([]byte, LogChunkSize)

	for w.buffer.Len() > 0 {
		n, err := w.buffer.Read(buf)
		if err != nil {
			// According to documentation, err should only be non-nil if
			// there's no data in the buffer. We've checked for this, so
			// this means that err should never be nil. Something is very
			// wrong if this happens, so let's abort!
			panic("non-empty buffer shouldn't return an error on Read")
		}

		part := logPart{
			JobID:   w.jobID,
			Content: string(buf[0:n]),
			Number:  w.logPartNumber,
		}
		w.logPartNumber++

		err = w.publishLogPart(part)
		if err != nil {
			switch err.(type) {
			case *amqp.Error:
				if w.reopenChannel() != nil {
					LoggerFromContext(w.ctx).WithField("err", err).Error("couldn't publish log part and couldn't reopen channel")
					// Close or something
					return
				}

				err = w.publishLogPart(part)
				LoggerFromContext(w.ctx).WithField("err", err).Error("couldn't publish log part, even after reopening channel")
			default:
				LoggerFromContext(w.ctx).WithField("err", err).Error("couldn't publish log part")
			}
		}
	}
}

func (w *LogWriter) publishLogPart(part logPart) error {
	part.UUID, _ = uuidFromContext(w.ctx)

	partBody, err := json.Marshal(part)
	if err != nil {
		return err
	}

	w.amqpChanMutex.RLock()
	err = w.amqpChan.Publish("reporting", "reporting.jobs.logs", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:log",
		Body:         partBody,
	})
	w.amqpChanMutex.RUnlock()

	return err
}

func (w *LogWriter) reopenChannel() error {
	w.amqpChanMutex.Lock()
	defer w.amqpChanMutex.Unlock()

	amqpChan, err := w.amqpConn.Channel()
	if err != nil {
		return err
	}

	// reopenChannel() shouldn't be called if the channel isn't already closed.
	// but we're closing the channel again, just in case, to avoid leaking
	// channels.
	w.amqpChan.Close()

	w.amqpChan = amqpChan

	return nil
}
<|endoftext|>


<commit_before>package path

import (
	"os"
	"sort"
	"strings"
	"utf8"
)

var ErrBadPattern = os.NewError("syntax error in pattern")

// Match returns true if name matches the shell file name pattern.
// The syntax used by pattern is:
//
//	pattern:
//		{ term }
//	term:
//		'*'         matches any sequence of non-/ characters
//		'?'         matches any single non-/ character
//		'[' [ '^' ] { character-range } ']'
//		            character class (must be non-empty)
//		c           matches character c (c != '*', '?', '\\', '[')
//		'\\' c      matches character c
//
//	character-range:
//		c           matches character c (c != '\\', '-', ']')
//		'\\' c      matches character c
//		lo '-' hi   matches character c for lo <= c <= hi
//
// Match requires pattern to match all of name, not just a substring.
// The only possible error return is when pattern is malformed.
//
func Match(pattern, name string) (matched bool, err os.Error) {
Pattern:
	for len(pattern) > 0 {
		var star bool
		var chunk string
		star, chunk, pattern = scanChunk(pattern)
		if star && chunk == "" {
			// Trailing * matches rest of string unless it has a /.
			return strings.Index(name, "/") < 0, nil
		}
		// Look for match at current position.
		t, ok, err := matchChunk(chunk, name)
		// if we're the last chunk, make sure we've exhausted the name
		// otherwise we'll give a false result even if we could still match
		// using the star
		if ok && (len(t) == 0 || len(pattern) > 0) {
			name = t
			continue
		}
		if err != nil {
			return false, err
		}
		if star {
			// Look for match skipping i+1 bytes.
			// Cannot skip /.
			for i := 0; i < len(name) && name[i] != '/'; i++ {
				t, ok, err := matchChunk(chunk, name[i+1:])
				if ok {
					// if we're the last chunk, make sure we exhausted the name
					if len(pattern) == 0 && len(t) > 0 {
						continue
					}
					name = t
					continue Pattern
				}
				if err != nil {
					return false, err
				}
			}
		}
		return false, nil
	}
	return len(name) == 0, nil
}

// scanChunk gets the next section of pattern, which is a non-star string
// possibly preceded by a star.
func scanChunk(pattern string) (star bool, chunk, rest string) {
	for len(pattern) > 0 && pattern[0] == '*' {
		pattern = pattern[1:]
		star = true
	}
	inrange := false
	var i int
Scan:
	for i = 0; i < len(pattern); i++ {
		switch pattern[i] {
		case '\\':
			// error check handled in matchChunk: bad pattern.
			if i+1 < len(pattern) {
				i++
			}
			continue
		case '[':
			inrange = true
		case ']':
			inrange = false
		case '*':
			if !inrange {
				break Scan
			}
		}
	}
	return star, pattern[0:i], pattern[i:]
}

// matchChunk checks whether chunk matches the beginning of s.
// If so, it returns the remainder of s (after the match).
// Chunk is all single-character operators: literals, char classes, and ?.
func matchChunk(chunk, s string) (rest string, ok bool, err os.Error) {
	for len(chunk) > 0 {
		if len(s) == 0 {
			return
		}
		switch chunk[0] {
		case '[':
			// character class
			r, n := utf8.DecodeRuneInString(s)
			s = s[n:]
			chunk = chunk[1:]
			// possibly negated
			notNegated := true
			if len(chunk) > 0 && chunk[0] == '^' {
				notNegated = false
				chunk = chunk[1:]
			}
			// parse all ranges
			match := false
			nrange := 0
			for {
				if len(chunk) > 0 && chunk[0] == ']' && nrange > 0 {
					chunk = chunk[1:]
					break
				}
				var lo, hi int
				if lo, chunk, err = getEsc(chunk); err != nil {
					return
				}
				hi = lo
				if chunk[0] == '-' {
					if hi, chunk, err = getEsc(chunk[1:]); err != nil {
						return
					}
				}
				if lo <= r && r <= hi {
					match = true
				}
				nrange++
			}
			if match != notNegated {
				return
			}

		case '?':
			if s[0] == '/' {
				return
			}
			_, n := utf8.DecodeRuneInString(s)
			s = s[n:]
			chunk = chunk[1:]

		case '\\':
			chunk = chunk[1:]
			if len(chunk) == 0 {
				err = ErrBadPattern
				return
			}
			fallthrough

		default:
			if chunk[0] != s[0] {
				return
			}
			s = s[1:]
			chunk = chunk[1:]
		}
	}
	return s, true, nil
}

// getEsc gets a possibly-escaped character from chunk, for a character class.
func getEsc(chunk string) (r int, nchunk string, err os.Error) {
	if len(chunk) == 0 || chunk[0] == '-' || chunk[0] == ']' {
		err = ErrBadPattern
		return
	}
	if chunk[0] == '\\' {
		chunk = chunk[1:]
		if len(chunk) == 0 {
			err = ErrBadPattern
			return
		}
	}
	r, n := utf8.DecodeRuneInString(chunk)
	if r == utf8.RuneError && n == 1 {
		err = ErrBadPattern
	}
	nchunk = chunk[n:]
	if len(nchunk) == 0 {
		err = ErrBadPattern
	}
	return
}

// Glob returns the names of all files matching pattern or nil
// if there is no matching file. The syntax of patterns is the same
// as in Match. The pattern may describe hierarchical names such as
// /usr/*/bin/ed.
//
func Glob(pattern string) (matches []string) {
	if !hasMeta(pattern) {
		if _, err := os.Stat(pattern); err == nil {
			return []string{pattern}
		}
		return nil
	}

	dir, file := Split(pattern)
	switch dir {
	case "":
		dir = "."
	case "/":
		// nothing
	default:
		dir = dir[0 : len(dir)-1] // chop off trailing '/'
	}

	if hasMeta(dir) {
		for _, d := range Glob(dir) {
			matches = glob(d, file, matches)
		}
	} else {
		return glob(dir, file, nil)
	}
	return matches
}

// glob searches for files matching pattern in the directory dir
// and appends them to matches.
func glob(dir, pattern string, matches []string) []string {
	if fi, err := os.Stat(dir); err != nil || !fi.IsDirectory() {
		return nil
	}
	d, err := os.Open(dir, os.O_RDONLY, 0666)
	if err != nil {
		return nil
	}
	defer d.Close()

	names, err := d.Readdirnames(-1)
	if err != nil {
		return nil
	}
	sort.SortStrings(names)

	for _, n := range names {
		matched, err := Match(pattern, n)
		if err != nil {
			return matches
		}
		if matched {
			matches = append(matches, Join(dir, n))
		}
	}
	return matches
}

// hasMeta returns true if path contains any of the magic characters
// recognized by Match.
func hasMeta(path string) bool {
	return strings.IndexAny(path, "*?[") != -1
}
<commit_msg>path: Fix Glob when it finds a file in directory position.<commit_after>package path

import (
	"os"
	"sort"
	"strings"
	"utf8"
)

var ErrBadPattern = os.NewError("syntax error in pattern")

// Match returns true if name matches the shell file name pattern.
// The syntax used by pattern is:
//
//	pattern:
//		{ term }
//	term:
//		'*'         matches any sequence of non-/ characters
//		'?'         matches any single non-/ character
//		'[' [ '^' ] { character-range } ']'
//		            character class (must be non-empty)
//		c           matches character c (c != '*', '?', '\\', '[')
//		'\\' c      matches character c
//
//	character-range:
//		c           matches character c (c != '\\', '-', ']')
//		'\\' c      matches character c
//		lo '-' hi   matches character c for lo <= c <= hi
//
// Match requires pattern to match all of name, not just a substring.
// The only possible error return is when pattern is malformed.
//
func Match(pattern, name string) (matched bool, err os.Error) {
Pattern:
	for len(pattern) > 0 {
		var star bool
		var chunk string
		star, chunk, pattern = scanChunk(pattern)
		if star && chunk == "" {
			// Trailing * matches rest of string unless it has a /.
			return strings.Index(name, "/") < 0, nil
		}
		// Look for match at current position.
		t, ok, err := matchChunk(chunk, name)
		// if we're the last chunk, make sure we've exhausted the name
		// otherwise we'll give a false result even if we could still match
		// using the star
		if ok && (len(t) == 0 || len(pattern) > 0) {
			name = t
			continue
		}
		if err != nil {
			return false, err
		}
		if star {
			// Look for match skipping i+1 bytes.
			// Cannot skip /.
			for i := 0; i < len(name) && name[i] != '/'; i++ {
				t, ok, err := matchChunk(chunk, name[i+1:])
				if ok {
					// if we're the last chunk, make sure we exhausted the name
					if len(pattern) == 0 && len(t) > 0 {
						continue
					}
					name = t
					continue Pattern
				}
				if err != nil {
					return false, err
				}
			}
		}
		return false, nil
	}
	return len(name) == 0, nil
}

// scanChunk gets the next section of pattern, which is a non-star string
// possibly preceded by a star.
func scanChunk(pattern string) (star bool, chunk, rest string) {
	for len(pattern) > 0 && pattern[0] == '*' {
		pattern = pattern[1:]
		star = true
	}
	inrange := false
	var i int
Scan:
	for i = 0; i < len(pattern); i++ {
		switch pattern[i] {
		case '\\':
			// error check handled in matchChunk: bad pattern.
			if i+1 < len(pattern) {
				i++
			}
			continue
		case '[':
			inrange = true
		case ']':
			inrange = false
		case '*':
			if !inrange {
				break Scan
			}
		}
	}
	return star, pattern[0:i], pattern[i:]
}

// matchChunk checks whether chunk matches the beginning of s.
// If so, it returns the remainder of s (after the match).
// Chunk is all single-character operators: literals, char classes, and ?.
func matchChunk(chunk, s string) (rest string, ok bool, err os.Error) {
	for len(chunk) > 0 {
		if len(s) == 0 {
			return
		}
		switch chunk[0] {
		case '[':
			// character class
			r, n := utf8.DecodeRuneInString(s)
			s = s[n:]
			chunk = chunk[1:]
			// possibly negated
			notNegated := true
			if len(chunk) > 0 && chunk[0] == '^' {
				notNegated = false
				chunk = chunk[1:]
			}
			// parse all ranges
			match := false
			nrange := 0
			for {
				if len(chunk) > 0 && chunk[0] == ']' && nrange > 0 {
					chunk = chunk[1:]
					break
				}
				var lo, hi int
				if lo, chunk, err = getEsc(chunk); err != nil {
					return
				}
				hi = lo
				if chunk[0] == '-' {
					if hi, chunk, err = getEsc(chunk[1:]); err != nil {
						return
					}
				}
				if lo <= r && r <= hi {
					match = true
				}
				nrange++
			}
			if match != notNegated {
				return
			}

		case '?':
			if s[0] == '/' {
				return
			}
			_, n := utf8.DecodeRuneInString(s)
			s = s[n:]
			chunk = chunk[1:]

		case '\\':
			chunk = chunk[1:]
			if len(chunk) == 0 {
				err = ErrBadPattern
				return
			}
			fallthrough

		default:
			if chunk[0] != s[0] {
				return
			}
			s = s[1:]
			chunk = chunk[1:]
		}
	}
	return s, true, nil
}

// getEsc gets a possibly-escaped character from chunk, for a character class.
func getEsc(chunk string) (r int, nchunk string, err os.Error) {
	if len(chunk) == 0 || chunk[0] == '-' || chunk[0] == ']' {
		err = ErrBadPattern
		return
	}
	if chunk[0] == '\\' {
		chunk = chunk[1:]
		if len(chunk) == 0 {
			err = ErrBadPattern
			return
		}
	}
	r, n := utf8.DecodeRuneInString(chunk)
	if r == utf8.RuneError && n == 1 {
		err = ErrBadPattern
	}
	nchunk = chunk[n:]
	if len(nchunk) == 0 {
		err = ErrBadPattern
	}
	return
}

// Glob returns the names of all files matching pattern or nil
// if there is no matching file. The syntax of patterns is the same
// as in Match. The pattern may describe hierarchical names such as
// /usr/*/bin/ed.
//
func Glob(pattern string) (matches []string) {
	if !hasMeta(pattern) {
		if _, err := os.Stat(pattern); err == nil {
			return []string{pattern}
		}
		return nil
	}

	dir, file := Split(pattern)
	switch dir {
	case "":
		dir = "."
	case "/":
		// nothing
	default:
		dir = dir[0 : len(dir)-1] // chop off trailing '/'
	}

	if hasMeta(dir) {
		for _, d := range Glob(dir) {
			matches = glob(d, file, matches)
		}
	} else {
		return glob(dir, file, nil)
	}
	return matches
}

// glob searches for files matching pattern in the directory dir
// and appends them to matches.
func glob(dir, pattern string, matches []string) []string {
	fi, err := os.Stat(dir)
	if err != nil {
		return nil
	}
	if !fi.IsDirectory() {
		return matches
	}
	d, err := os.Open(dir, os.O_RDONLY, 0666)
	if err != nil {
		return nil
	}
	defer d.Close()

	names, err := d.Readdirnames(-1)
	if err != nil {
		return nil
	}
	sort.SortStrings(names)

	for _, n := range names {
		matched, err := Match(pattern, n)
		if err != nil {
			return matches
		}
		if matched {
			matches = append(matches, Join(dir, n))
		}
	}
	return matches
}

// hasMeta returns true if path contains any of the magic characters
// recognized by Match.
func hasMeta(path string) bool {
	return strings.IndexAny(path, "*?[") != -1
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"bufio";
	"gob";
	"http";
	"io";
	"log";
	"net";
	"os";
	"rpc";
	"strconv";
	"sync";
)

// Call represents an active RPC
type Call struct {
	ServiceMethod	string;	// The name of the service and method to call.
	Args	interface{};	// The argument to the function (*struct).
	Reply	interface{};	// The reply from the function (*struct).
	Error	os.Error;	// After completion, the error status.
	Done	chan *Call;	// Strobes when call is complete; value is the error status.
	seq	uint64;
}

// Client represents an RPC Client.
type Client struct {
	sync.Mutex;	// protects pending, seq
	shutdown	os.Error;	// non-nil if the client is shut down
	sending	sync.Mutex;
	seq	uint64;
	conn io.ReadWriteCloser;
	enc	*gob.Encoder;
	dec	*gob.Decoder;
	pending	map[uint64] *Call;
}

func (client *Client) send(c *Call) {
	// Register this call.
	client.Lock();
	if client.shutdown != nil {
		client.Unlock();
		c.Error = client.shutdown;
		doNotBlock := c.Done <- c;
		return;
	}
	c.seq = client.seq;
	client.seq++;
	client.pending[c.seq] = c;
	client.Unlock();

	// Encode and send the request.
	request := new(Request);
	client.sending.Lock();
	request.Seq = c.seq;
	request.ServiceMethod = c.ServiceMethod;
	client.enc.Encode(request);
	err := client.enc.Encode(c.Args);
	if err != nil {
		panicln("rpc: client encode error:", err);
	}
	client.sending.Unlock();
}

func (client *Client) serve() {
	var err os.Error;
	for err == nil {
		response := new(Response);
		err = client.dec.Decode(response);
		if err != nil {
			break
		}
		seq := response.Seq;
		client.Lock();
		c := client.pending[seq];
		client.pending[seq] = c, false;
		client.Unlock();
		err = client.dec.Decode(c.Reply);
		c.Error = os.ErrorString(response.Error);
		// We don't want to block here.  It is the caller's responsibility to make
		// sure the channel has enough buffer space. See comment in Go().
		doNotBlock := c.Done <- c;
	}
	// Terminate pending calls.
	client.Lock();
	client.shutdown = err;
	for seq, call := range client.pending {
		call.Error = err;
		doNotBlock := call.Done <- call;
	}
	client.Unlock();
	log.Stderr("client protocol error:", err);
}

// NewClient returns a new Client to handle requests to the
// set of services at the other end of the connection.
func NewClient(conn io.ReadWriteCloser) *Client {
	client := new(Client);
	client.conn = conn;
	client.enc = gob.NewEncoder(conn);
	client.dec = gob.NewDecoder(conn);
	client.pending = make(map[uint64] *Call);
	go client.serve();
	return client;
}

// Dial connects to an HTTP RPC server at the specified network address.
func DialHTTP(network, address string) (*Client, os.Error) {
	conn, err := net.Dial(network, "", address);
	if err != nil {
		return nil, err
	}
	io.WriteString(conn, "CONNECT " + rpcPath + " HTTP/1.0\n\n");

	// Require successful HTTP response
	// before switching to RPC protocol.
	resp, err := http.ReadResponse(bufio.NewReader(conn));
	if err == nil && resp.Status == connected {
		return NewClient(conn), nil;
	}
	if err == nil {
		err = os.ErrorString("unexpected HTTP response: " + resp.Status);
	}
	conn.Close();
	return nil, &net.OpError{"dial-http", network, address, err};
}

// Dial connects to an RPC server at the specified network address.
func Dial(network, address string) (*Client, os.Error) {
	conn, err := net.Dial(network, "", address);
	if err != nil {
		return nil, err
	}
	return NewClient(conn), nil;
}

// Go invokes the function asynchronously.  It returns the Call structure representing
// the invocation.
func (client *Client) Go(serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call {
	c := new(Call);
	c.ServiceMethod = serviceMethod;
	c.Args = args;
	c.Reply = reply;
	if done == nil {
		done = make(chan *Call, 1);	// buffered.
	} else {
		// TODO(r): check cap > 0
		// If caller passes done != nil, it must arrange that
		// done has enough buffer for the number of simultaneous
		// RPCs that will be using that channel.
	}
	c.Done = done;
	if client.shutdown != nil {
		c.Error = client.shutdown;
		doNotBlock := c.Done <- c;
		return c;
	}
	client.send(c);
	return c;
}

// Call invokes the named function, waits for it to complete, and returns its error status.
func (client *Client) Call(serviceMethod string, args interface{}, reply interface{}) os.Error {
	if client.shutdown != nil {
		return client.shutdown
	}
	call := <-client.Go(serviceMethod, args, reply, nil).Done;
	return call.Error;
}
<commit_msg>post-submit tweaks to previous cl<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"bufio";
	"gob";
	"http";
	"io";
	"log";
	"net";
	"os";
	"rpc";
	"strconv";
	"sync";
)

// Call represents an active RPC
type Call struct {
	ServiceMethod	string;	// The name of the service and method to call.
	Args	interface{};	// The argument to the function (*struct).
	Reply	interface{};	// The reply from the function (*struct).
	Error	os.Error;	// After completion, the error status.
	Done	chan *Call;	// Strobes when call is complete; value is the error status.
	seq	uint64;
}

// Client represents an RPC Client.
type Client struct {
	sync.Mutex;	// protects pending, seq
	shutdown	os.Error;	// non-nil if the client is shut down
	sending	sync.Mutex;
	seq	uint64;
	conn io.ReadWriteCloser;
	enc	*gob.Encoder;
	dec	*gob.Decoder;
	pending	map[uint64] *Call;
}

func (client *Client) send(c *Call) {
	// Register this call.
	client.Lock();
	if client.shutdown != nil {
		c.Error = client.shutdown;
		client.Unlock();
		doNotBlock := c.Done <- c;
		return;
	}
	c.seq = client.seq;
	client.seq++;
	client.pending[c.seq] = c;
	client.Unlock();

	// Encode and send the request.
	request := new(Request);
	client.sending.Lock();
	request.Seq = c.seq;
	request.ServiceMethod = c.ServiceMethod;
	client.enc.Encode(request);
	err := client.enc.Encode(c.Args);
	if err != nil {
		panicln("rpc: client encode error:", err);
	}
	client.sending.Unlock();
}

func (client *Client) serve() {
	var err os.Error;
	for err == nil {
		response := new(Response);
		err = client.dec.Decode(response);
		if err != nil {
			if err == os.EOF {
				err = io.ErrUnexpectedEOF;
			}
			break
		}
		seq := response.Seq;
		client.Lock();
		c := client.pending[seq];
		client.pending[seq] = c, false;
		client.Unlock();
		err = client.dec.Decode(c.Reply);
		c.Error = os.ErrorString(response.Error);
		// We don't want to block here.  It is the caller's responsibility to make
		// sure the channel has enough buffer space. See comment in Go().
		doNotBlock := c.Done <- c;
	}
	// Terminate pending calls.
	client.Lock();
	client.shutdown = err;
	for seq, call := range client.pending {
		call.Error = err;
		doNotBlock := call.Done <- call;
	}
	client.Unlock();
	log.Stderr("client protocol error:", err);
}

// NewClient returns a new Client to handle requests to the
// set of services at the other end of the connection.
func NewClient(conn io.ReadWriteCloser) *Client {
	client := new(Client);
	client.conn = conn;
	client.enc = gob.NewEncoder(conn);
	client.dec = gob.NewDecoder(conn);
	client.pending = make(map[uint64] *Call);
	go client.serve();
	return client;
}

// Dial connects to an HTTP RPC server at the specified network address.
func DialHTTP(network, address string) (*Client, os.Error) {
	conn, err := net.Dial(network, "", address);
	if err != nil {
		return nil, err
	}
	io.WriteString(conn, "CONNECT " + rpcPath + " HTTP/1.0\n\n");

	// Require successful HTTP response
	// before switching to RPC protocol.
	resp, err := http.ReadResponse(bufio.NewReader(conn));
	if err == nil && resp.Status == connected {
		return NewClient(conn), nil;
	}
	if err == nil {
		err = os.ErrorString("unexpected HTTP response: " + resp.Status);
	}
	conn.Close();
	return nil, &net.OpError{"dial-http", network, address, err};
}

// Dial connects to an RPC server at the specified network address.
func Dial(network, address string) (*Client, os.Error) {
	conn, err := net.Dial(network, "", address);
	if err != nil {
		return nil, err
	}
	return NewClient(conn), nil;
}

// Go invokes the function asynchronously.  It returns the Call structure representing
// the invocation.
func (client *Client) Go(serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call {
	c := new(Call);
	c.ServiceMethod = serviceMethod;
	c.Args = args;
	c.Reply = reply;
	if done == nil {
		done = make(chan *Call, 1);	// buffered.
	} else {
		// TODO(r): check cap > 0
		// If caller passes done != nil, it must arrange that
		// done has enough buffer for the number of simultaneous
		// RPCs that will be using that channel.
	}
	c.Done = done;
	if client.shutdown != nil {
		c.Error = client.shutdown;
		doNotBlock := c.Done <- c;
		return c;
	}
	client.send(c);
	return c;
}

// Call invokes the named function, waits for it to complete, and returns its error status.
func (client *Client) Call(serviceMethod string, args interface{}, reply interface{}) os.Error {
	if client.shutdown != nil {
		return client.shutdown
	}
	call := <-client.Go(serviceMethod, args, reply, nil).Done;
	return call.Error;
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nodes

import (
	"fmt"
	"net"
	"sync"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/golang/glog"
)

type kubeNodes struct {
	client *client.Client
	// a means to list all minions
	nodeLister *cache.StoreToNodeLister
	reflector  *cache.Reflector
	// Used to stop the existing reflector.
	stopChan   chan struct{}
	goodNodes  []string       // guarded by stateLock
	nodeErrors map[string]int // guarded by stateLock
	stateLock  sync.RWMutex
}

func (self *kubeNodes) recordNodeError(name string) {
	self.stateLock.Lock()
	defer self.stateLock.Unlock()

	self.nodeErrors[name]++
}

func (self *kubeNodes) recordGoodNodes(nodes []string) {
	self.stateLock.Lock()
	defer self.stateLock.Unlock()

	self.goodNodes = nodes
}

func parseSelectorOrDie(s string) labels.Selector {
	selector, err := labels.Parse(s)
	if err != nil {
		panic(err)
	}
	return selector
}

func (self *kubeNodes) getNodeInfoAndHostname(node api.Node) (Info, string, error) {
	nodeInfo := Info{}
	hostname := ""
	var nodeErr error
	for _, addr := range node.Status.Addresses {
		switch addr.Type {
		case api.NodeExternalIP:
			nodeInfo.PublicIP = addr.Address
		case api.NodeInternalIP:
			nodeInfo.InternalIP = addr.Address
		case api.NodeHostName:
			hostname = addr.Address
		}
	}
	if hostname == "" {
		hostname = node.Name
	}
	if nodeInfo.InternalIP == "" {
		if hostname == nodeInfo.PublicIP {
			// If the only identifier we have for the node is a public IP, then use it;
			// don't force a DNS lookup
			nodeInfo.InternalIP = nodeInfo.PublicIP

		} else {
			addrs, err := net.LookupIP(hostname)
			if err == nil {
				nodeInfo.InternalIP = addrs[0].String()
			} else {
				glog.Errorf("Skipping host %s since looking up its IP failed - %s", node.Name, err)
				self.recordNodeError(node.Name)
				nodeErr = err
			}
		}
	}
	return nodeInfo, hostname, nodeErr
}

func (self *kubeNodes) List() (*NodeList, error) {
	nodeList := newNodeList()
	allNodes, err := self.nodeLister.List()
	if err != nil {
		glog.Errorf("failed to list minions via watch interface - %v", err)
		return nil, fmt.Errorf("failed to list minions via watch interface - %v", err)
	}
	glog.V(5).Infof("all kube nodes: %+v", allNodes)

	goodNodes := []string{}
	for _, node := range allNodes.Items {
		nodeInfo, hostname, err := self.getNodeInfoAndHostname(node)

		nodeList.Items[Host(hostname)] = nodeInfo
		if err == nil {
			goodNodes = append(goodNodes, node.Name)
		}
	}
	self.recordGoodNodes(goodNodes)
	glog.V(5).Infof("kube nodes found: %+v", nodeList)
	return nodeList, nil
}

func (self *kubeNodes) getState() string {
	self.stateLock.RLock()
	defer self.stateLock.RUnlock()

	state := "\tHealthy Nodes:\n"
	for _, node := range self.goodNodes {
		state += fmt.Sprintf("\t\t%s\n", node)
	}
	if len(self.nodeErrors) > 0 {
		state += fmt.Sprintf("\tNode Errors: %+v\n", self.nodeErrors)
	} else {
		state += "\tNo node errors\n"
	}
	return state
}

func (self *kubeNodes) DebugInfo() string {
	desc := "Kubernetes Nodes plugin: \n"
	desc += self.getState()
	desc += "\n"

	return desc
}

func NewKubeNodes(client *client.Client) (NodesApi, error) {
	if client == nil {
		return nil, fmt.Errorf("client is nil")
	}

	lw := cache.NewListWatchFromClient(client, "nodes", api.NamespaceAll, fields.Everything())
	nodeLister := &cache.StoreToNodeLister{Store: cache.NewStore(cache.MetaNamespaceKeyFunc)}
	reflector := cache.NewReflector(lw, &api.Node{}, nodeLister.Store, 0)
	stopChan := make(chan struct{})
	reflector.RunUntil(stopChan)

	return &kubeNodes{
		client:     client,
		nodeLister: nodeLister,
		reflector:  reflector,
		stopChan:   stopChan,
		nodeErrors: make(map[string]int),
	}, nil
}
<commit_msg>Remove nodes.parseSelectorOrDie()<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nodes

import (
	"fmt"
	"net"
	"sync"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/golang/glog"
)

type kubeNodes struct {
	client *client.Client
	// a means to list all minions
	nodeLister *cache.StoreToNodeLister
	reflector  *cache.Reflector
	// Used to stop the existing reflector.
	stopChan   chan struct{}
	goodNodes  []string       // guarded by stateLock
	nodeErrors map[string]int // guarded by stateLock
	stateLock  sync.RWMutex
}

func (self *kubeNodes) recordNodeError(name string) {
	self.stateLock.Lock()
	defer self.stateLock.Unlock()

	self.nodeErrors[name]++
}

func (self *kubeNodes) recordGoodNodes(nodes []string) {
	self.stateLock.Lock()
	defer self.stateLock.Unlock()

	self.goodNodes = nodes
}

func (self *kubeNodes) getNodeInfoAndHostname(node api.Node) (Info, string, error) {
	nodeInfo := Info{}
	hostname := ""
	var nodeErr error
	for _, addr := range node.Status.Addresses {
		switch addr.Type {
		case api.NodeExternalIP:
			nodeInfo.PublicIP = addr.Address
		case api.NodeInternalIP:
			nodeInfo.InternalIP = addr.Address
		case api.NodeHostName:
			hostname = addr.Address
		}
	}
	if hostname == "" {
		hostname = node.Name
	}
	if nodeInfo.InternalIP == "" {
		if hostname == nodeInfo.PublicIP {
			// If the only identifier we have for the node is a public IP, then use it;
			// don't force a DNS lookup
			nodeInfo.InternalIP = nodeInfo.PublicIP

		} else {
			addrs, err := net.LookupIP(hostname)
			if err == nil {
				nodeInfo.InternalIP = addrs[0].String()
			} else {
				glog.Errorf("Skipping host %s since looking up its IP failed - %s", node.Name, err)
				self.recordNodeError(node.Name)
				nodeErr = err
			}
		}
	}
	return nodeInfo, hostname, nodeErr
}

func (self *kubeNodes) List() (*NodeList, error) {
	nodeList := newNodeList()
	allNodes, err := self.nodeLister.List()
	if err != nil {
		glog.Errorf("failed to list minions via watch interface - %v", err)
		return nil, fmt.Errorf("failed to list minions via watch interface - %v", err)
	}
	glog.V(5).Infof("all kube nodes: %+v", allNodes)

	goodNodes := []string{}
	for _, node := range allNodes.Items {
		nodeInfo, hostname, err := self.getNodeInfoAndHostname(node)

		nodeList.Items[Host(hostname)] = nodeInfo
		if err == nil {
			goodNodes = append(goodNodes, node.Name)
		}
	}
	self.recordGoodNodes(goodNodes)
	glog.V(5).Infof("kube nodes found: %+v", nodeList)
	return nodeList, nil
}

func (self *kubeNodes) getState() string {
	self.stateLock.RLock()
	defer self.stateLock.RUnlock()

	state := "\tHealthy Nodes:\n"
	for _, node := range self.goodNodes {
		state += fmt.Sprintf("\t\t%s\n", node)
	}
	if len(self.nodeErrors) > 0 {
		state += fmt.Sprintf("\tNode Errors: %+v\n", self.nodeErrors)
	} else {
		state += "\tNo node errors\n"
	}
	return state
}

func (self *kubeNodes) DebugInfo() string {
	desc := "Kubernetes Nodes plugin: \n"
	desc += self.getState()
	desc += "\n"

	return desc
}

func NewKubeNodes(client *client.Client) (NodesApi, error) {
	if client == nil {
		return nil, fmt.Errorf("client is nil")
	}

	lw := cache.NewListWatchFromClient(client, "nodes", api.NamespaceAll, fields.Everything())
	nodeLister := &cache.StoreToNodeLister{Store: cache.NewStore(cache.MetaNamespaceKeyFunc)}
	reflector := cache.NewReflector(lw, &api.Node{}, nodeLister.Store, 0)
	stopChan := make(chan struct{})
	reflector.RunUntil(stopChan)

	return &kubeNodes{
		client:     client,
		nodeLister: nodeLister,
		reflector:  reflector,
		stopChan:   stopChan,
		nodeErrors: make(map[string]int),
	}, nil
}
<|endoftext|>


<commit_before>package htm

import (
//"math"
)

//Entries are positions of non-zero values
type SparseEntry struct {
	Row int
	Col int
}

//Sparse binary matrix stores indexes of non-zero entries in matrix
//to conserve space
type SparseBinaryMatrix struct {
	Width             int
	Height            int
	TotalNonZeroCount int
	Entries           []SparseEntry
}

func NewSparseBinaryMatrix(width int, height int) SparseBinaryMatrix {
	m := SparseBinaryMatrix{}
	m.Height = height
	m.Width = width
	//Intialize with 70% sparsity
	//m.Entries = make([]SparseEntry, int(math.Ceil(width*height*0.3)))
	return m
}

// func NewRandSparseBinaryMatrix() *SparseBinaryMatrix {
// }

// func (sm *SparseBinaryMatrix) Resize(width int, height int) {
// }

//Get value at col,row position
func (sm *SparseBinaryMatrix) Get(col int, row int) bool {
	for _, val := range sm.Entries {
		if val.Row == row && val.Col == col {
			return true
		}
	}
	return false
}

func (sm *SparseBinaryMatrix) delete(col int, row int) {
	for idx, val := range sm.Entries {
		if val.Row == row && val.Col == col {
			sm.Entries = append(sm.Entries[:idx], sm.Entries[idx+1:]...)
			break
		}
	}

}

//Set value at col,row position
func (sm *SparseBinaryMatrix) Set(col int, row int, value bool) {
	if !value {
		sm.delete(col, row)
		return
	}

	if sm.Get(col, row) {
		return
	}

	newEntry := SparseEntry{}
	newEntry.Col = col
	newEntry.Row = row
	sm.Entries = append(sm.Entries, newEntry)

}

//Replaces specified row with values, assumes values is ordered
//correctly
func (sm *SparseBinaryMatrix) ReplaceRow(row int, values []bool) {
	sm.validateRowCol(row, len(values))

	for i := 0; i < sm.Width; i++ {
		sm.Set(row, i, values[i])
	}
}

//Replaces row with true values at specified indices
func (sm *SparseBinaryMatrix) ReplaceRowByIndices(row int, indices []int) {
	sm.validateRow(row)

	for i := 0; i < sm.Width; i++ {
		val := false
		for x := 0; x < len(indices); x++ {
			if i == indices[x] {
				val = true
				break
			}
		}
		sm.Set(row, i, val)
	}
}

//Returns dense row
func (sm *SparseBinaryMatrix) GetDenseRow(row int) []bool {
	sm.validateRow(row)
	result := make([]bool, sm.Width)

	for i := 0; i < len(sm.Entries); i++ {
		if sm.Entries[i].Row == row {
			result[sm.Entries[i].Col] = true
		}
	}

	return result
}

//Sets a sparse row from dense representation
func (sm *SparseBinaryMatrix) SetRowFromDense(row int, denseRow []bool) {
	sm.validateRowCol(row, len(denseRow))
	for i := 0; i < sm.Width; i++ {
		sm.Set(i, row, denseRow[i])
	}
}

func (sm *SparseBinaryMatrix) validateCol(col int) {
	if col > sm.Width {
		panic("Specified row is wider than matrix.")
	}
}

func (sm *SparseBinaryMatrix) validateRow(row int) {
	if row > sm.Height {
		panic("Specified row is out of bounds.")
	}
}

func (sm *SparseBinaryMatrix) validateRowCol(row int, col int) {
	sm.validateCol(col)
	sm.validateRow(row)
}
<commit_msg>added sbm constructor<commit_after>package htm

import (
//"math"
)

//Entries are positions of non-zero values
type SparseEntry struct {
	Row int
	Col int
}

//Sparse binary matrix stores indexes of non-zero entries in matrix
//to conserve space
type SparseBinaryMatrix struct {
	Width             int
	Height            int
	TotalNonZeroCount int
	Entries           []SparseEntry
}

//Create new sparse binary matrix of specified size
func NewSparseBinaryMatrix(width int, height int) *SparseBinaryMatrix {
	m := &SparseBinaryMatrix{}
	m.Height = height
	m.Width = width
	//Intialize with 70% sparsity
	//m.Entries = make([]SparseEntry, int(math.Ceil(width*height*0.3)))
	return m
}

//Create sparse binary matrix from specified dense matrix
func NewSparseBinaryMatrixFromDense(values [][]bool) *SparseBinaryMatrix {
	if len(values) < 1 {
		panic("No values specified.")
	}
	m := &SparseBinaryMatrix{}
	m.Height = len(values)
	m.Width = len(values[0])

	for r := 0; r < len(values); r++ {
		m.SetRowFromDense(r, values[r])
	}

	return m
}

// func NewRandSparseBinaryMatrix() *SparseBinaryMatrix {
// }

// func (sm *SparseBinaryMatrix) Resize(width int, height int) {
// }

//Get value at col,row position
func (sm *SparseBinaryMatrix) Get(col int, row int) bool {
	for _, val := range sm.Entries {
		if val.Row == row && val.Col == col {
			return true
		}
	}
	return false
}

func (sm *SparseBinaryMatrix) delete(col int, row int) {
	for idx, val := range sm.Entries {
		if val.Row == row && val.Col == col {
			sm.Entries = append(sm.Entries[:idx], sm.Entries[idx+1:]...)
			break
		}
	}

}

//Set value at col,row position
func (sm *SparseBinaryMatrix) Set(col int, row int, value bool) {
	if !value {
		sm.delete(col, row)
		return
	}

	if sm.Get(col, row) {
		return
	}

	newEntry := SparseEntry{}
	newEntry.Col = col
	newEntry.Row = row
	sm.Entries = append(sm.Entries, newEntry)

}

//Replaces specified row with values, assumes values is ordered
//correctly
func (sm *SparseBinaryMatrix) ReplaceRow(row int, values []bool) {
	sm.validateRowCol(row, len(values))

	for i := 0; i < sm.Width; i++ {
		sm.Set(row, i, values[i])
	}
}

//Replaces row with true values at specified indices
func (sm *SparseBinaryMatrix) ReplaceRowByIndices(row int, indices []int) {
	sm.validateRow(row)

	for i := 0; i < sm.Width; i++ {
		val := false
		for x := 0; x < len(indices); x++ {
			if i == indices[x] {
				val = true
				break
			}
		}
		sm.Set(row, i, val)
	}
}

//Returns dense row
func (sm *SparseBinaryMatrix) GetDenseRow(row int) []bool {
	sm.validateRow(row)
	result := make([]bool, sm.Width)

	for i := 0; i < len(sm.Entries); i++ {
		if sm.Entries[i].Row == row {
			result[sm.Entries[i].Col] = true
		}
	}

	return result
}

//Sets a sparse row from dense representation
func (sm *SparseBinaryMatrix) SetRowFromDense(row int, denseRow []bool) {
	sm.validateRowCol(row, len(denseRow))
	for i := 0; i < sm.Width; i++ {
		sm.Set(i, row, denseRow[i])
	}
}

func (sm *SparseBinaryMatrix) validateCol(col int) {
	if col > sm.Width {
		panic("Specified row is wider than matrix.")
	}
}

func (sm *SparseBinaryMatrix) validateRow(row int) {
	if row > sm.Height {
		panic("Specified row is out of bounds.")
	}
}

func (sm *SparseBinaryMatrix) validateRowCol(row int, col int) {
	sm.validateCol(col)
	sm.validateRow(row)
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"
	"goshawkdb.io/common"
	msgs "goshawkdb.io/server/capnp"
	eng "goshawkdb.io/server/txnengine"
)

type versionCache map[common.VarUUId]*cached

type cached struct {
	txnId     *common.TxnId
	clockElem uint64
}

func NewVersionCache() versionCache {
	return make(map[common.VarUUId]*cached)
}

func (vc versionCache) UpdateFromCommit(txnId *common.TxnId, outcome *msgs.Outcome) {
	clock := eng.VectorClockFromCap(outcome.Commit())
	actions := outcome.Txn().Actions()
	for idx, l := 0, actions.Len(); idx < l; idx++ {
		action := actions.At(idx)
		if action.Which() != msgs.ACTION_READ {
			vUUId := common.MakeVarUUId(action.VarId())
			if c, found := vc[*vUUId]; found {
				c.txnId = txnId
				c.clockElem = clock.Clock[*vUUId]
			} else {
				vc[*vUUId] = &cached{
					txnId:     txnId,
					clockElem: clock.Clock[*vUUId],
				}
			}
		}
	}
}

func (vc versionCache) UpdateFromAbort(updates *msgs.Update_List) map[*msgs.Update][]*msgs.Action {
	validUpdates := make(map[*msgs.Update][]*msgs.Action)

	for idx, l := 0, updates.Len(); idx < l; idx++ {
		update := updates.At(idx)
		txnId := common.MakeTxnId(update.TxnId())
		clock := eng.VectorClockFromCap(update.Clock())
		actions := update.Actions()
		validActions := make([]*msgs.Action, 0, actions.Len())

		for idy, m := 0, actions.Len(); idy < m; idy++ {
			action := actions.At(idy)
			vUUId := common.MakeVarUUId(action.VarId())
			clockElem := clock.Clock[*vUUId]

			switch action.Which() {
			case msgs.ACTION_MISSING:
				if c, found := vc[*vUUId]; found {
					cmp := c.txnId.Compare(txnId)
					if clockElem > c.clockElem && cmp == common.EQ {
						panic(fmt.Sprintf("Clock version increased on missing for %v@%v (%v > %v)", vUUId, txnId, clockElem, c.clockElem))
					}
					if clockElem > c.clockElem || (clockElem == c.clockElem && cmp == common.LT) {
						delete(vc, *vUUId)
						validActions = append(validActions, &action)
					}
				}

			case msgs.ACTION_WRITE:
				if c, found := vc[*vUUId]; found {
					cmp := c.txnId.Compare(txnId)
					if clockElem > c.clockElem && cmp == common.EQ {
						panic(fmt.Sprintf("Clock version increased on write for %v@%v (%v > %v)", vUUId, txnId, clockElem, c.clockElem))
					}
					if clockElem > c.clockElem || (clockElem == c.clockElem && cmp == common.LT) {
						c.txnId = txnId
						c.clockElem = clockElem
						validActions = append(validActions, &action)
					}
				} else {
					vc[*vUUId] = &cached{
						txnId:     txnId,
						clockElem: clockElem,
					}
					validActions = append(validActions, &action)
				}

			default:
				panic(fmt.Sprintf("%v", action.Which()))
			}
		}

		if len(validActions) != 0 {
			validUpdates[&update] = validActions
		}
	}
	return validUpdates
}
<commit_msg>Improve logic.<commit_after>package client

import (
	"fmt"
	"goshawkdb.io/common"
	msgs "goshawkdb.io/server/capnp"
	eng "goshawkdb.io/server/txnengine"
)

type versionCache map[common.VarUUId]*cached

type cached struct {
	txnId     *common.TxnId
	clockElem uint64
}

func NewVersionCache() versionCache {
	return make(map[common.VarUUId]*cached)
}

func (vc versionCache) UpdateFromCommit(txnId *common.TxnId, outcome *msgs.Outcome) {
	clock := eng.VectorClockFromCap(outcome.Commit())
	actions := outcome.Txn().Actions()
	for idx, l := 0, actions.Len(); idx < l; idx++ {
		action := actions.At(idx)
		if action.Which() != msgs.ACTION_READ {
			vUUId := common.MakeVarUUId(action.VarId())
			if c, found := vc[*vUUId]; found {
				c.txnId = txnId
				c.clockElem = clock.Clock[*vUUId]
			} else {
				vc[*vUUId] = &cached{
					txnId:     txnId,
					clockElem: clock.Clock[*vUUId],
				}
			}
		}
	}
}

func (vc versionCache) UpdateFromAbort(updates *msgs.Update_List) map[*msgs.Update][]*msgs.Action {
	validUpdates := make(map[*msgs.Update][]*msgs.Action)

	for idx, l := 0, updates.Len(); idx < l; idx++ {
		update := updates.At(idx)
		txnId := common.MakeTxnId(update.TxnId())
		clock := eng.VectorClockFromCap(update.Clock())
		actions := update.Actions()
		validActions := make([]*msgs.Action, 0, actions.Len())

		for idy, m := 0, actions.Len(); idy < m; idy++ {
			action := actions.At(idy)
			vUUId := common.MakeVarUUId(action.VarId())
			clockElem := clock.Clock[*vUUId]

			switch action.Which() {
			case msgs.ACTION_MISSING:
				if c, found := vc[*vUUId]; found {
					cmp := c.txnId.Compare(txnId)
					if cmp == common.EQ && clockElem != c.clockElem {
						panic(fmt.Sprintf("Clock version changed on missing for %v@%v (new:%v != old:%v)", vUUId, txnId, clockElem, c.clockElem))
					}
					if clockElem > c.clockElem || (clockElem == c.clockElem && cmp == common.LT) {
						delete(vc, *vUUId)
						validActions = append(validActions, &action)
					}
				}

			case msgs.ACTION_WRITE:
				if c, found := vc[*vUUId]; found {
					cmp := c.txnId.Compare(txnId)
					if cmp == common.EQ && clockElem != c.clockElem {
						panic(fmt.Sprintf("Clock version changed on write for %v@%v (new:%v != old:%v)", vUUId, txnId, clockElem, c.clockElem))
					}
					if clockElem > c.clockElem || (clockElem == c.clockElem && cmp == common.LT) {
						c.txnId = txnId
						c.clockElem = clockElem
						validActions = append(validActions, &action)
					}
				} else {
					vc[*vUUId] = &cached{
						txnId:     txnId,
						clockElem: clockElem,
					}
					validActions = append(validActions, &action)
				}

			default:
				panic(fmt.Sprintf("%v", action.Which()))
			}
		}

		if len(validActions) != 0 {
			validUpdates[&update] = validActions
		}
	}
	return validUpdates
}
<|endoftext|>


<commit_before>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package handle

import (
	"context"
	"encoding/json"
	"time"

	"github.com/cznic/mathutil"
	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/infoschema"
	"github.com/pingcap/tidb/store/tikv/oracle"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/sqlexec"
	"go.uber.org/zap"
)

// GCStats will garbage collect the useless stats info. For dropped tables, we will first update their version so that
// other tidb could know that table is deleted.
func (h *Handle) GCStats(is infoschema.InfoSchema, ddlLease time.Duration) error {
	ctx := context.Background()
	// To make sure that all the deleted tables' schema and stats info have been acknowledged to all tidb,
	// we only garbage collect version before 10 lease.
	lease := mathutil.MaxInt64(int64(h.Lease()), int64(ddlLease))
	offset := DurationToTS(10 * time.Duration(lease))
	now := oracle.GoTimeToTS(time.Now())
	if now < offset {
		return nil
	}
	gcVer := now - offset
	rows, _, err := h.execRestrictedSQL(ctx, "select table_id from mysql.stats_meta where version < %?", gcVer)
	if err != nil {
		return errors.Trace(err)
	}
	for _, row := range rows {
		if err := h.gcTableStats(is, row.GetInt64(0)); err != nil {
			return errors.Trace(err)
		}
	}
	return h.removeDeletedExtendedStats(gcVer)
}

func (h *Handle) gcTableStats(is infoschema.InfoSchema, physicalID int64) error {
	ctx := context.Background()
	rows, _, err := h.execRestrictedSQL(ctx, "select is_index, hist_id from mysql.stats_histograms where table_id = %?", physicalID)
	if err != nil {
		return errors.Trace(err)
	}
	// The table has already been deleted in stats and acknowledged to all tidb,
	// we can safely remove the meta info now.
	if len(rows) == 0 {
		_, _, err = h.execRestrictedSQL(ctx, "delete from mysql.stats_meta where table_id = %?", physicalID)
		if err != nil {
			return errors.Trace(err)
		}
	}
	h.mu.Lock()
	tbl, ok := h.getTableByPhysicalID(is, physicalID)
	h.mu.Unlock()
	if !ok {
		return errors.Trace(h.DeleteTableStatsFromKV([]int64{physicalID}))
	}
	tblInfo := tbl.Meta()
	for _, row := range rows {
		isIndex, histID := row.GetInt64(0), row.GetInt64(1)
		find := false
		if isIndex == 1 {
			for _, idx := range tblInfo.Indices {
				if idx.ID == histID {
					find = true
					break
				}
			}
		} else {
			for _, col := range tblInfo.Columns {
				if col.ID == histID {
					find = true
					break
				}
			}
		}
		if !find {
			if err := h.deleteHistStatsFromKV(physicalID, histID, int(isIndex)); err != nil {
				return errors.Trace(err)
			}
		}
	}
	// Mark records in mysql.stats_extended as `deleted`.
	rows, _, err = h.execRestrictedSQL(ctx, "select name, column_ids from mysql.stats_extended where table_id = %? and status in (%?, %?)", physicalID, StatsStatusAnalyzed, StatsStatusInited)
	if err != nil {
		return errors.Trace(err)
	}
	if len(rows) == 0 {
		return nil
	}
	for _, row := range rows {
		statsName, strColIDs := row.GetString(0), row.GetString(1)
		var colIDs []int64
		err = json.Unmarshal([]byte(strColIDs), &colIDs)
		if err != nil {
			logutil.BgLogger().Debug("decode column IDs failed", zap.String("column_ids", strColIDs), zap.Error(err))
			return errors.Trace(err)
		}
		for _, colID := range colIDs {
			found := false
			for _, col := range tblInfo.Columns {
				if colID == col.ID {
					found = true
					break
				}
			}
			if !found {
				err = h.MarkExtendedStatsDeleted(statsName, physicalID, true)
				if err != nil {
					logutil.BgLogger().Debug("update stats_extended status failed", zap.String("stats_name", statsName), zap.Error(err))
					return errors.Trace(err)
				}
				break
			}
		}
	}
	return nil
}

// deleteHistStatsFromKV deletes all records about a column or an index and updates version.
func (h *Handle) deleteHistStatsFromKV(physicalID int64, histID int64, isIndex int) (err error) {
	h.mu.Lock()
	defer h.mu.Unlock()

	ctx := context.Background()
	exec := h.mu.ctx.(sqlexec.SQLExecutor)
	_, err = exec.ExecuteInternal(ctx, "begin")
	if err != nil {
		return errors.Trace(err)
	}
	defer func() {
		err = finishTransaction(ctx, exec, err)
	}()
	txn, err := h.mu.ctx.Txn(true)
	if err != nil {
		return errors.Trace(err)
	}
	startTS := txn.StartTS()
	// First of all, we update the version. If this table doesn't exist, it won't have any problem. Because we cannot delete anything.
	if _, err = exec.ExecuteInternal(ctx, "update mysql.stats_meta set version = %? where table_id = %? ", startTS, physicalID); err != nil {
		return err
	}
	// delete histogram meta
	if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_histograms where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
		return err
	}
	// delete top n data
	if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_top_n where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
		return err
	}
	// delete all buckets
	if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_buckets where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
		return err
	}
	// delete all fm sketch
	if _, err := exec.ExecuteInternal(ctx, "delete from mysql.stats_fm_sketch where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
		return err
	}
	return nil
}

// DeleteTableStatsFromKV deletes table statistics from kv.
// A statsID refers to statistic of a table or a partition.
func (h *Handle) DeleteTableStatsFromKV(statsIDs []int64) (err error) {
	h.mu.Lock()
	defer h.mu.Unlock()
	exec := h.mu.ctx.(sqlexec.SQLExecutor)
	_, err = exec.ExecuteInternal(context.Background(), "begin")
	if err != nil {
		return errors.Trace(err)
	}
	defer func() {
		err = finishTransaction(context.Background(), exec, err)
	}()
	txn, err := h.mu.ctx.Txn(true)
	if err != nil {
		return errors.Trace(err)
	}
	ctx := context.Background()
	startTS := txn.StartTS()
	for _, statsID := range statsIDs {
		// We only update the version so that other tidb will know that this table is deleted.
		if _, err = exec.ExecuteInternal(ctx, "update mysql.stats_meta set version = %? where table_id = %? ", startTS, statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_histograms where table_id = %?", statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_buckets where table_id = %?", statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_top_n where table_id = %?", statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_feedback where table_id = %?", statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "update mysql.stats_extended set version = %?, status = %? where table_id = %? and status in (%?, %?)", startTS, StatsStatusDeleted, statsID, StatsStatusAnalyzed, StatsStatusInited); err != nil {
			return err
		}
	}
	return nil
}

func (h *Handle) removeDeletedExtendedStats(version uint64) (err error) {
	h.mu.Lock()
	defer h.mu.Unlock()
	exec := h.mu.ctx.(sqlexec.SQLExecutor)
	ctx := context.Background()
	_, err = exec.ExecuteInternal(ctx, "begin pessimistic")
	if err != nil {
		return errors.Trace(err)
	}
	defer func() {
		err = finishTransaction(ctx, exec, err)
	}()
	const sql = "delete from mysql.stats_extended where status = %? and version < %?"
	_, err = exec.ExecuteInternal(ctx, sql, StatsStatusDeleted, version)
	return
}
<commit_msg>statistics: print log when tidb marks extended stats as deleted internally (#23834)<commit_after>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package handle

import (
	"context"
	"encoding/json"
	"time"

	"github.com/cznic/mathutil"
	"github.com/pingcap/errors"
	"github.com/pingcap/tidb/infoschema"
	"github.com/pingcap/tidb/store/tikv/oracle"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/sqlexec"
	"go.uber.org/zap"
)

// GCStats will garbage collect the useless stats info. For dropped tables, we will first update their version so that
// other tidb could know that table is deleted.
func (h *Handle) GCStats(is infoschema.InfoSchema, ddlLease time.Duration) error {
	ctx := context.Background()
	// To make sure that all the deleted tables' schema and stats info have been acknowledged to all tidb,
	// we only garbage collect version before 10 lease.
	lease := mathutil.MaxInt64(int64(h.Lease()), int64(ddlLease))
	offset := DurationToTS(10 * time.Duration(lease))
	now := oracle.GoTimeToTS(time.Now())
	if now < offset {
		return nil
	}
	gcVer := now - offset
	rows, _, err := h.execRestrictedSQL(ctx, "select table_id from mysql.stats_meta where version < %?", gcVer)
	if err != nil {
		return errors.Trace(err)
	}
	for _, row := range rows {
		if err := h.gcTableStats(is, row.GetInt64(0)); err != nil {
			return errors.Trace(err)
		}
	}
	return h.removeDeletedExtendedStats(gcVer)
}

func (h *Handle) gcTableStats(is infoschema.InfoSchema, physicalID int64) error {
	ctx := context.Background()
	rows, _, err := h.execRestrictedSQL(ctx, "select is_index, hist_id from mysql.stats_histograms where table_id = %?", physicalID)
	if err != nil {
		return errors.Trace(err)
	}
	// The table has already been deleted in stats and acknowledged to all tidb,
	// we can safely remove the meta info now.
	if len(rows) == 0 {
		_, _, err = h.execRestrictedSQL(ctx, "delete from mysql.stats_meta where table_id = %?", physicalID)
		if err != nil {
			return errors.Trace(err)
		}
	}
	h.mu.Lock()
	tbl, ok := h.getTableByPhysicalID(is, physicalID)
	h.mu.Unlock()
	if !ok {
		logutil.BgLogger().Info("remove stats in GC due to dropped table", zap.Int64("table_id", physicalID))
		return errors.Trace(h.DeleteTableStatsFromKV([]int64{physicalID}))
	}
	tblInfo := tbl.Meta()
	for _, row := range rows {
		isIndex, histID := row.GetInt64(0), row.GetInt64(1)
		find := false
		if isIndex == 1 {
			for _, idx := range tblInfo.Indices {
				if idx.ID == histID {
					find = true
					break
				}
			}
		} else {
			for _, col := range tblInfo.Columns {
				if col.ID == histID {
					find = true
					break
				}
			}
		}
		if !find {
			if err := h.deleteHistStatsFromKV(physicalID, histID, int(isIndex)); err != nil {
				return errors.Trace(err)
			}
		}
	}
	// Mark records in mysql.stats_extended as `deleted`.
	rows, _, err = h.execRestrictedSQL(ctx, "select name, column_ids from mysql.stats_extended where table_id = %? and status in (%?, %?)", physicalID, StatsStatusAnalyzed, StatsStatusInited)
	if err != nil {
		return errors.Trace(err)
	}
	if len(rows) == 0 {
		return nil
	}
	for _, row := range rows {
		statsName, strColIDs := row.GetString(0), row.GetString(1)
		var colIDs []int64
		err = json.Unmarshal([]byte(strColIDs), &colIDs)
		if err != nil {
			logutil.BgLogger().Debug("decode column IDs failed", zap.String("column_ids", strColIDs), zap.Error(err))
			return errors.Trace(err)
		}
		for _, colID := range colIDs {
			found := false
			for _, col := range tblInfo.Columns {
				if colID == col.ID {
					found = true
					break
				}
			}
			if !found {
				logutil.BgLogger().Info("mark mysql.stats_extended record as 'deleted' in GC due to dropped columns", zap.String("table_name", tblInfo.Name.L), zap.Int64("table_id", physicalID), zap.String("stats_name", statsName), zap.Int64("dropped_column_id", colID))
				err = h.MarkExtendedStatsDeleted(statsName, physicalID, true)
				if err != nil {
					logutil.BgLogger().Debug("update stats_extended status failed", zap.String("stats_name", statsName), zap.Error(err))
					return errors.Trace(err)
				}
				break
			}
		}
	}
	return nil
}

// deleteHistStatsFromKV deletes all records about a column or an index and updates version.
func (h *Handle) deleteHistStatsFromKV(physicalID int64, histID int64, isIndex int) (err error) {
	h.mu.Lock()
	defer h.mu.Unlock()

	ctx := context.Background()
	exec := h.mu.ctx.(sqlexec.SQLExecutor)
	_, err = exec.ExecuteInternal(ctx, "begin")
	if err != nil {
		return errors.Trace(err)
	}
	defer func() {
		err = finishTransaction(ctx, exec, err)
	}()
	txn, err := h.mu.ctx.Txn(true)
	if err != nil {
		return errors.Trace(err)
	}
	startTS := txn.StartTS()
	// First of all, we update the version. If this table doesn't exist, it won't have any problem. Because we cannot delete anything.
	if _, err = exec.ExecuteInternal(ctx, "update mysql.stats_meta set version = %? where table_id = %? ", startTS, physicalID); err != nil {
		return err
	}
	// delete histogram meta
	if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_histograms where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
		return err
	}
	// delete top n data
	if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_top_n where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
		return err
	}
	// delete all buckets
	if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_buckets where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
		return err
	}
	// delete all fm sketch
	if _, err := exec.ExecuteInternal(ctx, "delete from mysql.stats_fm_sketch where table_id = %? and hist_id = %? and is_index = %?", physicalID, histID, isIndex); err != nil {
		return err
	}
	return nil
}

// DeleteTableStatsFromKV deletes table statistics from kv.
// A statsID refers to statistic of a table or a partition.
func (h *Handle) DeleteTableStatsFromKV(statsIDs []int64) (err error) {
	h.mu.Lock()
	defer h.mu.Unlock()
	exec := h.mu.ctx.(sqlexec.SQLExecutor)
	_, err = exec.ExecuteInternal(context.Background(), "begin")
	if err != nil {
		return errors.Trace(err)
	}
	defer func() {
		err = finishTransaction(context.Background(), exec, err)
	}()
	txn, err := h.mu.ctx.Txn(true)
	if err != nil {
		return errors.Trace(err)
	}
	ctx := context.Background()
	startTS := txn.StartTS()
	for _, statsID := range statsIDs {
		// We only update the version so that other tidb will know that this table is deleted.
		if _, err = exec.ExecuteInternal(ctx, "update mysql.stats_meta set version = %? where table_id = %? ", startTS, statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_histograms where table_id = %?", statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_buckets where table_id = %?", statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_top_n where table_id = %?", statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "delete from mysql.stats_feedback where table_id = %?", statsID); err != nil {
			return err
		}
		if _, err = exec.ExecuteInternal(ctx, "update mysql.stats_extended set version = %?, status = %? where table_id = %? and status in (%?, %?)", startTS, StatsStatusDeleted, statsID, StatsStatusAnalyzed, StatsStatusInited); err != nil {
			return err
		}
	}
	return nil
}

func (h *Handle) removeDeletedExtendedStats(version uint64) (err error) {
	h.mu.Lock()
	defer h.mu.Unlock()
	exec := h.mu.ctx.(sqlexec.SQLExecutor)
	ctx := context.Background()
	_, err = exec.ExecuteInternal(ctx, "begin pessimistic")
	if err != nil {
		return errors.Trace(err)
	}
	defer func() {
		err = finishTransaction(ctx, exec, err)
	}()
	const sql = "delete from mysql.stats_extended where status = %? and version < %?"
	_, err = exec.ExecuteInternal(ctx, sql, StatsStatusDeleted, version)
	return
}
<|endoftext|>