<commit_before>package ackhandler

import (
	"time"

	"github.com/golang/mock/gomock"

	"github.com/lucas-clemente/quic-go/internal/congestion"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/utils"
	"github.com/lucas-clemente/quic-go/internal/wire"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Received Packet Handler", func() {
	var handler ReceivedPacketHandler
	var sentPackets *MockSentPacketTracker

	BeforeEach(func() {
		sentPackets = NewMockSentPacketTracker(mockCtrl)
		handler = newReceivedPacketHandler(
			sentPackets,
			&congestion.RTTStats{},
			utils.DefaultLogger,
			protocol.VersionWhatever,
		)
	})

	It("generates ACKs for different packet number spaces", func() {
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sendTime := time.Now().Add(-time.Second)
		sentPackets.EXPECT().ReceivedPacket(protocol.EncryptionInitial).Times(2)
		sentPackets.EXPECT().ReceivedPacket(protocol.EncryptionHandshake).Times(2)
		sentPackets.EXPECT().ReceivedPacket(protocol.Encryption1RTT).Times(2)
		Expect(handler.ReceivedPacket(2, protocol.EncryptionInitial, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(1, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(5, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(3, protocol.EncryptionInitial, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(2, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(4, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		initialAck := handler.GetAckFrame(protocol.EncryptionInitial)
		Expect(initialAck).ToNot(BeNil())
		Expect(initialAck.AckRanges).To(HaveLen(1))
		Expect(initialAck.AckRanges[0]).To(Equal(wire.AckRange{Smallest: 2, Largest: 3}))
		Expect(initialAck.DelayTime).To(BeZero())
		handshakeAck := handler.GetAckFrame(protocol.EncryptionHandshake)
		Expect(handshakeAck).ToNot(BeNil())
		Expect(handshakeAck.AckRanges).To(HaveLen(1))
		Expect(handshakeAck.AckRanges[0]).To(Equal(wire.AckRange{Smallest: 1, Largest: 2}))
		Expect(handshakeAck.DelayTime).To(BeZero())
		oneRTTAck := handler.GetAckFrame(protocol.Encryption1RTT)
		Expect(oneRTTAck).ToNot(BeNil())
		Expect(oneRTTAck.AckRanges).To(HaveLen(1))
		Expect(oneRTTAck.AckRanges[0]).To(Equal(wire.AckRange{Smallest: 4, Largest: 5}))
		Expect(oneRTTAck.DelayTime).To(BeNumerically("~", time.Second, 50*time.Millisecond))
	})

	It("uses the same packet number space for 0-RTT and 1-RTT packets", func() {
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sentPackets.EXPECT().ReceivedPacket(protocol.Encryption0RTT)
		sentPackets.EXPECT().ReceivedPacket(protocol.Encryption1RTT)
		sendTime := time.Now().Add(-time.Second)
		Expect(handler.ReceivedPacket(2, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(3, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		ack := handler.GetAckFrame(protocol.Encryption1RTT)
		Expect(ack).ToNot(BeNil())
		Expect(ack.AckRanges).To(HaveLen(1))
		Expect(ack.AckRanges[0]).To(Equal(wire.AckRange{Smallest: 2, Largest: 3}))
	})

	It("rejects 0-RTT packets with higher packet numbers than 1-RTT packets", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).Times(3)
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sendTime := time.Now()
		Expect(handler.ReceivedPacket(10, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(11, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(12, protocol.Encryption0RTT, sendTime, true)).To(MatchError("received packet number 12 on a 0-RTT packet after receiving 11 on a 1-RTT packet"))
	})

	It("allows reordered 0-RTT packets", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).Times(3)
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sendTime := time.Now()
		Expect(handler.ReceivedPacket(10, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(12, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(11, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
	})

	It("drops Initial packets", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).Times(2)
		sendTime := time.Now().Add(-time.Second)
		Expect(handler.ReceivedPacket(2, protocol.EncryptionInitial, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(1, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.GetAckFrame(protocol.EncryptionInitial)).ToNot(BeNil())
		handler.DropPackets(protocol.EncryptionInitial)
		Expect(handler.GetAckFrame(protocol.EncryptionInitial)).To(BeNil())
		Expect(handler.GetAckFrame(protocol.EncryptionHandshake)).ToNot(BeNil())
	})

	It("drops Handshake packets", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).Times(2)
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sendTime := time.Now().Add(-time.Second)
		Expect(handler.ReceivedPacket(1, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(2, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.GetAckFrame(protocol.EncryptionHandshake)).ToNot(BeNil())
		handler.DropPackets(protocol.EncryptionInitial)
		Expect(handler.GetAckFrame(protocol.EncryptionHandshake)).To(BeNil())
		Expect(handler.GetAckFrame(protocol.Encryption1RTT)).ToNot(BeNil())
	})

	It("does nothing when dropping 0-RTT packets", func() {
		handler.DropPackets(protocol.Encryption0RTT)
	})

	It("drops old ACK ranges", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).AnyTimes()
		sendTime := time.Now()
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().Times(2)
		Expect(handler.ReceivedPacket(1, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(2, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		ack := handler.GetAckFrame(protocol.Encryption1RTT)
		Expect(ack).ToNot(BeNil())
		Expect(ack.LowestAcked()).To(Equal(protocol.PacketNumber(1)))
		Expect(ack.LargestAcked()).To(Equal(protocol.PacketNumber(2)))
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked()
		Expect(handler.ReceivedPacket(3, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().Return(protocol.PacketNumber(2))
		Expect(handler.ReceivedPacket(4, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		ack = handler.GetAckFrame(protocol.Encryption1RTT)
		Expect(ack).ToNot(BeNil())
		Expect(ack.LowestAcked()).To(Equal(protocol.PacketNumber(2)))
		Expect(ack.LargestAcked()).To(Equal(protocol.PacketNumber(4)))
	})

	It("says if packets are duplicates", func() {
		sendTime := time.Now()
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		// Initial
		Expect(handler.IsPotentiallyDuplicate(3, protocol.EncryptionInitial)).To(BeFalse())
		Expect(handler.ReceivedPacket(3, protocol.EncryptionInitial, sendTime, true)).To(Succeed())
		Expect(handler.IsPotentiallyDuplicate(3, protocol.EncryptionInitial)).To(BeTrue())
		// Handshake
		Expect(handler.IsPotentiallyDuplicate(3, protocol.EncryptionHandshake)).To(BeFalse())
		Expect(handler.ReceivedPacket(3, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.IsPotentiallyDuplicate(3, protocol.EncryptionHandshake)).To(BeTrue())
		// 0-RTT
		Expect(handler.IsPotentiallyDuplicate(3, protocol.Encryption0RTT)).To(BeFalse())
		Expect(handler.ReceivedPacket(3, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
		Expect(handler.IsPotentiallyDuplicate(3, protocol.Encryption0RTT)).To(BeTrue())
		// 1-RTT
		Expect(handler.IsPotentiallyDuplicate(3, protocol.Encryption1RTT)).To(BeTrue())
		Expect(handler.IsPotentiallyDuplicate(4, protocol.Encryption1RTT)).To(BeFalse())
		Expect(handler.ReceivedPacket(4, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.IsPotentiallyDuplicate(4, protocol.Encryption1RTT)).To(BeTrue())
	})
})
<commit_msg>fix broken unit test in ackhandler<commit_after>package ackhandler

import (
	"time"

	"github.com/golang/mock/gomock"

	"github.com/lucas-clemente/quic-go/internal/congestion"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/utils"
	"github.com/lucas-clemente/quic-go/internal/wire"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Received Packet Handler", func() {
	var handler ReceivedPacketHandler
	var sentPackets *MockSentPacketTracker

	BeforeEach(func() {
		sentPackets = NewMockSentPacketTracker(mockCtrl)
		handler = newReceivedPacketHandler(
			sentPackets,
			&congestion.RTTStats{},
			utils.DefaultLogger,
			protocol.VersionWhatever,
		)
	})

	It("generates ACKs for different packet number spaces", func() {
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sendTime := time.Now().Add(-time.Second)
		sentPackets.EXPECT().ReceivedPacket(protocol.EncryptionInitial).Times(2)
		sentPackets.EXPECT().ReceivedPacket(protocol.EncryptionHandshake).Times(2)
		sentPackets.EXPECT().ReceivedPacket(protocol.Encryption1RTT).Times(2)
		Expect(handler.ReceivedPacket(2, protocol.EncryptionInitial, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(1, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(5, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(3, protocol.EncryptionInitial, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(2, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(4, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		initialAck := handler.GetAckFrame(protocol.EncryptionInitial)
		Expect(initialAck).ToNot(BeNil())
		Expect(initialAck.AckRanges).To(HaveLen(1))
		Expect(initialAck.AckRanges[0]).To(Equal(wire.AckRange{Smallest: 2, Largest: 3}))
		Expect(initialAck.DelayTime).To(BeZero())
		handshakeAck := handler.GetAckFrame(protocol.EncryptionHandshake)
		Expect(handshakeAck).ToNot(BeNil())
		Expect(handshakeAck.AckRanges).To(HaveLen(1))
		Expect(handshakeAck.AckRanges[0]).To(Equal(wire.AckRange{Smallest: 1, Largest: 2}))
		Expect(handshakeAck.DelayTime).To(BeZero())
		oneRTTAck := handler.GetAckFrame(protocol.Encryption1RTT)
		Expect(oneRTTAck).ToNot(BeNil())
		Expect(oneRTTAck.AckRanges).To(HaveLen(1))
		Expect(oneRTTAck.AckRanges[0]).To(Equal(wire.AckRange{Smallest: 4, Largest: 5}))
		Expect(oneRTTAck.DelayTime).To(BeNumerically("~", time.Second, 50*time.Millisecond))
	})

	It("uses the same packet number space for 0-RTT and 1-RTT packets", func() {
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sentPackets.EXPECT().ReceivedPacket(protocol.Encryption0RTT)
		sentPackets.EXPECT().ReceivedPacket(protocol.Encryption1RTT)
		sendTime := time.Now().Add(-time.Second)
		Expect(handler.ReceivedPacket(2, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(3, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		ack := handler.GetAckFrame(protocol.Encryption1RTT)
		Expect(ack).ToNot(BeNil())
		Expect(ack.AckRanges).To(HaveLen(1))
		Expect(ack.AckRanges[0]).To(Equal(wire.AckRange{Smallest: 2, Largest: 3}))
	})

	It("rejects 0-RTT packets with higher packet numbers than 1-RTT packets", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).Times(3)
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sendTime := time.Now()
		Expect(handler.ReceivedPacket(10, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(11, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(12, protocol.Encryption0RTT, sendTime, true)).To(MatchError("received packet number 12 on a 0-RTT packet after receiving 11 on a 1-RTT packet"))
	})

	It("allows reordered 0-RTT packets", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).Times(3)
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sendTime := time.Now()
		Expect(handler.ReceivedPacket(10, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(12, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(11, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
	})

	It("drops Initial packets", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).Times(2)
		sendTime := time.Now().Add(-time.Second)
		Expect(handler.ReceivedPacket(2, protocol.EncryptionInitial, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(1, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.GetAckFrame(protocol.EncryptionInitial)).ToNot(BeNil())
		handler.DropPackets(protocol.EncryptionInitial)
		Expect(handler.GetAckFrame(protocol.EncryptionInitial)).To(BeNil())
		Expect(handler.GetAckFrame(protocol.EncryptionHandshake)).ToNot(BeNil())
	})

	It("drops Handshake packets", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).Times(2)
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		sendTime := time.Now().Add(-time.Second)
		Expect(handler.ReceivedPacket(1, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(2, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.GetAckFrame(protocol.EncryptionHandshake)).ToNot(BeNil())
		handler.DropPackets(protocol.EncryptionInitial)
		Expect(handler.GetAckFrame(protocol.EncryptionHandshake)).To(BeNil())
		Expect(handler.GetAckFrame(protocol.Encryption1RTT)).ToNot(BeNil())
	})

	It("does nothing when dropping 0-RTT packets", func() {
		handler.DropPackets(protocol.Encryption0RTT)
	})

	It("drops old ACK ranges", func() {
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).AnyTimes()
		sendTime := time.Now()
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().Times(2)
		Expect(handler.ReceivedPacket(1, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.ReceivedPacket(2, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		ack := handler.GetAckFrame(protocol.Encryption1RTT)
		Expect(ack).ToNot(BeNil())
		Expect(ack.LowestAcked()).To(Equal(protocol.PacketNumber(1)))
		Expect(ack.LargestAcked()).To(Equal(protocol.PacketNumber(2)))
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked()
		Expect(handler.ReceivedPacket(3, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().Return(protocol.PacketNumber(2))
		Expect(handler.ReceivedPacket(4, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		ack = handler.GetAckFrame(protocol.Encryption1RTT)
		Expect(ack).ToNot(BeNil())
		Expect(ack.LowestAcked()).To(Equal(protocol.PacketNumber(2)))
		Expect(ack.LargestAcked()).To(Equal(protocol.PacketNumber(4)))
	})

	It("says if packets are duplicates", func() {
		sendTime := time.Now()
		sentPackets.EXPECT().ReceivedPacket(gomock.Any()).AnyTimes()
		sentPackets.EXPECT().GetLowestPacketNotConfirmedAcked().AnyTimes()
		// Initial
		Expect(handler.IsPotentiallyDuplicate(3, protocol.EncryptionInitial)).To(BeFalse())
		Expect(handler.ReceivedPacket(3, protocol.EncryptionInitial, sendTime, true)).To(Succeed())
		Expect(handler.IsPotentiallyDuplicate(3, protocol.EncryptionInitial)).To(BeTrue())
		// Handshake
		Expect(handler.IsPotentiallyDuplicate(3, protocol.EncryptionHandshake)).To(BeFalse())
		Expect(handler.ReceivedPacket(3, protocol.EncryptionHandshake, sendTime, true)).To(Succeed())
		Expect(handler.IsPotentiallyDuplicate(3, protocol.EncryptionHandshake)).To(BeTrue())
		// 0-RTT
		Expect(handler.IsPotentiallyDuplicate(3, protocol.Encryption0RTT)).To(BeFalse())
		Expect(handler.ReceivedPacket(3, protocol.Encryption0RTT, sendTime, true)).To(Succeed())
		Expect(handler.IsPotentiallyDuplicate(3, protocol.Encryption0RTT)).To(BeTrue())
		// 1-RTT
		Expect(handler.IsPotentiallyDuplicate(3, protocol.Encryption1RTT)).To(BeTrue())
		Expect(handler.IsPotentiallyDuplicate(4, protocol.Encryption1RTT)).To(BeFalse())
		Expect(handler.ReceivedPacket(4, protocol.Encryption1RTT, sendTime, true)).To(Succeed())
		Expect(handler.IsPotentiallyDuplicate(4, protocol.Encryption1RTT)).To(BeTrue())
	})
})
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package wsus implements WSUS repository importer.
package wsus

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/csv"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"github.com/google/hashr/core/hashr"
	"github.com/google/hashr/importers/common"

	"google.golang.org/api/storage/v1"
)

type updateFormat int

const (
	// RepoName contains the repository name.
	RepoName              = "wsus"
	path7z                = "/usr/bin/7z"
	exe      updateFormat = iota
	cabArchive
)

var (
	storageClient = &storage.Service{}
	gcsBucket     string
)

type update struct {
	id              string
	remotePath      string
	md5hash         string
	quickSha256hash string
	localPath       string
	tempDir         string
	format          updateFormat
	updateTitle     string
}

// Preprocess extracts the contents of Windows Update file.
func (u *update) Preprocess() (string, error) {
	if err := u.download(); err != nil {
		return "", err
	}

	extractionDir := filepath.Join(u.tempDir, "extracted")
	if err := os.Mkdir(extractionDir, 0755); err != nil {
		return "", fmt.Errorf("could not create target %s directory: %v", extractionDir, err)
	}

	switch u.format {
	case cabArchive, exe:
		if err := u.recursiveExtract(extractionDir); err != nil {
			return "", err
		}
	}

	return extractionDir, nil
}

// ID returns non-unique Windows Update ID.
func (u *update) ID() string {
	return u.id
}

// RepoName returns repository name.
func (u *update) RepoName() string {
	return RepoName
}

// RepoPath returns repository path.
func (u *update) RepoPath() string {
	return fmt.Sprintf("gs://%s/", gcsBucket)
}

// LocalPath returns local path to a Windows Update file.
func (u *update) LocalPath() string {
	return u.localPath
}

// RemotePath returns remote path to a Windows Update file stored in the GCS bucket.
func (u *update) RemotePath() string {
	return fmt.Sprintf("gs://%s/%s", gcsBucket, u.remotePath)
}

// Description provides additional description for a Windows Update file.
func (u *update) Description() string {
	return u.updateTitle
}

// QuickSHA256Hash calculates sha256 hash of a Windows Update file metadata.
func (u *update) QuickSHA256Hash() (string, error) {
	if u.quickSha256hash != "" {
		return u.quickSha256hash, nil
	}

	u.quickSha256hash = fmt.Sprintf("%x", sha256.Sum256([]byte(u.id+u.md5hash)))

	return u.quickSha256hash, nil
}

func (u *update) download() error {
	_, file := filepath.Split(u.remotePath)

	resp, err := storageClient.Objects.Get(gcsBucket, u.remotePath).Download()
	if err != nil {
		glog.Exit(err)
		return err
	}
	defer resp.Body.Close()

	u.tempDir, err = common.LocalTempDir(u.ID())
	if err != nil {
		glog.Exit(err)
		return err
	}

	u.localPath = filepath.Join(u.tempDir, file)
	out, err := os.Create(u.localPath)
	if err != nil {
		glog.Exit(err)
		return fmt.Errorf("error while creating %s: %v", u.localPath, err)
	}

	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return fmt.Errorf("error while writing to %s: %v", u.localPath, err)
	}

	glog.Infof("Update %s successfully written to %s", u.remotePath, u.localPath)

	return nil
}

func (u *update) recursiveExtract(extractionDir string) error {
	if err := extract7z(u.localPath, extractionDir); err != nil {
		return err
	}

	var files []string
	if err := filepath.Walk(extractionDir, walk(&files)); err != nil {
		return err
	}

	if err := recursiveExtract(files); err != nil {
		return err
	}

	return nil
}

func extractFile(fileCmdOutput string) bool {
	toBeExtracted := []string{"compressed", "archive", "application/x-msi", "application/vnd.ms-msi"}
	for _, s := range toBeExtracted {
		if strings.Contains(fileCmdOutput, s) {
			return true
		}
	}

	return false
}

func extractExt(ext string) bool {
	toBeExtracted := []string{".cab", ".msi", ".7z", ".msu", ".msp"}
	for _, e := range toBeExtracted {
		if strings.EqualFold(e, ext) {
			return true
		}
	}
	return false
}

func recursiveExtract(files []string) error {
	for _, fullPath := range files {
		_, filename := filepath.Split(fullPath)
		extension := filepath.Ext(filename)

		var tryToExtract bool
		// Since the files come from MS we can trust the extension, but if there is none, we should
		// try to detect the mime type. The easiest and most reliable solution is to run file binary.
		if extension == "" {
			out, err := shellCommand("/usr/bin/file", "--mime-type", fullPath)
			if err != nil {
				return err
			}
			if extractFile(out) {
				tryToExtract = true
			}
		}

		if extractExt(extension) || tryToExtract {
			targetDir := fmt.Sprintf("%s_%s", fullPath, "extracted")
			if err := os.Mkdir(targetDir, 0755); err != nil {
				return fmt.Errorf("could not create target %s directory: %v", targetDir, err)
			}
			if err := extract7z(fullPath, targetDir); err != nil {
				glog.Errorf("error while extracting archive file: %v", err)
				continue
			}

			var extractedFiles []string
			if err := filepath.Walk(targetDir, walk(&extractedFiles)); err != nil {
				return err
			}

			if len(extractedFiles) > 0 {
				if err := recursiveExtract(extractedFiles); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

// This is mainly to enable testing.
var execute = func(name string, args ...string) *exec.Cmd {
	return exec.Command(name, args...)
}

func shellCommand(binary string, args ...string) (string, error) {
	cmd := execute(binary, args...)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("error while executing %s: %v\nStdout: %v\nStderr: %v", binary, err, stdout.String(), stderr.String())
	}

	return stdout.String(), nil
}

func extract7z(filepath, targetDir string) error {
	_, err := shellCommand(path7z, "x", filepath, fmt.Sprintf("-o%s", targetDir))
	if err != nil {
		return fmt.Errorf("error while running 7z: %v", err)
	}

	return nil
}

func walk(files *[]string) filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			glog.Errorf("Could not open %s: %v", path, err)
			return nil
		}
		if info.IsDir() {
			return nil
		}

		*files = append(*files, path)
		return nil
	}
}

// NewRepo returns new instance of a WSUS repository.
func NewRepo(ctx context.Context, storageService *storage.Service, gcsWSUSBucket string) (*Repo, error) {
	storageClient = storageService
	gcsBucket = gcsWSUSBucket
	return &Repo{}, nil
}

// Repo holds data related to a Windows WSUS repository.
type Repo struct {
	updates []*update
}

// RepoName returns repository name.
func (r *Repo) RepoName() string {
	return RepoName
}

// RepoPath returns repository path.
func (r *Repo) RepoPath() string {
	return fmt.Sprintf("gs://%s/", gcsBucket)
}

type wsusUpdate struct {
	filename     string
	kbArticle    string
	defaultTitle string
}

// DiscoverRepo traverses the repository and looks for files that are related to WSUS packages.
func (r *Repo) DiscoverRepo() ([]hashr.Source, error) {
	updates, err := csvMapping()
	if err != nil {
		glog.Warningf("Could not get CSV mapping file: %v", err)
	}

	var token string
	for {
		resp, err := storageClient.Objects.List(gcsBucket).PageToken(token).Do()
		if err != nil {
			return nil, err
		}
		for _, obj := range resp.Items {
			_, filename := filepath.Split(obj.Name)
			ext := filepath.Ext(filename)
			sha1 := filename[0 : len(filename)-len(ext)]

			var id string
			var updateTitle string
			if val, exists := updates[sha1]; exists && val.filename != "" {
				id = val.filename
				updateTitle = val.defaultTitle
				if val.kbArticle != "" {
					updateTitle = fmt.Sprintf("KB%s %s", val.kbArticle, updateTitle)
				}
			} else {
				id = sha1
			}

			if strings.EqualFold(ext, ".cab") {
				r.updates = append(r.updates, &update{id: id, remotePath: obj.Name, md5hash: obj.Md5Hash, format: cabArchive, updateTitle: updateTitle})
			} else if strings.EqualFold(ext, ".exe") {
				r.updates = append(r.updates, &update{id: id, remotePath: obj.Name, md5hash: obj.Md5Hash, format: exe, updateTitle: updateTitle})
			}
		}
		token = resp.NextPageToken
		if token == "" {
			break
		}
	}

	var sources []hashr.Source
	for _, update := range r.updates {
		sources = append(sources, update)
	}

	return sources, nil
}

func csvMapping() (map[string]*wsusUpdate, error) {
	resp, err := storageClient.Objects.Get(gcsBucket, "export.csv").Download()
	if err != nil {
		return nil, fmt.Errorf("mapping file (%s) is not present", filepath.Join(gcsBucket, "export.csv"))
	}
	defer resp.Body.Close()

	var bodyBytes []byte
	if resp.StatusCode == http.StatusOK {
		bodyBytes, err = io.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
	}

	csvReader := csv.NewReader(bytes.NewReader(bodyBytes))
	csvReader.Comma = ';'
	rows, err := csvReader.ReadAll()
	if err != nil {
		return nil, fmt.Errorf("unable to parse CSV mapping file: %v", err)
	}

	updates := make(map[string]*wsusUpdate)

	for _, row := range rows[1:] {
		if val, exists := updates[row[0]]; exists {
			val.defaultTitle = fmt.Sprintf("%s, %s", val.defaultTitle, row[3])
		} else {
			updates[row[0]] = &wsusUpdate{filename: row[1], kbArticle: row[2], defaultTitle: row[3]}
		}
	}

	return updates, nil
}
<commit_msg>Change how quick hash is calculated for WSUS<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package wsus implements WSUS repository importer.
package wsus

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/csv"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"github.com/google/hashr/core/hashr"
	"github.com/google/hashr/importers/common"

	"google.golang.org/api/storage/v1"
)

type updateFormat int

const (
	// RepoName contains the repository name.
	RepoName              = "wsus"
	path7z                = "/usr/bin/7z"
	exe      updateFormat = iota
	cabArchive
)

var (
	storageClient = &storage.Service{}
	gcsBucket     string
)

type update struct {
	id              string
	remotePath      string
	md5hash         string
	quickSha256hash string
	localPath       string
	tempDir         string
	format          updateFormat
	updateTitle     string
}

// Preprocess extracts the contents of Windows Update file.
func (u *update) Preprocess() (string, error) {
	if err := u.download(); err != nil {
		return "", err
	}

	extractionDir := filepath.Join(u.tempDir, "extracted")
	if err := os.Mkdir(extractionDir, 0755); err != nil {
		return "", fmt.Errorf("could not create target %s directory: %v", extractionDir, err)
	}

	switch u.format {
	case cabArchive, exe:
		if err := u.recursiveExtract(extractionDir); err != nil {
			return "", err
		}
	}

	return extractionDir, nil
}

// ID returns non-unique Windows Update ID.
func (u *update) ID() string {
	return u.id
}

// RepoName returns repository name.
func (u *update) RepoName() string {
	return RepoName
}

// RepoPath returns repository path.
func (u *update) RepoPath() string {
	return fmt.Sprintf("gs://%s/", gcsBucket)
}

// LocalPath returns local path to a Windows Update file.
func (u *update) LocalPath() string {
	return u.localPath
}

// RemotePath returns remote path to a Windows Update file stored in the GCS bucket.
func (u *update) RemotePath() string {
	return fmt.Sprintf("gs://%s/%s", gcsBucket, u.remotePath)
}

// Description provides additional description for a Windows Update file.
func (u *update) Description() string {
	return u.updateTitle
}

// QuickSHA256Hash calculates sha256 hash of a Windows Update file metadata.
func (u *update) QuickSHA256Hash() (string, error) {
	if u.quickSha256hash != "" {
		return u.quickSha256hash, nil
	}

	u.quickSha256hash = fmt.Sprintf("%x", sha256.Sum256([]byte(u.md5hash)))

	return u.quickSha256hash, nil
}

func (u *update) download() error {
	_, file := filepath.Split(u.remotePath)

	resp, err := storageClient.Objects.Get(gcsBucket, u.remotePath).Download()
	if err != nil {
		glog.Exit(err)
		return err
	}
	defer resp.Body.Close()

	u.tempDir, err = common.LocalTempDir(u.ID())
	if err != nil {
		glog.Exit(err)
		return err
	}

	u.localPath = filepath.Join(u.tempDir, file)
	out, err := os.Create(u.localPath)
	if err != nil {
		glog.Exit(err)
		return fmt.Errorf("error while creating %s: %v", u.localPath, err)
	}

	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return fmt.Errorf("error while writing to %s: %v", u.localPath, err)
	}

	glog.Infof("Update %s successfully written to %s", u.remotePath, u.localPath)

	return nil
}

func (u *update) recursiveExtract(extractionDir string) error {
	if err := extract7z(u.localPath, extractionDir); err != nil {
		return err
	}

	var files []string
	if err := filepath.Walk(extractionDir, walk(&files)); err != nil {
		return err
	}

	if err := recursiveExtract(files); err != nil {
		return err
	}

	return nil
}

func extractFile(fileCmdOutput string) bool {
	toBeExtracted := []string{"compressed", "archive", "application/x-msi", "application/vnd.ms-msi"}
	for _, s := range toBeExtracted {
		if strings.Contains(fileCmdOutput, s) {
			return true
		}
	}

	return false
}

func extractExt(ext string) bool {
	toBeExtracted := []string{".cab", ".msi", ".7z", ".msu", ".msp"}
	for _, e := range toBeExtracted {
		if strings.EqualFold(e, ext) {
			return true
		}
	}
	return false
}

func recursiveExtract(files []string) error {
	for _, fullPath := range files {
		_, filename := filepath.Split(fullPath)
		extension := filepath.Ext(filename)

		var tryToExtract bool
		// Since the files come from MS we can trust the extension, but if there is none, we should
		// try to detect the mime type. The easiest and most reliable solution is to run file binary.
		if extension == "" {
			out, err := shellCommand("/usr/bin/file", "--mime-type", fullPath)
			if err != nil {
				return err
			}
			if extractFile(out) {
				tryToExtract = true
			}
		}

		if extractExt(extension) || tryToExtract {
			targetDir := fmt.Sprintf("%s_%s", fullPath, "extracted")
			if err := os.Mkdir(targetDir, 0755); err != nil {
				return fmt.Errorf("could not create target %s directory: %v", targetDir, err)
			}
			if err := extract7z(fullPath, targetDir); err != nil {
				glog.Errorf("error while extracting archive file: %v", err)
				continue
			}

			var extractedFiles []string
			if err := filepath.Walk(targetDir, walk(&extractedFiles)); err != nil {
				return err
			}

			if len(extractedFiles) > 0 {
				if err := recursiveExtract(extractedFiles); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

// This is mainly to enable testing.
var execute = func(name string, args ...string) *exec.Cmd {
	return exec.Command(name, args...)
}

func shellCommand(binary string, args ...string) (string, error) {
	cmd := execute(binary, args...)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	err := cmd.Run()
	if err != nil {
		return "", fmt.Errorf("error while executing %s: %v\nStdout: %v\nStderr: %v", binary, err, stdout.String(), stderr.String())
	}

	return stdout.String(), nil
}

func extract7z(filepath, targetDir string) error {
	_, err := shellCommand(path7z, "x", filepath, fmt.Sprintf("-o%s", targetDir))
	if err != nil {
		return fmt.Errorf("error while running 7z: %v", err)
	}

	return nil
}

func walk(files *[]string) filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			glog.Errorf("Could not open %s: %v", path, err)
			return nil
		}
		if info.IsDir() {
			return nil
		}

		*files = append(*files, path)
		return nil
	}
}

// NewRepo returns new instance of a WSUS repository.
func NewRepo(ctx context.Context, storageService *storage.Service, gcsWSUSBucket string) (*Repo, error) {
	storageClient = storageService
	gcsBucket = gcsWSUSBucket
	return &Repo{}, nil
}

// Repo holds data related to a Windows WSUS repository.
type Repo struct {
	updates []*update
}

// RepoName returns repository name.
func (r *Repo) RepoName() string {
	return RepoName
}

// RepoPath returns repository path.
func (r *Repo) RepoPath() string {
	return fmt.Sprintf("gs://%s/", gcsBucket)
}

type wsusUpdate struct {
	filename     string
	kbArticle    string
	defaultTitle string
}

// DiscoverRepo traverses the repository and looks for files that are related to WSUS packages.
func (r *Repo) DiscoverRepo() ([]hashr.Source, error) {
	updates, err := csvMapping()
	if err != nil {
		glog.Warningf("Could not get CSV mapping file: %v", err)
	}

	var token string
	for {
		resp, err := storageClient.Objects.List(gcsBucket).PageToken(token).Do()
		if err != nil {
			return nil, err
		}
		for _, obj := range resp.Items {
			_, filename := filepath.Split(obj.Name)
			ext := filepath.Ext(filename)
			sha1 := filename[0 : len(filename)-len(ext)]

			var id string
			var updateTitle string
			if val, exists := updates[sha1]; exists && val.filename != "" {
				id = val.filename
				updateTitle = val.defaultTitle
				if val.kbArticle != "" {
					updateTitle = fmt.Sprintf("KB%s %s", val.kbArticle, updateTitle)
				}
			} else {
				id = sha1
			}

			if strings.EqualFold(ext, ".cab") {
				r.updates = append(r.updates, &update{id: id, remotePath: obj.Name, md5hash: obj.Md5Hash, format: cabArchive, updateTitle: updateTitle})
			} else if strings.EqualFold(ext, ".exe") {
				r.updates = append(r.updates, &update{id: id, remotePath: obj.Name, md5hash: obj.Md5Hash, format: exe, updateTitle: updateTitle})
			}
		}
		token = resp.NextPageToken
		if token == "" {
			break
		}
	}

	var sources []hashr.Source
	for _, update := range r.updates {
		sources = append(sources, update)
	}

	return sources, nil
}

func csvMapping() (map[string]*wsusUpdate, error) {
	resp, err := storageClient.Objects.Get(gcsBucket, "export.csv").Download()
	if err != nil {
		return nil, fmt.Errorf("mapping file (%s) is not present", filepath.Join(gcsBucket, "export.csv"))
	}
	defer resp.Body.Close()

	var bodyBytes []byte
	if resp.StatusCode == http.StatusOK {
		bodyBytes, err = io.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
	}

	csvReader := csv.NewReader(bytes.NewReader(bodyBytes))
	csvReader.Comma = ';'
	rows, err := csvReader.ReadAll()
	if err != nil {
		return nil, fmt.Errorf("unable to parse CSV mapping file: %v", err)
	}

	updates := make(map[string]*wsusUpdate)

	for _, row := range rows[1:] {
		if val, exists := updates[row[0]]; exists {
			val.defaultTitle = fmt.Sprintf("%s, %s", val.defaultTitle, row[3])
		} else {
			updates[row[0]] = &wsusUpdate{filename: row[1], kbArticle: row[2], defaultTitle: row[3]}
		}
	}

	return updates, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import syscall "syscall"

// Support types and routines for OS library

func WriteString(fd int64, s string) (ret int64, err *Error);

export func StringToBytes(b *[]byte, s string) bool {
	if len(s) >= len(b) {
		return false
	}
	for i := 0; i < len(s); i++ {
		b[i] = s[i]
	}
	b[len(s)] = '\000';	// not necessary - memory is zeroed - but be explicit
	return true
}

// Errors are singleton structures. Use the Print()/String() methods to get their contents --
// it handles the nil (no error) case.

export type Error struct {
	s string
}

const NoError = "No Error"

func (e *Error) Print() {
	if e == nil {
		WriteString(2, NoError)
	} else {
		WriteString(2, e.s)
	}
}

func (e *Error) String() string {
	if e == nil {
		return NoError
	} else {
		return e.s
	}
}

var ErrorTab = new(map[int64] *Error);

func ErrnoToError(errno int64) *Error {
	if errno == 0 {
		return nil
	}
	err, ok := ErrorTab[errno]
	if ok {
		return err
	}
	e := new(Error);
	e.s = syscall.errstr(errno);
	ErrorTab[errno] = e;
	return e;
}

export var (
	ENONE = ErrnoToError(syscall.ENONE);
	EPERM = ErrnoToError(syscall.EPERM);
	ENOENT = ErrnoToError(syscall.ENOENT);
	ESRCH = ErrnoToError(syscall.ESRCH);
	EINTR = ErrnoToError(syscall.EINTR);
	EIO = ErrnoToError(syscall.EIO);
	ENXIO = ErrnoToError(syscall.ENXIO);
	E2BIG = ErrnoToError(syscall.E2BIG);
	ENOEXEC = ErrnoToError(syscall.ENOEXEC);
	EBADF = ErrnoToError(syscall.EBADF);
	ECHILD = ErrnoToError(syscall.ECHILD);
	EDEADLK = ErrnoToError(syscall.EDEADLK);
	ENOMEM = ErrnoToError(syscall.ENOMEM);
	EACCES = ErrnoToError(syscall.EACCES);
	EFAULT = ErrnoToError(syscall.EFAULT);
	ENOTBLK = ErrnoToError(syscall.ENOTBLK);
	EBUSY = ErrnoToError(syscall.EBUSY);
	EEXIST = ErrnoToError(syscall.EEXIST);
	EXDEV = ErrnoToError(syscall.EXDEV);
	ENODEV = ErrnoToError(syscall.ENODEV);
	ENOTDIR = ErrnoToError(syscall.ENOTDIR);
	EISDIR = ErrnoToError(syscall.EISDIR);
	EINVAL = ErrnoToError(syscall.EINVAL);
	ENFILE = ErrnoToError(syscall.ENFILE);
	EMFILE = ErrnoToError(syscall.EMFILE);
	ENOTTY = ErrnoToError(syscall.ENOTTY);
	ETXTBSY = ErrnoToError(syscall.ETXTBSY);
	EFBIG = ErrnoToError(syscall.EFBIG);
	ENOSPC = ErrnoToError(syscall.ENOSPC);
	ESPIPE = ErrnoToError(syscall.ESPIPE);
	EROFS = ErrnoToError(syscall.EROFS);
	EMLINK = ErrnoToError(syscall.EMLINK);
	EPIPE = ErrnoToError(syscall.EPIPE);
	EDOM = ErrnoToError(syscall.EDOM);
	ERANGE = ErrnoToError(syscall.ERANGE);
	EAGAIN = ErrnoToError(syscall.EAGAIN);
)

export func Open(name string, mode int64, flags int64) (ret int64, err *Error) {
	var buf [512]byte;
	if !StringToBytes(&buf, name) {
		return -1, ErrnoToError(syscall.ENAMETOOLONG)
	}
	r, e := syscall.open(&buf[0], mode, flags);
	return r, ErrnoToError(e)
}

export func Close(fd int64) (ret int64, err *Error) {
	r, e := syscall.close(fd);
	return r, ErrnoToError(e)
}

export func Read(fd int64, b *[]byte) (ret int64, err *Error) {
	r, e := syscall.read(fd, &b[0], int64(len(b)));
	return r, ErrnoToError(e)
}

export func Write(fd int64, b *[]byte) (ret int64, err *Error) {
	r, e := syscall.write(fd, &b[0], int64(len(b)));
	return r, ErrnoToError(e)
}

export func WriteString(fd int64, s string) (ret int64, err *Error) {
	b := new([]byte, len(s)+1);
	if !StringToBytes(b, s) {
		return -1, ErrnoToError(syscall.ENAMETOOLONG)
	}
	r, e := syscall.write(fd, &b[0], int64(len(s)));
	return r, ErrnoToError(e)
}

<commit_msg>make FD a struct with Read, Write, etc. as methods<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import syscall "syscall"

// Support types and routines for OS library

// FDs are wrappers for file descriptors
export type FD struct {
	fd int64
}

// Errors are singleton structures. Use the Print()/String() methods to get their contents --
// they handle the nil (no error) case.
export type Error struct {
	s string
}

export func NewFD(fd int64) *FD {
	if fd < 0 {
		return nil
	}
	n := new(FD);	// TODO(r): how about return &FD{fd} ?
	n.fd = fd;
	return n;
}

export var (
	Stdin = NewFD(0);
	Stdout = NewFD(1);
	Stderr = NewFD(2);
)

export func StringToBytes(b *[]byte, s string) bool {
	if len(s) >= len(b) {
		return false
	}
	for i := 0; i < len(s); i++ {
		b[i] = s[i]
	}
	b[len(s)] = '\000';	// not necessary - memory is zeroed - but be explicit
	return true
}

var ErrorTab = new(map[int64] *Error);

func ErrnoToError(errno int64) *Error {
	if errno == 0 {
		return nil
	}
	err, ok := ErrorTab[errno]
	if ok {
		return err
	}
	e := new(Error);
	e.s = syscall.errstr(errno);
	ErrorTab[errno] = e;
	return e;
}

export var (
	ENONE = ErrnoToError(syscall.ENONE);
	EPERM = ErrnoToError(syscall.EPERM);
	ENOENT = ErrnoToError(syscall.ENOENT);
	ESRCH = ErrnoToError(syscall.ESRCH);
	EINTR = ErrnoToError(syscall.EINTR);
	EIO = ErrnoToError(syscall.EIO);
	ENXIO = ErrnoToError(syscall.ENXIO);
	E2BIG = ErrnoToError(syscall.E2BIG);
	ENOEXEC = ErrnoToError(syscall.ENOEXEC);
	EBADF = ErrnoToError(syscall.EBADF);
	ECHILD = ErrnoToError(syscall.ECHILD);
	EDEADLK = ErrnoToError(syscall.EDEADLK);
	ENOMEM = ErrnoToError(syscall.ENOMEM);
	EACCES = ErrnoToError(syscall.EACCES);
	EFAULT = ErrnoToError(syscall.EFAULT);
	ENOTBLK = ErrnoToError(syscall.ENOTBLK);
	EBUSY = ErrnoToError(syscall.EBUSY);
	EEXIST = ErrnoToError(syscall.EEXIST);
	EXDEV = ErrnoToError(syscall.EXDEV);
	ENODEV = ErrnoToError(syscall.ENODEV);
	ENOTDIR = ErrnoToError(syscall.ENOTDIR);
	EISDIR = ErrnoToError(syscall.EISDIR);
	EINVAL = ErrnoToError(syscall.EINVAL);
	ENFILE = ErrnoToError(syscall.ENFILE);
	EMFILE = ErrnoToError(syscall.EMFILE);
	ENOTTY = ErrnoToError(syscall.ENOTTY);
	ETXTBSY = ErrnoToError(syscall.ETXTBSY);
	EFBIG = ErrnoToError(syscall.EFBIG);
	ENOSPC = ErrnoToError(syscall.ENOSPC);
	ESPIPE = ErrnoToError(syscall.ESPIPE);
	EROFS = ErrnoToError(syscall.EROFS);
	EMLINK = ErrnoToError(syscall.EMLINK);
	EPIPE = ErrnoToError(syscall.EPIPE);
	EDOM = ErrnoToError(syscall.EDOM);
	ERANGE = ErrnoToError(syscall.ERANGE);
	EAGAIN = ErrnoToError(syscall.EAGAIN);
)

export func Open(name string, mode int64, flags int64) (fd *FD, err *Error) {
	var buf [512]byte;
	if !StringToBytes(&buf, name) {
		return nil, EINVAL
	}
	r, e := syscall.open(&buf[0], mode, flags);
	return NewFD(r), ErrnoToError(e)
}

func (fd *FD) Close() *Error {
	if fd == nil {
		return EINVAL
	}
	r, e := syscall.close(fd.fd);
	fd.fd = -1;  // so it can't be closed again
	return ErrnoToError(e)
}

func (fd *FD) Read(b *[]byte) (ret int64, err *Error) {
	if fd == nil {
		return -1, EINVAL
	}
	r, e := syscall.read(fd.fd, &b[0], int64(len(b)));
	return r, ErrnoToError(e)
}

func (fd *FD) Write(b *[]byte) (ret int64, err *Error) {
	if fd == nil {
		return -1, EINVAL
	}
	r, e := syscall.write(fd.fd, &b[0], int64(len(b)));
	return r, ErrnoToError(e)
}

func (fd *FD) WriteString(s string) (ret int64, err *Error) {
	if fd == nil {
		return -1, EINVAL
	}
	b := new([]byte, len(s)+1);
	if !StringToBytes(b, s) {
		return -1, EINVAL
	}
	r, e := syscall.write(fd.fd, &b[0], int64(len(s)));
	return r, ErrnoToError(e)
}

const NoError = "No Error"

func (e *Error) String() string {
	if e == nil {
		return NoError
	} else {
		return e.s
	}
}

func (e *Error) Print() {
	Stderr.WriteString(e.String())
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2020 Red Hat, Inc.
 *
 */

package libvmi

import (
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"

	v1 "kubevirt.io/api/core/v1"
)

// WithContainerImage specifies the name of the container image to be used.
func WithContainerImage(name string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		diskName := "disk0"
		addDisk(vmi, newDisk(diskName, v1.DiskBusVirtio))
		addVolume(vmi, newContainerVolume(diskName, name))
	}
}

// WithPersistentVolumeClaim specifies the name of the PersistentVolumeClaim to be used.
func WithPersistentVolumeClaim(diskName, pvcName string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addDisk(vmi, newDisk(diskName, v1.DiskBusVirtio))
		addVolume(vmi, newPersistentVolumeClaimVolume(diskName, pvcName))
	}
}

// WithDataVolume specifies the name of the DataVolume to be used.
func WithDataVolume(diskName, pvcName string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addDisk(vmi, newDisk(diskName, v1.DiskBusVirtio))
		addVolume(vmi, newDataVolume(diskName, pvcName))
	}
}

// WithEmptyDisk specifies the name of the EmptyDisk to be used.
func WithEmptyDisk(diskName string, bus v1.DiskBus, capacity resource.Quantity) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addDisk(vmi, newDisk(diskName, bus))
		addVolume(vmi, newEmptyDisk(diskName, capacity))
	}
}

// WithCDRom specifies a CDRom drive backed by a PVC to be used.
func WithCDRom(cdRomName string, bus v1.DiskBus, claimName string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addDisk(vmi, newCDRom(cdRomName, bus))
		addVolume(vmi, newPersistentVolumeClaimVolume(cdRomName, claimName))
	}
}

func addDisk(vmi *v1.VirtualMachineInstance, disk v1.Disk) {
	if !diskExists(vmi, disk) {
		vmi.Spec.Domain.Devices.Disks = append(vmi.Spec.Domain.Devices.Disks, disk)
	}
}

func addVolume(vmi *v1.VirtualMachineInstance, volume v1.Volume) {
	if !volumeExists(vmi, volume) {
		vmi.Spec.Volumes = append(vmi.Spec.Volumes, volume)
	}
}

func getVolume(vmi *v1.VirtualMachineInstance, name string) *v1.Volume {
	for i := range vmi.Spec.Volumes {
		if vmi.Spec.Volumes[i].Name == name {
			return &vmi.Spec.Volumes[i]
		}
	}
	return nil
}

func diskExists(vmi *v1.VirtualMachineInstance, disk v1.Disk) bool {
	for _, d := range vmi.Spec.Domain.Devices.Disks {
		if d.Name == disk.Name {
			return true
		}
	}
	return false
}

func volumeExists(vmi *v1.VirtualMachineInstance, volume v1.Volume) bool {
	for _, v := range vmi.Spec.Volumes {
		if v.Name == volume.Name {
			return true
		}
	}
	return false
}

func newDisk(name string, bus v1.DiskBus) v1.Disk {
	return v1.Disk{
		Name: name,
		DiskDevice: v1.DiskDevice{
			Disk: &v1.DiskTarget{
				Bus: bus,
			},
		},
	}
}

func newCDRom(name string, bus v1.DiskBus) v1.Disk {
	return v1.Disk{
		Name: name,
		DiskDevice: v1.DiskDevice{
			CDRom: &v1.CDRomTarget{
				Bus: bus,
			},
		},
	}
}

func newVolume(name string) v1.Volume {
	return v1.Volume{Name: name}
}

func newContainerVolume(name, image string) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			ContainerDisk: &v1.ContainerDiskSource{
				Image: image,
			},
		},
	}
}

func newPersistentVolumeClaimVolume(name, claimName string) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
				PersistentVolumeClaimVolumeSource: k8sv1.PersistentVolumeClaimVolumeSource{
					ClaimName: claimName,
				}},
		},
	}
}

func newDataVolume(name, dataVolumeName string) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			DataVolume: &v1.DataVolumeSource{
				Name: dataVolumeName,
			},
		},
	}
}

func newEmptyDisk(name string, capacity resource.Quantity) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			EmptyDisk: &v1.EmptyDiskSource{
				Capacity: capacity,
			},
		},
	}
}
<commit_msg>tests/libvmi: Add WithFilesystemPVC and WithFilesystemDV<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2020 Red Hat, Inc.
 *
 */

package libvmi

import (
	"fmt"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"

	v1 "kubevirt.io/api/core/v1"
)

// WithContainerImage specifies the name of the container image to be used.
func WithContainerImage(name string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		diskName := "disk0"
		addDisk(vmi, newDisk(diskName, v1.DiskBusVirtio))
		addVolume(vmi, newContainerVolume(diskName, name))
	}
}

// WithPersistentVolumeClaim specifies the name of the PersistentVolumeClaim to be used.
func WithPersistentVolumeClaim(diskName, pvcName string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addDisk(vmi, newDisk(diskName, v1.DiskBusVirtio))
		addVolume(vmi, newPersistentVolumeClaimVolume(diskName, pvcName))
	}
}

// WithDataVolume specifies the name of the DataVolume to be used.
func WithDataVolume(diskName, pvcName string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addDisk(vmi, newDisk(diskName, v1.DiskBusVirtio))
		addVolume(vmi, newDataVolume(diskName, pvcName))
	}
}

// WithEmptyDisk specifies the name of the EmptyDisk to be used.
func WithEmptyDisk(diskName string, bus v1.DiskBus, capacity resource.Quantity) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addDisk(vmi, newDisk(diskName, bus))
		addVolume(vmi, newEmptyDisk(diskName, capacity))
	}
}

// WithCDRom specifies a CDRom drive backed by a PVC to be used.
func WithCDRom(cdRomName string, bus v1.DiskBus, claimName string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addDisk(vmi, newCDRom(cdRomName, bus))
		addVolume(vmi, newPersistentVolumeClaimVolume(cdRomName, claimName))
	}
}

// WithFilesystemPVC specifies a filesystem backed by a PVC to be used.
func WithFilesystemPVC(claimName string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addFilesystem(vmi, newFilesystem(claimName))
		addVolume(vmi, newPersistentVolumeClaimVolume(claimName, claimName))
	}
}

// WithFilesystemDV specifies a filesystem backed by a DV to be used.
func WithFilesystemDV(dataVolumeName string) Option {
	return func(vmi *v1.VirtualMachineInstance) {
		addFilesystem(vmi, newFilesystem(dataVolumeName))
		addVolume(vmi, newDataVolume(dataVolumeName, dataVolumeName))
	}
}

func addDisk(vmi *v1.VirtualMachineInstance, disk v1.Disk) {
	if !diskExists(vmi, disk) {
		vmi.Spec.Domain.Devices.Disks = append(vmi.Spec.Domain.Devices.Disks, disk)
	}
}

func addVolume(vmi *v1.VirtualMachineInstance, volume v1.Volume) {
	if !volumeExists(vmi, volume) {
		vmi.Spec.Volumes = append(vmi.Spec.Volumes, volume)
	}
}

func addFilesystem(vmi *v1.VirtualMachineInstance, filesystem v1.Filesystem) {
	if filesystemExists(vmi, filesystem) {
		panic(fmt.Errorf("filesystem %s already exists", filesystem.Name))
	}

	vmi.Spec.Domain.Devices.Filesystems = append(vmi.Spec.Domain.Devices.Filesystems, filesystem)
}

func getVolume(vmi *v1.VirtualMachineInstance, name string) *v1.Volume {
	for i := range vmi.Spec.Volumes {
		if vmi.Spec.Volumes[i].Name == name {
			return &vmi.Spec.Volumes[i]
		}
	}
	return nil
}

func diskExists(vmi *v1.VirtualMachineInstance, disk v1.Disk) bool {
	for _, d := range vmi.Spec.Domain.Devices.Disks {
		if d.Name == disk.Name {
			return true
		}
	}
	return false
}

func volumeExists(vmi *v1.VirtualMachineInstance, volume v1.Volume) bool {
	for _, v := range vmi.Spec.Volumes {
		if v.Name == volume.Name {
			return true
		}
	}
	return false
}

func filesystemExists(vmi *v1.VirtualMachineInstance, filesystem v1.Filesystem) bool {
	for _, f := range vmi.Spec.Domain.Devices.Filesystems {
		if f.Name == filesystem.Name {
			return true
		}
	}
	return false
}

func newDisk(name string, bus v1.DiskBus) v1.Disk {
	return v1.Disk{
		Name: name,
		DiskDevice: v1.DiskDevice{
			Disk: &v1.DiskTarget{
				Bus: bus,
			},
		},
	}
}

func newCDRom(name string, bus v1.DiskBus) v1.Disk {
	return v1.Disk{
		Name: name,
		DiskDevice: v1.DiskDevice{
			CDRom: &v1.CDRomTarget{
				Bus: bus,
			},
		},
	}
}

func newFilesystem(name string) v1.Filesystem {
	return v1.Filesystem{
		Name:     name,
		Virtiofs: &v1.FilesystemVirtiofs{},
	}
}

func newVolume(name string) v1.Volume {
	return v1.Volume{Name: name}
}

func newContainerVolume(name, image string) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			ContainerDisk: &v1.ContainerDiskSource{
				Image: image,
			},
		},
	}
}

func newPersistentVolumeClaimVolume(name, claimName string) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
				PersistentVolumeClaimVolumeSource: k8sv1.PersistentVolumeClaimVolumeSource{
					ClaimName: claimName,
				}},
		},
	}
}

func newDataVolume(name, dataVolumeName string) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			DataVolume: &v1.DataVolumeSource{
				Name: dataVolumeName,
			},
		},
	}
}

func newEmptyDisk(name string, capacity resource.Quantity) v1.Volume {
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			EmptyDisk: &v1.EmptyDiskSource{
				Capacity: capacity,
			},
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"expvar"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/mat/besticon/besticon"
	"github.com/mat/besticon/besticon/iconserver/assets"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("index")
	renderHTMLTemplate(w, 200, indexHTML, nil)
}

func iconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		http.Redirect(w, r, "/", 302)
		return
	}

	icons, e := fetchIcons(url)
	switch {
	case e != nil:
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: e})
	case len(icons) <= 0:
		errNoIcons := errors.New("this poor site has no icons at all :-(")
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: errNoIcons})
	default:
		renderHTMLTemplate(w, 200, iconsHTML, pageInfo{Icons: icons, URL: url})
	}
}

const urlParam = "url"
const bestParam = "i_am_feeling_lucky"

func apiHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		errMissingURL := errors.New("need url query parameter")
		writeAPIError(w, 400, errMissingURL)
		return
	}

	bestMode := r.FormValue(bestParam) == "yes"
	if bestMode {
		icon, e := fetchBestIcon(url)
		if e != nil {
			writeAPIError(w, 404, e)
			return
		}

		http.Redirect(w, r, icon.URL, 302)
	} else {
		icons, e := fetchIcons(url)
		if e != nil {
			writeAPIError(w, 404, e)
			return
		}

		writeAPIIcons(w, url, icons)
	}
}

func fetchIcons(url string) ([]besticon.Icon, error) {
	fetchCount.Add(1)
	icons, err := besticon.FetchIcons(url)
	if err != nil {
		fetchErrors.Add(1)
	}
	return icons, err
}

func fetchBestIcon(url string) (*besticon.Icon, error) {
	fetchCount.Add(1)
	icon, err := besticon.FetchBestIcon(url)
	if err != nil {
		fetchErrors.Add(1)
	}
	return icon, err
}

func statsHandler(w http.ResponseWriter, r *http.Request) {
	stats := stats{}
	stats.Stats = append(stats.Stats, pair{"Current Time", time.Now().String()})
	stats.Stats = append(stats.Stats, pair{"Last Deploy", parseUnixTimeStamp(os.Getenv("DEPLOYED_AT")).String()})
	stats.Stats = append(stats.Stats, pair{"Deployed Git Revision", os.Getenv("GIT_REVISION")})
	renderHTMLTemplate(w, 200, statsHTML, stats)
}

func parseUnixTimeStamp(s string) time.Time {
	ts, err := strconv.Atoi(s)
	if err != nil {
		return time.Unix(0, 0)
	}

	return time.Unix(int64(ts), 0)
}

func writeAPIError(w http.ResponseWriter, httpStatus int, e error) {
	data := struct {
		Error string `json:"error"`
	}{
		e.Error(),
	}
	renderJSONResponse(w, httpStatus, data)
}

func writeAPIIcons(w http.ResponseWriter, url string, icons []besticon.Icon) {
	data := struct {
		URL   string          `json:"url"`
		Icons []besticon.Icon `json:"icons"`
	}{
		url,
		icons,
	}
	renderJSONResponse(w, 200, data)
}

func renderJSONResponse(w http.ResponseWriter, httpStatus int, data interface{}) {
	w.Header().Add("Content-Type", "application/json")
	w.WriteHeader(httpStatus)
	enc := json.NewEncoder(w)
	enc.Encode(data)
}

type pageInfo struct {
	URL   string
	Icons []besticon.Icon
	Error error
}

type stats struct {
	Stats []pair
}
type pair struct {
	Name, Value string
}

func (pi pageInfo) Host() string {
	u := pi.URL
	url, _ := url.Parse(u)
	if url != nil && url.Host != "" {
		return url.Host
	}
	return pi.URL
}

func (pi pageInfo) Best() string {
	if len(pi.Icons) > 0 {
		best := pi.Icons[0]
		return best.URL
	}
	return ""
}

func renderHTMLTemplate(w http.ResponseWriter, httpStatus int, templ *template.Template, data interface{}) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(httpStatus)

	err := templ.Execute(w, data)
	if err != nil {
		err = fmt.Errorf("server: could not generate output: %s", err)
		logger.Print(err)
		w.Write([]byte(err.Error()))
	}
}

func startServer(port int) {
	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/icons", iconsHandler)
	http.HandleFunc("/api/icons", apiHandler)
	http.HandleFunc("/stats", statsHandler)

	serveAsset("/pure-0.5.0-min.css", "besticon/iconserver/assets/pure-0.5.0-min.css")
	serveAsset("/grids-responsive-0.5.0-min.css", "besticon/iconserver/assets/grids-responsive-0.5.0-min.css")
	serveAsset("/main-min.css", "besticon/iconserver/assets/main-min.css")

	serveAsset("/icon.svg", "besticon/iconserver/assets/icon.svg")
	serveAsset("/favicon.ico", "besticon/iconserver/assets/favicon.ico")
	serveAsset("/apple-touch-icon.png", "besticon/iconserver/assets/apple-touch-icon.png")

	logger.Print("Starting server on port ", port, "...")
	e := http.ListenAndServe(":"+strconv.Itoa(port), NewLoggingMux())
	if e != nil {
		fmt.Printf("cannot start server: %s\n", e)
	}
}

func serveAsset(path string, assetPath string) {
	http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
		assetInfo, err := assets.AssetInfo(assetPath)
		if err != nil {
			panic(err)
		}

		http.ServeContent(w, r, assetInfo.Name(), assetInfo.ModTime(), bytes.NewReader(assets.MustAsset(assetPath)))
	})
}

func main() {
	port := flag.Int("port", 0, "Port in server mode")
	flag.Parse()

	if *port > 0 {
		startServer(*port)
	} else {
		flag.PrintDefaults()
	}
}

func init() {
	indexHTML = templateFromAsset("besticon/iconserver/assets/index.html", "index.html")
	iconsHTML = templateFromAsset("besticon/iconserver/assets/icons.html", "icons.html")
	statsHTML = templateFromAsset("besticon/iconserver/assets/stats.html", "stats.html")
}

func templateFromAsset(assetPath, templateName string) *template.Template {
	bytes := assets.MustAsset(assetPath)
	return template.Must(template.New(templateName).Funcs(funcMap).Parse(string(bytes)))
}

var indexHTML *template.Template
var iconsHTML *template.Template
var statsHTML *template.Template

var logger = log.New(os.Stdout, "besticon: ", log.LstdFlags|log.Ltime)

type loggingWriter struct {
	http.ResponseWriter
	status int
	length int
}

func (w *loggingWriter) WriteHeader(status int) {
	w.status = status
	w.ResponseWriter.WriteHeader(status)
}

func (w *loggingWriter) Write(b []byte) (int, error) {
	if w.status == 0 {
		w.status = 200
	}

	bytesWritten, err := w.ResponseWriter.Write(b)
	if err == nil {
		w.length += bytesWritten
	}
	return bytesWritten, err
}

func NewLoggingMux() http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		start := time.Now()
		writer := loggingWriter{w, 0, 0}
		http.DefaultServeMux.ServeHTTP(&writer, req)
		end := time.Now()
		duration := end.Sub(start)

		logger.Printf("%s %s %d \"%s\" %s %v %d",
			req.Method,
			req.URL,
			writer.status,
			req.UserAgent(),
			req.Referer(),
			duration,
			writer.length,
		)
	}
}

var (
	fetchCount  = expvar.NewInt("fetchCount")
	fetchErrors = expvar.NewInt("fetchErrors")
)

var funcMap = template.FuncMap{
	"GoogleAnalyticsID": GoogleAnalyticsID,
}

func GoogleAnalyticsID() string {
	return os.Getenv("ICONS_ANALYTICS_ID")
}
<commit_msg>Emit current time with expvar.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"expvar"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/mat/besticon/besticon"
	"github.com/mat/besticon/besticon/iconserver/assets"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("index")
	renderHTMLTemplate(w, 200, indexHTML, nil)
}

func iconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		http.Redirect(w, r, "/", 302)
		return
	}

	icons, e := fetchIcons(url)
	switch {
	case e != nil:
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: e})
	case len(icons) <= 0:
		errNoIcons := errors.New("this poor site has no icons at all :-(")
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: errNoIcons})
	default:
		renderHTMLTemplate(w, 200, iconsHTML, pageInfo{Icons: icons, URL: url})
	}
}

const urlParam = "url"
const bestParam = "i_am_feeling_lucky"

func apiHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		errMissingURL := errors.New("need url query parameter")
		writeAPIError(w, 400, errMissingURL)
		return
	}

	bestMode := r.FormValue(bestParam) == "yes"
	if bestMode {
		icon, e := fetchBestIcon(url)
		if e != nil {
			writeAPIError(w, 404, e)
			return
		}

		http.Redirect(w, r, icon.URL, 302)
	} else {
		icons, e := fetchIcons(url)
		if e != nil {
			writeAPIError(w, 404, e)
			return
		}

		writeAPIIcons(w, url, icons)
	}
}

func fetchIcons(url string) ([]besticon.Icon, error) {
	fetchCount.Add(1)
	icons, err := besticon.FetchIcons(url)
	if err != nil {
		fetchErrors.Add(1)
	}
	return icons, err
}

func fetchBestIcon(url string) (*besticon.Icon, error) {
	fetchCount.Add(1)
	icon, err := besticon.FetchBestIcon(url)
	if err != nil {
		fetchErrors.Add(1)
	}
	return icon, err
}

func statsHandler(w http.ResponseWriter, r *http.Request) {
	stats := stats{}
	stats.Stats = append(stats.Stats, pair{"Current Time", time.Now().String()})
	stats.Stats = append(stats.Stats, pair{"Last Deploy", parseUnixTimeStamp(os.Getenv("DEPLOYED_AT")).String()})
	stats.Stats = append(stats.Stats, pair{"Deployed Git Revision", os.Getenv("GIT_REVISION")})
	renderHTMLTemplate(w, 200, statsHTML, stats)
}

func parseUnixTimeStamp(s string) time.Time {
	ts, err := strconv.Atoi(s)
	if err != nil {
		return time.Unix(0, 0)
	}

	return time.Unix(int64(ts), 0)
}

func writeAPIError(w http.ResponseWriter, httpStatus int, e error) {
	data := struct {
		Error string `json:"error"`
	}{
		e.Error(),
	}
	renderJSONResponse(w, httpStatus, data)
}

func writeAPIIcons(w http.ResponseWriter, url string, icons []besticon.Icon) {
	data := struct {
		URL   string          `json:"url"`
		Icons []besticon.Icon `json:"icons"`
	}{
		url,
		icons,
	}
	renderJSONResponse(w, 200, data)
}

func renderJSONResponse(w http.ResponseWriter, httpStatus int, data interface{}) {
	w.Header().Add("Content-Type", "application/json")
	w.WriteHeader(httpStatus)
	enc := json.NewEncoder(w)
	enc.Encode(data)
}

type pageInfo struct {
	URL   string
	Icons []besticon.Icon
	Error error
}

type stats struct {
	Stats []pair
}
type pair struct {
	Name, Value string
}

func (pi pageInfo) Host() string {
	u := pi.URL
	url, _ := url.Parse(u)
	if url != nil && url.Host != "" {
		return url.Host
	}
	return pi.URL
}

func (pi pageInfo) Best() string {
	if len(pi.Icons) > 0 {
		best := pi.Icons[0]
		return best.URL
	}
	return ""
}

func renderHTMLTemplate(w http.ResponseWriter, httpStatus int, templ *template.Template, data interface{}) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(httpStatus)

	err := templ.Execute(w, data)
	if err != nil {
		err = fmt.Errorf("server: could not generate output: %s", err)
		logger.Print(err)
		w.Write([]byte(err.Error()))
	}
}

func startServer(port int) {
	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/icons", iconsHandler)
	http.HandleFunc("/api/icons", apiHandler)
	http.HandleFunc("/stats", statsHandler)

	serveAsset("/pure-0.5.0-min.css", "besticon/iconserver/assets/pure-0.5.0-min.css")
	serveAsset("/grids-responsive-0.5.0-min.css", "besticon/iconserver/assets/grids-responsive-0.5.0-min.css")
	serveAsset("/main-min.css", "besticon/iconserver/assets/main-min.css")

	serveAsset("/icon.svg", "besticon/iconserver/assets/icon.svg")
	serveAsset("/favicon.ico", "besticon/iconserver/assets/favicon.ico")
	serveAsset("/apple-touch-icon.png", "besticon/iconserver/assets/apple-touch-icon.png")

	logger.Print("Starting server on port ", port, "...")
	e := http.ListenAndServe(":"+strconv.Itoa(port), NewLoggingMux())
	if e != nil {
		fmt.Printf("cannot start server: %s\n", e)
	}
}

func serveAsset(path string, assetPath string) {
	http.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
		assetInfo, err := assets.AssetInfo(assetPath)
		if err != nil {
			panic(err)
		}

		http.ServeContent(w, r, assetInfo.Name(), assetInfo.ModTime(), bytes.NewReader(assets.MustAsset(assetPath)))
	})
}

func main() {
	port := flag.Int("port", 0, "Port in server mode")
	flag.Parse()

	if *port > 0 {
		startServer(*port)
	} else {
		flag.PrintDefaults()
	}
}

func init() {
	indexHTML = templateFromAsset("besticon/iconserver/assets/index.html", "index.html")
	iconsHTML = templateFromAsset("besticon/iconserver/assets/icons.html", "icons.html")
	statsHTML = templateFromAsset("besticon/iconserver/assets/stats.html", "stats.html")
}

func templateFromAsset(assetPath, templateName string) *template.Template {
	bytes := assets.MustAsset(assetPath)
	return template.Must(template.New(templateName).Funcs(funcMap).Parse(string(bytes)))
}

var indexHTML *template.Template
var iconsHTML *template.Template
var statsHTML *template.Template

var logger = log.New(os.Stdout, "besticon: ", log.LstdFlags|log.Ltime)

type loggingWriter struct {
	http.ResponseWriter
	status int
	length int
}

func (w *loggingWriter) WriteHeader(status int) {
	w.status = status
	w.ResponseWriter.WriteHeader(status)
}

func (w *loggingWriter) Write(b []byte) (int, error) {
	if w.status == 0 {
		w.status = 200
	}

	bytesWritten, err := w.ResponseWriter.Write(b)
	if err == nil {
		w.length += bytesWritten
	}
	return bytesWritten, err
}

func NewLoggingMux() http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		start := time.Now()
		writer := loggingWriter{w, 0, 0}
		http.DefaultServeMux.ServeHTTP(&writer, req)
		end := time.Now()
		duration := end.Sub(start)

		logger.Printf("%s %s %d \"%s\" %s %v %d",
			req.Method,
			req.URL,
			writer.status,
			req.UserAgent(),
			req.Referer(),
			duration,
			writer.length,
		)
	}
}

func init() {
	expvar.NewString("timeStartup").Set(time.Now().String())
	expvar.Publish("timeCurrent", expvar.Func(func() interface{} { return time.Now() }))
}

var (
	fetchCount  = expvar.NewInt("fetchCount")
	fetchErrors = expvar.NewInt("fetchErrors")
)

var funcMap = template.FuncMap{
	"GoogleAnalyticsID": GoogleAnalyticsID,
}

func GoogleAnalyticsID() string {
	return os.Getenv("ICONS_ANALYTICS_ID")
}
<|endoftext|>


<commit_before>package filepathfilter

import (
	"fmt"
	"path/filepath"
	"regexp"
	"strings"
)

type Pattern interface {
	// HasPrefix returns whether the receiving Pattern will match a fullpath
	// that contains the prefix "prefix".
	//
	// For instance, if the receiving pattern were to match 'a/b/c.txt',
	// HasPrefix() will return true for:
	//
	//   - 'a', and 'a/'
	//   - 'a/b', and 'a/b/'
	HasPrefix(prefix string) bool

	Match(filename string) bool
	// String returns a string representation (see: regular expressions) of
	// the underlying pattern used to match filenames against this Pattern.
	String() string
}

type Filter struct {
	include []Pattern
	exclude []Pattern
}

func NewFromPatterns(include, exclude []Pattern) *Filter {
	return &Filter{include: include, exclude: exclude}
}

func New(include, exclude []string) *Filter {
	return NewFromPatterns(convertToPatterns(include), convertToPatterns(exclude))
}

// Include returns the result of calling String() on each Pattern in the
// include set of this *Filter.
func (f *Filter) Include() []string { return patternsToStrings(f.include...) }

// Exclude returns the result of calling String() on each Pattern in the
// exclude set of this *Filter.
func (f *Filter) Exclude() []string { return patternsToStrings(f.exclude...) }

// patternsToStrings maps the given set of Pattern's to a string slice by
// calling String() on each pattern.
func patternsToStrings(ps ...Pattern) []string {
	s := make([]string, 0, len(ps))
	for _, p := range ps {
		s = append(s, p.String())
	}

	return s
}

func (f *Filter) Allows(filename string) bool {
	_, allowed := f.AllowsPattern(filename)
	return allowed
}

// HasPrefix returns whether the given prefix "prefix" is a prefix for all
// included Patterns, and not a prefix for any excluded Patterns.
func (f *Filter) HasPrefix(prefix string) bool {
	if f == nil {
		return true
	}

	parts := strings.Split(prefix, sep)

L:
	for i := len(parts); i > 0; i-- {
		prefix := strings.Join(parts[:i], sep)

		for _, p := range f.exclude {
			if p.Match(prefix) {
				break L
			}
		}

		if len(f.include) == 0 {
			return true
		}

		for _, p := range f.include {
			if p.HasPrefix(prefix) {
				return true
			}
		}
	}
	return false
}

// AllowsPattern returns whether the given filename is permitted by the
// inclusion/exclusion rules of this filter, as well as the pattern that either
// allowed or disallowed that filename.
//
// In special cases, such as a nil `*Filter` receiver, the absence of any
// patterns, or the given filename not being matched by any pattern, the empty
// string "" will be returned in place of the pattern.
func (f *Filter) AllowsPattern(filename string) (pattern string, allowed bool) {
	if f == nil {
		return "", true
	}

	if len(f.include)+len(f.exclude) == 0 {
		return "", true
	}

	cleanedName := filepath.Clean(filename)

	if len(f.include) > 0 {
		matched := false
		for _, inc := range f.include {
			matched = inc.Match(cleanedName)
			if matched {
				pattern = inc.String()
				break
			}
		}
		if !matched {
			return "", false
		}
	}

	if len(f.exclude) > 0 {
		for _, ex := range f.exclude {
			if ex.Match(cleanedName) {
				return ex.String(), false
			}
		}
	}

	return pattern, true
}

const (
	sep = string(filepath.Separator)
)

func NewPattern(rawpattern string) Pattern {
	cleanpattern := filepath.Clean(rawpattern)

	// Special case local dir, matches all (inc subpaths)
	if _, local := localDirSet[cleanpattern]; local {
		return noOpMatcher{}
	}

	hasPathSep := strings.Contains(cleanpattern, sep)
	ext := filepath.Ext(cleanpattern)
	plen := len(cleanpattern)
	if plen > 1 && !hasPathSep && strings.HasPrefix(cleanpattern, "*") && cleanpattern[1:plen] == ext {
		return &simpleExtPattern{ext: ext}
	}

	// special case * when there are no path separators
	// filepath.Match never allows * to match a path separator, which is correct
	// for gitignore IF the pattern includes a path separator, but not otherwise
	// So *.txt should match in any subdir, as should test*, but sub/*.txt would
	// only match directly in the sub dir
	// Don't need to test cross-platform separators as both cleaned above
	if !hasPathSep && strings.Contains(cleanpattern, "*") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*", ".*", -1))
		return &pathlessWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	// Also support ** with path separators
	if hasPathSep && strings.Contains(cleanpattern, "**") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*\\*", ".*", -1))
		return &doubleWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	if hasPathSep && strings.HasPrefix(cleanpattern, sep) {
		rel := cleanpattern[1:len(cleanpattern)]
		prefix := rel
		if strings.HasSuffix(rel, sep) {
			rel = rel[0 : len(rel)-1]
		} else {
			prefix += sep
		}

		return &pathPrefixPattern{
			rawPattern: cleanpattern,
			relative:   rel,
			prefix:     prefix,
		}
	}

	return &pathPattern{
		rawPattern: cleanpattern,
		prefix:     cleanpattern + sep,
		suffix:     sep + cleanpattern,
		inner:      sep + cleanpattern + sep,
	}
}

func convertToPatterns(rawpatterns []string) []Pattern {
	patterns := make([]Pattern, len(rawpatterns))
	for i, raw := range rawpatterns {
		patterns[i] = NewPattern(raw)
	}
	return patterns
}

type pathPrefixPattern struct {
	rawPattern string
	relative   string
	prefix     string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPrefixPattern) Match(name string) bool {
	if name == p.relative || strings.HasPrefix(name, p.prefix) {
		return true
	}
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched
}

func (p *pathPrefixPattern) HasPrefix(name string) bool {
	return strings.HasPrefix(p.relative, name)
}

// String returns a string representation of the underlying pattern for which
// this *pathPrefixPattern is matching.
func (p *pathPrefixPattern) String() string {
	return p.rawPattern
}

type pathPattern struct {
	rawPattern string
	prefix     string
	suffix     string
	inner      string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPattern) Match(name string) bool {
	if strings.HasPrefix(name, p.prefix) || strings.HasSuffix(name, p.suffix) || strings.Contains(name, p.inner) {
		return true
	}
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched
}

func (p *pathPattern) HasPrefix(name string) bool {
	return strings.HasPrefix(p.prefix, name)
}

// String returns a string representation of the underlying pattern for which
// this *pathPattern is matching.
func (p *pathPattern) String() string {
	return p.rawPattern
}

type simpleExtPattern struct {
	ext string
}

func (p *simpleExtPattern) Match(name string) bool {
	return strings.HasSuffix(name, p.ext)
}

func (p *simpleExtPattern) HasPrefix(name string) bool {
	return true
}

// String returns a string representation of the underlying pattern for which
// this *simpleExtPattern is matching.
func (p *simpleExtPattern) String() string {
	return fmt.Sprintf("*%s", p.ext)
}

type pathlessWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathlessWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(filepath.Base(name))
}

func (p *pathlessWildcardPattern) HasPrefix(name string) bool {
	lit, ok := p.wildcardRE.LiteralPrefix()
	if !ok {
		return true
	}

	return strings.HasPrefix(name, lit)
}

// String returns a string representation of the underlying pattern for which
// this *pathlessWildcardPattern is matching.
func (p *pathlessWildcardPattern) String() string {
	return p.rawPattern
}

type doubleWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *doubleWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(name)
}

func (p *doubleWildcardPattern) HasPrefix(name string) bool {
	lit, ok := p.wildcardRE.LiteralPrefix()
	if !ok {
		return true
	}

	return strings.HasPrefix(name, lit)
}

// String returns a string representation of the underlying pattern for which
// this *doubleWildcardPattern is matching.
func (p *doubleWildcardPattern) String() string {
	return p.rawPattern
}

type noOpMatcher struct {
}

func (n noOpMatcher) Match(name string) bool {
	return true
}

func (n noOpMatcher) HasPrefix(name string) bool {
	return true
}

func (n noOpMatcher) String() string {
	return ""
}

var localDirSet = map[string]struct{}{
	"*":   struct{}{},
	"*.*": struct{}{},
	".":   struct{}{},
	"./":  struct{}{},
	".\\": struct{}{},
}
<commit_msg>filepathfilter: Correctly implement fmt.Stringer on *noOpMatcher<commit_after>package filepathfilter

import (
	"fmt"
	"path/filepath"
	"regexp"
	"strings"
)

type Pattern interface {
	// HasPrefix returns whether the receiving Pattern will match a fullpath
	// that contains the prefix "prefix".
	//
	// For instance, if the receiving pattern were to match 'a/b/c.txt',
	// HasPrefix() will return true for:
	//
	//   - 'a', and 'a/'
	//   - 'a/b', and 'a/b/'
	HasPrefix(prefix string) bool

	Match(filename string) bool
	// String returns a string representation (see: regular expressions) of
	// the underlying pattern used to match filenames against this Pattern.
	String() string
}

type Filter struct {
	include []Pattern
	exclude []Pattern
}

func NewFromPatterns(include, exclude []Pattern) *Filter {
	return &Filter{include: include, exclude: exclude}
}

func New(include, exclude []string) *Filter {
	return NewFromPatterns(convertToPatterns(include), convertToPatterns(exclude))
}

// Include returns the result of calling String() on each Pattern in the
// include set of this *Filter.
func (f *Filter) Include() []string { return patternsToStrings(f.include...) }

// Exclude returns the result of calling String() on each Pattern in the
// exclude set of this *Filter.
func (f *Filter) Exclude() []string { return patternsToStrings(f.exclude...) }

// patternsToStrings maps the given set of Pattern's to a string slice by
// calling String() on each pattern.
func patternsToStrings(ps ...Pattern) []string {
	s := make([]string, 0, len(ps))
	for _, p := range ps {
		s = append(s, p.String())
	}

	return s
}

func (f *Filter) Allows(filename string) bool {
	_, allowed := f.AllowsPattern(filename)
	return allowed
}

// HasPrefix returns whether the given prefix "prefix" is a prefix for all
// included Patterns, and not a prefix for any excluded Patterns.
func (f *Filter) HasPrefix(prefix string) bool {
	if f == nil {
		return true
	}

	parts := strings.Split(prefix, sep)

L:
	for i := len(parts); i > 0; i-- {
		prefix := strings.Join(parts[:i], sep)

		for _, p := range f.exclude {
			if p.Match(prefix) {
				break L
			}
		}

		if len(f.include) == 0 {
			return true
		}

		for _, p := range f.include {
			if p.HasPrefix(prefix) {
				return true
			}
		}
	}
	return false
}

// AllowsPattern returns whether the given filename is permitted by the
// inclusion/exclusion rules of this filter, as well as the pattern that either
// allowed or disallowed that filename.
//
// In special cases, such as a nil `*Filter` receiver, the absence of any
// patterns, or the given filename not being matched by any pattern, the empty
// string "" will be returned in place of the pattern.
func (f *Filter) AllowsPattern(filename string) (pattern string, allowed bool) {
	if f == nil {
		return "", true
	}

	if len(f.include)+len(f.exclude) == 0 {
		return "", true
	}

	cleanedName := filepath.Clean(filename)

	if len(f.include) > 0 {
		matched := false
		for _, inc := range f.include {
			matched = inc.Match(cleanedName)
			if matched {
				pattern = inc.String()
				break
			}
		}
		if !matched {
			return "", false
		}
	}

	if len(f.exclude) > 0 {
		for _, ex := range f.exclude {
			if ex.Match(cleanedName) {
				return ex.String(), false
			}
		}
	}

	return pattern, true
}

const (
	sep = string(filepath.Separator)
)

func NewPattern(rawpattern string) Pattern {
	cleanpattern := filepath.Clean(rawpattern)

	// Special case local dir, matches all (inc subpaths)
	if _, local := localDirSet[cleanpattern]; local {
		return &noOpMatcher{pattern: cleanpattern}
	}

	hasPathSep := strings.Contains(cleanpattern, sep)
	ext := filepath.Ext(cleanpattern)
	plen := len(cleanpattern)
	if plen > 1 && !hasPathSep && strings.HasPrefix(cleanpattern, "*") && cleanpattern[1:plen] == ext {
		return &simpleExtPattern{ext: ext}
	}

	// special case * when there are no path separators
	// filepath.Match never allows * to match a path separator, which is correct
	// for gitignore IF the pattern includes a path separator, but not otherwise
	// So *.txt should match in any subdir, as should test*, but sub/*.txt would
	// only match directly in the sub dir
	// Don't need to test cross-platform separators as both cleaned above
	if !hasPathSep && strings.Contains(cleanpattern, "*") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*", ".*", -1))
		return &pathlessWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	// Also support ** with path separators
	if hasPathSep && strings.Contains(cleanpattern, "**") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*\\*", ".*", -1))
		return &doubleWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	if hasPathSep && strings.HasPrefix(cleanpattern, sep) {
		rel := cleanpattern[1:len(cleanpattern)]
		prefix := rel
		if strings.HasSuffix(rel, sep) {
			rel = rel[0 : len(rel)-1]
		} else {
			prefix += sep
		}

		return &pathPrefixPattern{
			rawPattern: cleanpattern,
			relative:   rel,
			prefix:     prefix,
		}
	}

	return &pathPattern{
		rawPattern: cleanpattern,
		prefix:     cleanpattern + sep,
		suffix:     sep + cleanpattern,
		inner:      sep + cleanpattern + sep,
	}
}

func convertToPatterns(rawpatterns []string) []Pattern {
	patterns := make([]Pattern, len(rawpatterns))
	for i, raw := range rawpatterns {
		patterns[i] = NewPattern(raw)
	}
	return patterns
}

type pathPrefixPattern struct {
	rawPattern string
	relative   string
	prefix     string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPrefixPattern) Match(name string) bool {
	if name == p.relative || strings.HasPrefix(name, p.prefix) {
		return true
	}
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched
}

func (p *pathPrefixPattern) HasPrefix(name string) bool {
	return strings.HasPrefix(p.relative, name)
}

// String returns a string representation of the underlying pattern for which
// this *pathPrefixPattern is matching.
func (p *pathPrefixPattern) String() string {
	return p.rawPattern
}

type pathPattern struct {
	rawPattern string
	prefix     string
	suffix     string
	inner      string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPattern) Match(name string) bool {
	if strings.HasPrefix(name, p.prefix) || strings.HasSuffix(name, p.suffix) || strings.Contains(name, p.inner) {
		return true
	}
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched
}

func (p *pathPattern) HasPrefix(name string) bool {
	return strings.HasPrefix(p.prefix, name)
}

// String returns a string representation of the underlying pattern for which
// this *pathPattern is matching.
func (p *pathPattern) String() string {
	return p.rawPattern
}

type simpleExtPattern struct {
	ext string
}

func (p *simpleExtPattern) Match(name string) bool {
	return strings.HasSuffix(name, p.ext)
}

func (p *simpleExtPattern) HasPrefix(name string) bool {
	return true
}

// String returns a string representation of the underlying pattern for which
// this *simpleExtPattern is matching.
func (p *simpleExtPattern) String() string {
	return fmt.Sprintf("*%s", p.ext)
}

type pathlessWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathlessWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(filepath.Base(name))
}

func (p *pathlessWildcardPattern) HasPrefix(name string) bool {
	lit, ok := p.wildcardRE.LiteralPrefix()
	if !ok {
		return true
	}

	return strings.HasPrefix(name, lit)
}

// String returns a string representation of the underlying pattern for which
// this *pathlessWildcardPattern is matching.
func (p *pathlessWildcardPattern) String() string {
	return p.rawPattern
}

type doubleWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *doubleWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(name)
}

func (p *doubleWildcardPattern) HasPrefix(name string) bool {
	lit, ok := p.wildcardRE.LiteralPrefix()
	if !ok {
		return true
	}

	return strings.HasPrefix(name, lit)
}

// String returns a string representation of the underlying pattern for which
// this *doubleWildcardPattern is matching.
func (p *doubleWildcardPattern) String() string {
	return p.rawPattern
}

type noOpMatcher struct {
	pattern string
}

func (n *noOpMatcher) Match(name string) bool {
	return true
}

func (n *noOpMatcher) HasPrefix(name string) bool {
	return true
}

func (n *noOpMatcher) String() string {
	if n == nil {
		return ""
	}
	return n.pattern
}

var localDirSet = map[string]struct{}{
	"*":   struct{}{},
	"*.*": struct{}{},
	".":   struct{}{},
	"./":  struct{}{},
	".\\": struct{}{},
}
<|endoftext|>


<commit_before>package panli

import (
	"github.com/catorpilor/leetcode/utils"
)

func reverseWithAlloc(head *utils.ListNode) *utils.ListNode {
	if head == nil {
		return head
	}
	rh := &utils.ListNode{Val: head.Val, Next: head.Next}
	next := rh.Next
	rh.Next = nil
	for next != nil {
		node := &utils.ListNode{Val: next.Val, Next: next.Next}
		temp := node.Next
		node.Next = rh
		rh = node
		next = temp
	}
	return rh
}

func reverse(head *utils.ListNode) *utils.ListNode {
	if head == nil {
		return head
	}
	next := head.Next
	head.Next = nil
	for next != nil {
		temp := next.Next
		next.Next = head
		head = next
		next = temp
	}
	return head
}

// IsPalindromeWithAlloc is an O(n) time, O(n) space algorithm
func IsPalindromeWithAlloc(head *utils.ListNode) bool {
	// reverse linked list
	rev := reverseWithAlloc(head)
	// fmt.Printf("reversed linked list: %s\n", rev)
	// fmt.Printf("origin linked list: %s\n", head)
	for rev != nil {
		if rev.Val == head.Val {
			rev, head = rev.Next, head.Next
		} else {
			return false
		}
	}
	return true
}

func IsPalindrome(head *utils.ListNode) bool {
	if head == nil || head.Next == nil {
		return true
	}
	slow, fast := head, head
	for fast.Next != nil && fast.Next.Next != nil {
		slow, fast = slow.Next, fast.Next.Next
	}
	slow.Next = reverse(slow.Next)
	slow = slow.Next
	for slow != nil {
		if head.Val != slow.Val {
			return false
		}
		slow = slow.Next
		head = head.Next
	}
	return true
}
<commit_msg>solve 234 using two pointers<commit_after>package panli

import (
	"github.com/catorpilor/leetcode/utils"
)

func reverseWithAlloc(head *utils.ListNode) *utils.ListNode {
	if head == nil {
		return head
	}
	rh := &utils.ListNode{Val: head.Val, Next: head.Next}
	next := rh.Next
	rh.Next = nil
	for next != nil {
		node := &utils.ListNode{Val: next.Val, Next: next.Next}
		temp := node.Next
		node.Next = rh
		rh = node
		next = temp
	}
	return rh
}

func reverse(head *utils.ListNode) *utils.ListNode {
	if head == nil {
		return head
	}
	next := head.Next
	head.Next = nil
	for next != nil {
		temp := next.Next
		next.Next = head
		head = next
		next = temp
	}
	return head
}

// IsPalindromeWithAlloc is an O(n) time, O(n) space algorithm
func IsPalindromeWithAlloc(head *utils.ListNode) bool {
	// reverse linked list
	rev := reverseWithAlloc(head)
	// fmt.Printf("reversed linked list: %s\n", rev)
	// fmt.Printf("origin linked list: %s\n", head)
	for rev != nil {
		if rev.Val == head.Val {
			rev, head = rev.Next, head.Next
		} else {
			return false
		}
	}
	return true
}

// IsPalindrome use two pointers time complexity O(N), space complexity O(1)
func IsPalindrome(head *utils.ListNode) bool {
	if head == nil || head.Next == nil {
		return true
	}
	slow, fast := head, head
	for fast.Next != nil && fast.Next.Next != nil {
		slow, fast = slow.Next, fast.Next.Next
	}
	slow.Next = reverse(slow.Next)
	slow = slow.Next
	for slow != nil {
		if head.Val != slow.Val {
			return false
		}
		slow = slow.Next
		head = head.Next
	}
	return true
}
<|endoftext|>


<commit_before>// main_test.go
package main

import (
	"reflect"
	"testing"
)

func TestGetKeymapFromOutput(t *testing.T) {
	var buf []byte
	var keymap map[uint8]string

	const test1 = "keycode  19 = 0 parenright 0 parenright\n" +
		"keycode  20 = minus underscore minus underscore\n" +
		"keycode  21 = equal plus equal plus"
	result1 := map[uint8]string{19: "0", 20: "minus", 21: "equal"}

	const test2 = "keycode 119 = Delete NoSymbol Delete\n" +
		"keycode 120 =\n" +
		"keycode 121 = XF86AudioMute NoSymbol XF86AudioMute"
	result2 := map[uint8]string{119: "Delete", 121: "XF86AudioMute"}

	// Test1. Simple
	buf = []byte(test1)
	keymap = GetKeymapFromOutput(buf)
	if !reflect.DeepEqual(keymap, result1) {
		t.Fail()
	}

	// Test2. With empty keys
	buf = []byte(test2)
	keymap = GetKeymapFromOutput(buf)
	if !reflect.DeepEqual(keymap, result2) {
		t.Fail()
	}
}

func TestGetKeyNumsFromOutput(t *testing.T) {
	var buf []byte
	var keyNums []uint8

	const test1 = "key press 36\nkey release 41\nkey press   41"
	var result1 = []uint8{36, 41}

	const test2 = ""
	var result2 = []uint8{}

	// Test1. Simple
	buf = []byte(test1)
	keyNums = GetKeyNumsFromOutput(buf)
	if !reflect.DeepEqual(keyNums, result1) {
		t.Fail()
	}

	// Test2. Clear
	buf = []byte(test2)
	keyNums = GetKeyNumsFromOutput(buf)
	if !reflect.DeepEqual(keyNums, result2) {
		t.Fail()
	}
}

func TestGetFileType(t *testing.T) {
	tests := map[string]string{
		"":                                "",
		"out.csv":                         "csv",
		"out.jsl":                         "jsl",
		"out.json":                        "json",
		"//sfd.dsf//.f./out.out.csv":      "csv",
		"..\\data.all.jsl.gz":             "jsl.gz",
		"../full.csv/first.data.json":     "json",
		"//sfd.dsf//.f./WhiteBear.csv.Gz": "csv.gz",
		"out.JsL":                         "jsl",
		"////////":                        "",
		"\\\\\\":                          "",
		"file1.Json.gz":                   "json.gz",
		"out.csv.json.jsl":                "jsl",
		"out.jsl.json.csv":                "csv",
	}
	for test, res := range tests {
		if GetFileType(test) != res {
			t.Log("On test ", test, " result is ", GetFileType(test), " but right is ", res, "")
		}
		t.Fail()
	}
}
<commit_msg>Fixed GetFileType test<commit_after>// main_test.go
package main

import (
	"reflect"
	"testing"
)

func TestGetKeymapFromOutput(t *testing.T) {
	var buf []byte
	var keymap map[uint8]string

	const test1 = "keycode  19 = 0 parenright 0 parenright\n" +
		"keycode  20 = minus underscore minus underscore\n" +
		"keycode  21 = equal plus equal plus"
	result1 := map[uint8]string{19: "0", 20: "minus", 21: "equal"}

	const test2 = "keycode 119 = Delete NoSymbol Delete\n" +
		"keycode 120 =\n" +
		"keycode 121 = XF86AudioMute NoSymbol XF86AudioMute"
	result2 := map[uint8]string{119: "Delete", 121: "XF86AudioMute"}

	// Test1. Simple
	buf = []byte(test1)
	keymap = GetKeymapFromOutput(buf)
	if !reflect.DeepEqual(keymap, result1) {
		t.Fail()
	}

	// Test2. With empty keys
	buf = []byte(test2)
	keymap = GetKeymapFromOutput(buf)
	if !reflect.DeepEqual(keymap, result2) {
		t.Fail()
	}
}

func TestGetKeyNumsFromOutput(t *testing.T) {
	var buf []byte
	var keyNums []uint8

	const test1 = "key press 36\nkey release 41\nkey press   41"
	var result1 = []uint8{36, 41}

	const test2 = ""
	var result2 = []uint8{}

	// Test1. Simple
	buf = []byte(test1)
	keyNums = GetKeyNumsFromOutput(buf)
	if !reflect.DeepEqual(keyNums, result1) {
		t.Fail()
	}

	// Test2. Clear
	buf = []byte(test2)
	keyNums = GetKeyNumsFromOutput(buf)
	if !reflect.DeepEqual(keyNums, result2) {
		t.Fail()
	}
}

func TestGetFileType(t *testing.T) {
	tests := map[string]string{
		"":                                "",
		"out.csv":                         "csv",
		"out.jsl":                         "jsl",
		"out.json":                        "json",
		"//sfd.dsf//.f./out.out.csv":      "csv",
		"..\\data.all.jsl.gz":             "jsl.gz",
		"../full.csv/first.data.json":     "json",
		"//sfd.dsf//.f./WhiteBear.csv.Gz": "csv.gz",
		"out.JsL":                         "jsl",
		"////////":                        "",
		"\\\\\\":                          "",
		"file1.Json.gz":                   "json.gz",
		"out.csv.json.jsl":                "jsl",
		"out.jsl.json.csv":                "csv",
	}
	for test, res := range tests {
		if GetFileType(test) != res {
			t.Log("On test ", test, " result is ", GetFileType(test), " but right is ", res, "")
			t.Fail()
		}
	}
}
<|endoftext|>