<commit_before>package azurerm

import (
	"fmt"
	"log"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/network"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmVirtualNetwork() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmVirtualNetworkCreate,
		Read:   resourceArmVirtualNetworkRead,
		Update: resourceArmVirtualNetworkCreate,
		Delete: resourceArmVirtualNetworkDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"address_space": {
				Type:     schema.TypeList,
				Required: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"dns_servers": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"subnet": {
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": {
							Type:     schema.TypeString,
							Required: true,
						},
						"address_prefix": {
							Type:     schema.TypeString,
							Required: true,
						},
						"security_group": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
				Set: resourceAzureSubnetHash,
			},

			"location": locationSchema(),

			"resource_group_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceArmVirtualNetworkCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ArmClient)
	vnetClient := client.vnetClient

	log.Printf("[INFO] preparing arguments for Azure ARM virtual network creation.")

	name := d.Get("name").(string)
	location := d.Get("location").(string)
	resGroup := d.Get("resource_group_name").(string)
	tags := d.Get("tags").(map[string]interface{})

	vnet := network.VirtualNetwork{
		Name:                           &name,
		Location:                       &location,
		VirtualNetworkPropertiesFormat: getVirtualNetworkProperties(d),
		Tags: expandTags(tags),
	}

	_, err := vnetClient.CreateOrUpdate(resGroup, name, vnet, make(chan struct{}))
	if err != nil {
		return err
	}

	read, err := vnetClient.Get(resGroup, name, "")
	if err != nil {
		return err
	}
	if read.ID == nil {
		return fmt.Errorf("Cannot read Virtual Network %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*read.ID)

	return resourceArmVirtualNetworkRead(d, meta)
}

func resourceArmVirtualNetworkRead(d *schema.ResourceData, meta interface{}) error {
	vnetClient := meta.(*ArmClient).vnetClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	name := id.Path["virtualNetworks"]

	resp, err := vnetClient.Get(resGroup, name, "")
	if err != nil {
		if resp.StatusCode == http.StatusNotFound {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error making Read request on Azure virtual network %s: %s", name, err)
	}

	vnet := *resp.VirtualNetworkPropertiesFormat

	// update appropriate values
	d.Set("resource_group_name", resGroup)
	d.Set("name", resp.Name)
	d.Set("location", resp.Location)
	d.Set("address_space", vnet.AddressSpace.AddressPrefixes)

	subnets := &schema.Set{
		F: resourceAzureSubnetHash,
	}

	for _, subnet := range *vnet.Subnets {
		s := map[string]interface{}{}

		s["name"] = *subnet.Name
		s["address_prefix"] = *subnet.SubnetPropertiesFormat.AddressPrefix
		if subnet.SubnetPropertiesFormat.NetworkSecurityGroup != nil {
			s["security_group"] = *subnet.SubnetPropertiesFormat.NetworkSecurityGroup.ID
		}

		subnets.Add(s)
	}
	d.Set("subnet", subnets)

	if vnet.DhcpOptions != nil && vnet.DhcpOptions.DNSServers != nil {
		dnses := []string{}
		for _, dns := range *vnet.DhcpOptions.DNSServers {
			dnses = append(dnses, dns)
		}
		d.Set("dns_servers", dnses)
	}

	flattenAndSetTags(d, resp.Tags)

	return nil
}

func resourceArmVirtualNetworkDelete(d *schema.ResourceData, meta interface{}) error {
	vnetClient := meta.(*ArmClient).vnetClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	name := id.Path["virtualNetworks"]

	_, err = vnetClient.Delete(resGroup, name, make(chan struct{}))

	return err
}

func getVirtualNetworkProperties(d *schema.ResourceData) *network.VirtualNetworkPropertiesFormat {
	// first; get address space prefixes:
	prefixes := []string{}
	for _, prefix := range d.Get("address_space").([]interface{}) {
		prefixes = append(prefixes, prefix.(string))
	}

	// then; the dns servers:
	dnses := []string{}
	for _, dns := range d.Get("dns_servers").([]interface{}) {
		dnses = append(dnses, dns.(string))
	}

	// then; the subnets:
	subnets := []network.Subnet{}
	if subs := d.Get("subnet").(*schema.Set); subs.Len() > 0 {
		for _, subnet := range subs.List() {
			subnet := subnet.(map[string]interface{})

			name := subnet["name"].(string)
			prefix := subnet["address_prefix"].(string)
			secGroup := subnet["security_group"].(string)

			var subnetObj network.Subnet
			subnetObj.Name = &name
			subnetObj.SubnetPropertiesFormat = &network.SubnetPropertiesFormat{}
			subnetObj.SubnetPropertiesFormat.AddressPrefix = &prefix

			if secGroup != "" {
				subnetObj.SubnetPropertiesFormat.NetworkSecurityGroup = &network.SecurityGroup{
					ID: &secGroup,
				}
			}

			subnets = append(subnets, subnetObj)
		}
	}

	// finally; return the struct:
	return &network.VirtualNetworkPropertiesFormat{
		AddressSpace: &network.AddressSpace{
			AddressPrefixes: &prefixes,
		},
		DhcpOptions: &network.DhcpOptions{
			DNSServers: &dnses,
		},
		Subnets: &subnets,
	}
}

func resourceAzureSubnetHash(v interface{}) int {
	m := v.(map[string]interface{})
	subnet := m["name"].(string) + m["address_prefix"].(string)
	if securityGroup, present := m["security_group"]; present {
		subnet = subnet + securityGroup.(string)
	}
	return hashcode.String(subnet)
}
<commit_msg>Locking the NSG to only operate on one resource at a time in the create<commit_after>package azurerm

import (
	"fmt"
	"log"
	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/network"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmVirtualNetwork() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmVirtualNetworkCreate,
		Read:   resourceArmVirtualNetworkRead,
		Update: resourceArmVirtualNetworkCreate,
		Delete: resourceArmVirtualNetworkDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"address_space": {
				Type:     schema.TypeList,
				Required: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"dns_servers": {
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"subnet": {
				Type:     schema.TypeSet,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": {
							Type:     schema.TypeString,
							Required: true,
						},
						"address_prefix": {
							Type:     schema.TypeString,
							Required: true,
						},
						"security_group": {
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
				Set: resourceAzureSubnetHash,
			},

			"location": locationSchema(),

			"resource_group_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceArmVirtualNetworkCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ArmClient)
	vnetClient := client.vnetClient

	log.Printf("[INFO] preparing arguments for Azure ARM virtual network creation.")

	name := d.Get("name").(string)
	location := d.Get("location").(string)
	resGroup := d.Get("resource_group_name").(string)
	tags := d.Get("tags").(map[string]interface{})

	vnet := network.VirtualNetwork{
		Name:                           &name,
		Location:                       &location,
		VirtualNetworkPropertiesFormat: getVirtualNetworkProperties(d),
		Tags: expandTags(tags),
	}

	networkSecurityGroupNames := make([]string, 0)
	for _, subnet := range *vnet.VirtualNetworkPropertiesFormat.Subnets {
		if subnet.NetworkSecurityGroup != nil {
			subnetId := *subnet.NetworkSecurityGroup.ID
			id, err := parseAzureResourceID(subnetId)
			if err != nil {
				return fmt.Errorf("[ERROR] Unable to Parse Network Security Group ID '%s': %+v", subnetId, err)
			}
			nsgName := id.Path["networkSecurityGroups"]
			networkSecurityGroupNames = append(networkSecurityGroupNames, nsgName)
		}
	}

	azureRMVirtualNetworkLockNetworkSecurityGroups(&networkSecurityGroupNames)
	defer azureRMVirtualNetworkUnlockNetworkSecurityGroups(&networkSecurityGroupNames)

	_, err := vnetClient.CreateOrUpdate(resGroup, name, vnet, make(chan struct{}))
	if err != nil {
		return err
	}

	read, err := vnetClient.Get(resGroup, name, "")
	if err != nil {
		return err
	}
	if read.ID == nil {
		return fmt.Errorf("Cannot read Virtual Network %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*read.ID)

	return resourceArmVirtualNetworkRead(d, meta)
}

func resourceArmVirtualNetworkRead(d *schema.ResourceData, meta interface{}) error {
	vnetClient := meta.(*ArmClient).vnetClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	name := id.Path["virtualNetworks"]

	resp, err := vnetClient.Get(resGroup, name, "")
	if err != nil {
		if resp.StatusCode == http.StatusNotFound {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error making Read request on Azure virtual network %s: %s", name, err)
	}

	vnet := *resp.VirtualNetworkPropertiesFormat

	// update appropriate values
	d.Set("resource_group_name", resGroup)
	d.Set("name", resp.Name)
	d.Set("location", resp.Location)
	d.Set("address_space", vnet.AddressSpace.AddressPrefixes)

	subnets := &schema.Set{
		F: resourceAzureSubnetHash,
	}

	for _, subnet := range *vnet.Subnets {
		s := map[string]interface{}{}

		s["name"] = *subnet.Name
		s["address_prefix"] = *subnet.SubnetPropertiesFormat.AddressPrefix
		if subnet.SubnetPropertiesFormat.NetworkSecurityGroup != nil {
			s["security_group"] = *subnet.SubnetPropertiesFormat.NetworkSecurityGroup.ID
		}

		subnets.Add(s)
	}
	d.Set("subnet", subnets)

	if vnet.DhcpOptions != nil && vnet.DhcpOptions.DNSServers != nil {
		dnses := []string{}
		for _, dns := range *vnet.DhcpOptions.DNSServers {
			dnses = append(dnses, dns)
		}
		d.Set("dns_servers", dnses)
	}

	flattenAndSetTags(d, resp.Tags)

	return nil
}

func resourceArmVirtualNetworkDelete(d *schema.ResourceData, meta interface{}) error {
	vnetClient := meta.(*ArmClient).vnetClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	name := id.Path["virtualNetworks"]

	// TODO: lock any associated NSG's

	_, err = vnetClient.Delete(resGroup, name, make(chan struct{}))

	return err
}

func getVirtualNetworkProperties(d *schema.ResourceData) *network.VirtualNetworkPropertiesFormat {
	// first; get address space prefixes:
	prefixes := []string{}
	for _, prefix := range d.Get("address_space").([]interface{}) {
		prefixes = append(prefixes, prefix.(string))
	}

	// then; the dns servers:
	dnses := []string{}
	for _, dns := range d.Get("dns_servers").([]interface{}) {
		dnses = append(dnses, dns.(string))
	}

	// then; the subnets:
	subnets := []network.Subnet{}
	if subs := d.Get("subnet").(*schema.Set); subs.Len() > 0 {
		for _, subnet := range subs.List() {
			subnet := subnet.(map[string]interface{})

			name := subnet["name"].(string)
			prefix := subnet["address_prefix"].(string)
			secGroup := subnet["security_group"].(string)

			var subnetObj network.Subnet
			subnetObj.Name = &name
			subnetObj.SubnetPropertiesFormat = &network.SubnetPropertiesFormat{}
			subnetObj.SubnetPropertiesFormat.AddressPrefix = &prefix

			if secGroup != "" {
				subnetObj.SubnetPropertiesFormat.NetworkSecurityGroup = &network.SecurityGroup{
					ID: &secGroup,
				}
			}

			subnets = append(subnets, subnetObj)
		}
	}

	// finally; return the struct:
	return &network.VirtualNetworkPropertiesFormat{
		AddressSpace: &network.AddressSpace{
			AddressPrefixes: &prefixes,
		},
		DhcpOptions: &network.DhcpOptions{
			DNSServers: &dnses,
		},
		Subnets: &subnets,
	}
}

func resourceAzureSubnetHash(v interface{}) int {
	m := v.(map[string]interface{})
	subnet := m["name"].(string) + m["address_prefix"].(string)
	if securityGroup, present := m["security_group"]; present {
		subnet = subnet + securityGroup.(string)
	}
	return hashcode.String(subnet)
}

func azureRMVirtualNetworkUnlockNetworkSecurityGroups(networkSecurityGroupNames *[]string) {
	for _, networkSecurityGroupName := range *networkSecurityGroupNames {
		armMutexKV.Unlock(networkSecurityGroupName)
	}
}
func azureRMVirtualNetworkLockNetworkSecurityGroups(networkSecurityGroupNames *[]string) {
	for _, networkSecurityGroupName := range *networkSecurityGroupNames {
		armMutexKV.Lock(networkSecurityGroupName)
	}
}<|endoftext|>


<commit_before>package workstation

import (
	"fmt"
	"io"
	"os/exec"
	"syscall"

	"math"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"

	"encoding/csv"
	"github.com/xshellinc/tools/dialogs"
	"github.com/xshellinc/tools/lib/help"
)

const cleanTemplate = `
select disk %s
clean
create partition primary
active
assign letter=N
remove letter=N
format fs=fat32 label=KERNEL quick
`

type windows struct {
	*workstation
	ddPath string
}

// Initializes windows workstation
func newWorkstation(disk string) WorkStation {
	m := new(MountInfo)
	var ms []*MountInfo
	return &windows{&workstation{disk, runtime.GOOS, true, m, ms}, ""}
}

// Lists available mounts
func (w *windows) ListRemovableDisk() ([]*MountInfo, error) {
	log.Debug("Listing disks...")
	fmt.Println("[+] Listing available disks...")
	var out = []*MountInfo{}

	// stdout, err := help.ExecCmd("wmic", []string{"diskdrive", "get", "DeviceID,index,InterfaceType,MediaType,Model,Size", "/format:csv"})
	// ugly fix for windows 7 bug where `format:csv` is broken. Also GO double escaping quoted arguments.
	cmd := exec.Command(`cmd`)
	cmd.SysProcAttr = &syscall.SysProcAttr{}
	cmd.SysProcAttr.CmdLine = `cmd /s /c "wmic diskdrive get DeviceID,index,InterfaceType,MediaType,Model,Size /format:"%WINDIR%\System32\wbem\en-US\csv""`
	stdoutb, err := cmd.Output()
	stdout := string(stdoutb)
	log.Debug(stdout)
	if err != nil {
		stdout = ""
	}

	r := csv.NewReader(strings.NewReader(strings.TrimSpace(stdout)))
	r.TrimLeadingSpace = true
	r.Read() //skip the first line
	for {
		if record, err := r.Read(); err == io.EOF {
			break
		} else if err == nil {
			if !strings.Contains(record[4], "Removable") || strings.Contains(record[3], "IDE") {
				continue
			}
			var p = &MountInfo{}
			size := record[6]
			p.deviceSize = size
			sizeInt, _ := strconv.Atoi(size)
			sizeFloat := math.Ceil(float64(sizeInt) / 1024 / 1024 / 1024)
			p.deviceName = record[5] + " [" + strconv.Itoa(int(sizeFloat)) + "GB]"
			p.diskName = `\\?\Device\Harddisk` + record[2] + `\Partition0`
			p.diskNameRaw = record[2]
			out = append(out, p)
		}
	}
	log.WithField("out", out).Debug("got drives")
	if !(len(out) > 0) {
		return nil, fmt.Errorf("[-] No removable disks found, please insert your SD card and try again.\n[-] Please remember to run this tool as an administrator.")
	}
	w.workstation.mounts = out
	return out, nil
}

// Unmounts the disk
func (w *windows) Unmount() error {
	return nil
}

// Ejects the mounted disk
func (w *windows) Eject() error {
	return nil
}

const diskSelectionTries = 3
const writeAttempts = 5

// CopyToDisk Notifies user to choose a mount, after that it tries to copy the data
func (w *windows) CopyToDisk(img string) (job *help.BackgroundJob, err error) {
	log.Debug("CopyToDisk")
	_, err = w.ListRemovableDisk()
	if err != nil {
		fmt.Println("[-] SD card is not found, please insert an unlocked SD card")
		return nil, err
	}

	var dev *MountInfo
	if len(w.Disk) == 0 {
		rng := make([]string, len(w.workstation.mounts))
		for i, e := range w.workstation.mounts {
			rng[i] = fmt.Sprintf(dialogs.PrintColored("%s")+" - "+dialogs.PrintColored("%s")+" (%s)", e.deviceName, e.diskName, e.deviceSize)
		}
		num := dialogs.SelectOneDialog("Select disk to format: ", rng)
		dev = w.workstation.mounts[num]
	} else {
		for _, e := range w.workstation.mounts {
			if e.diskName == w.Disk {
				dev = e
				break
			}
		}
		if dev == nil {
			return nil, fmt.Errorf("Disk name not recognised, try to list disks with " + dialogs.PrintColored("disks") + " argument")
		}
	}

	w.workstation.mount = dev
	fmt.Printf("[+] Writing image to %s\n", dev.diskName)
	log.WithField("image", img).WithField("mount", "N:").Debugf("Writing image to %s", dev.diskName)

	if err := w.CleanDisk(dev.diskName); err != nil {
		return nil, err
	}

	job = help.NewBackgroundJob()
	go func() {
		defer job.Close()
		job.Active(true)
		help.ExecCmd("unzip", []string{img, "-d", "N:\\"})
		fmt.Println("\r[+] Done writing image to N:")
	}()

	return job, nil
}

// WriteToDisk notifies user to choose a mount, after that it tries to write the data with `diskSelectionTries` number of retries
func (w *windows) WriteToDisk(img string) (job *help.BackgroundJob, err error) {
	for attempt := 0; attempt < diskSelectionTries; attempt++ {
		if attempt > 0 && !dialogs.YesNoDialog("Continue?") {
			break
		}

		_, err = w.ListRemovableDisk()
		if err != nil {
			fmt.Println("[-] SD card not found, please insert an unlocked SD card")
			continue
		}
		if len(w.Disk) == 0 {
			rng := make([]string, len(w.workstation.mounts))
			for i, e := range w.workstation.mounts {
				rng[i] = fmt.Sprintf(dialogs.PrintColored("%s")+" - "+dialogs.PrintColored("%s"), e.deviceName, e.diskName)
			}
			num := dialogs.SelectOneDialog("Select disk to use: ", rng)

			w.workstation.mount = w.workstation.mounts[num]
		} else {
			for _, e := range w.workstation.mounts {
				if e.diskName == w.Disk || e.diskNameRaw == w.Disk {
					w.workstation.mount = e
					break
				}
			}
			if w.workstation.mount == nil {
				return nil, fmt.Errorf("Disk name not recognised, try to list disks with " + dialogs.PrintColored("disks") + " argument")
			}
		}
		break
	}

	if err != nil {
		return nil, err
	}

	if w.ddPath == "" {
		if err := w.getDDBinary(); err != nil {
			log.Error(err)
			fmt.Println("[-] Error downloading dd binary")
			return nil, err
		}
	}

	if len(w.Disk) == 0 && !dialogs.YesNoDialog("Are you sure? ") {
		return nil, nil
	}

	fmt.Printf("[+] Writing %s to %s\n", img, w.workstation.mount.deviceName)

	job = help.NewBackgroundJob()
	go func() {
		defer job.Close()

		var err error
		for attempt := 0; attempt < writeAttempts; attempt++ {
			if attempt > 0 {
				if !dialogs.YesNoDialog("Retry flashing?") {
					break
				}
			}
			job.Active(true)
			var out []byte
			if out, err = exec.Command(w.ddPath,
				"--filter=removable",
				fmt.Sprintf("if=%s", img),
				fmt.Sprintf("of=%s", w.workstation.mount.diskName),
				"bs=1M").CombinedOutput(); err != nil {
				log.WithField("out", string(out)).Error("Error while executing: `", w.ddPath)
				job.Active(false)
				fmt.Println("\r[-] Can't write to disk.")
			} else {
				sout := string(out)
				job.Active(false)
				log.WithField("out", sout).Debug("dd finished")
				if strings.Contains(sout, "Error ") {
					if strings.Contains(sout, "Access is denied") || strings.Contains(sout, "The device is not ready") {
						fmt.Println("\n[-] Can't write to disk. Please make sure to run this tool as administrator, close all Explorer windows, try reconnecting your disk and finally reboot your computer.\n [-] You may need to run this tool with `clean` argument to clean your disk partition table before applying image.")
						if dialogs.YesNoDialog("Or we can try to clean it's partitions right now, should we proceed?") {
							if derr := w.CleanDisk(""); derr != nil {
								fmt.Println("[-] Disk cleaning failed:", derr)
								continue
							} else {
								for !dialogs.YesNoDialog("[+] Disk formatted, now please reconnect the device. Type yes once you've done it.") {
								}
							}
						}
						continue
					} else {
						fmt.Println(sout)
						continue
					}
				}
				fmt.Printf("\r[+] Done writing %s to %s \n", img, w.workstation.mount.diskName)
				return
			}
		}

		if err != nil {
			job.Error(err)
		}

		job.Error(fmt.Errorf("Image wasn't flashed"))
	}()

	return job, nil
}

func (w *windows) getDDBinary() error {
	dst := help.GetTempDir() + help.Separator()
	url := "https://cdn.isaax.io/isaax-distro/utilities/dd/ddrelease64.zip"

	if help.Exists(dst + "ddrelease64.exe") {
		w.ddPath = dst + "ddrelease64.exe"
		return nil
	}

	wg := &sync.WaitGroup{}
	fileName, bar, err := help.DownloadFromUrlWithAttemptsAsync(url, dst, 5, wg)
	if err != nil {
		return err
	}

	bar.Prefix(fmt.Sprintf("[+] Download %-15s", fileName))
	bar.Start()
	wg.Wait()
	bar.Finish()
	time.Sleep(time.Second)

	log.WithField("dst", dst).Debug("Extracting")
	if out, err := exec.Command("unzip", "-o", dst+"ddrelease64.zip", "-d", dst).CombinedOutput(); err != nil {
		return err
	} else {
		log.Debug(string(out))
	}
	w.ddPath = dst + "ddrelease64.exe"
	return nil
}

// CleanDisk cleans target disk partitions
func (w *windows) CleanDisk(disk string) error {
	fmt.Println("[+] Cleaning disk...")
	var last error
	for attempt := 0; attempt < diskSelectionTries; attempt++ {
		if attempt > 0 && !dialogs.YesNoDialog("Continue?") {
			break
		}

		if _, err := w.ListRemovableDisk(); err != nil {
			fmt.Println("[-] SD card not found, please insert an unlocked SD card")
			last = err
			continue
		}

		rng := make([]string, len(w.workstation.mounts))
		for i, e := range w.workstation.mounts {
			rng[i] = fmt.Sprintf(dialogs.PrintColored("%s")+" - "+dialogs.PrintColored("%s"), e.deviceName, e.diskName)
		}
		num := dialogs.SelectOneDialog("Select disk to clean: ", rng)

		w.workstation.mount = w.workstation.mounts[num]
		break
	}

	if last != nil {
		return last
	}

	dst := help.GetTempDir() + help.Separator() + "clean_script.txt"
	if dialogs.YesNoDialog("Are you sure you want to clean this disk? ") {
		fmt.Printf("[+] Cleaning disk %s (%s)\n", w.workstation.mount.diskNameRaw, w.workstation.mount.deviceName)
		help.CreateFile(dst)
		help.WriteFile(dst, fmt.Sprintf(cleanTemplate, w.workstation.mount.diskNameRaw))

		if help.Exists(dst) {
			if out, err := exec.Command("diskpart", "/s", dst).CombinedOutput(); err != nil {
				return err
			} else {
				log.Debug(string(out))
				fmt.Println(string(out))
				return nil
			}
		}
	}
	return nil
}

func (w *windows) PrintDisks() {
	w.workstation.printDisks(w)
}
<commit_msg>Drop windows 7 support<commit_after>package workstation

import (
	"fmt"
	"io"
	"os/exec"
	"syscall"

	"math"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"

	"encoding/csv"
	"github.com/xshellinc/tools/dialogs"
	"github.com/xshellinc/tools/lib/help"
)

const cleanTemplate = `
select disk %s
clean
create partition primary
active
assign letter=N
remove letter=N
format fs=fat32 label=KERNEL quick
`

type windows struct {
	*workstation
	ddPath string
}

// Initializes windows workstation
func newWorkstation(disk string) WorkStation {
	m := new(MountInfo)
	var ms []*MountInfo
	return &windows{&workstation{disk, runtime.GOOS, true, m, ms}, ""}
}

// Lists available mounts
func (w *windows) ListRemovableDisk() ([]*MountInfo, error) {
	log.Debug("Listing disks...")
	fmt.Println("[+] Listing available disks...")
	var out = []*MountInfo{}

	// stdout, err := help.ExecCmd("wmic", []string{"diskdrive", "get", "DeviceID,index,InterfaceType,MediaType,Model,Size", "/format:csv"})
	// ugly fix for windows 7 bug where `format:csv` is broken. Also GO double escaping quoted arguments.
	cmd := exec.Command(`cmd`)
	cmd.SysProcAttr = &syscall.SysProcAttr{}
	cmd.SysProcAttr.CmdLine = `cmd /s /c "wmic diskdrive get DeviceID,index,InterfaceType,MediaType,Model,Size /format:csv"`
	stdoutb, err := cmd.Output()
	stdout := string(stdoutb)
	log.Debug(stdout)
	if err != nil {
		stdout = ""
	}

	r := csv.NewReader(strings.NewReader(strings.TrimSpace(stdout)))
	r.TrimLeadingSpace = true
	r.Read() //skip the first line
	for {
		if record, err := r.Read(); err == io.EOF {
			break
		} else if err == nil {
			if !strings.Contains(record[4], "Removable") || strings.Contains(record[3], "IDE") {
				continue
			}
			var p = &MountInfo{}
			size := record[6]
			p.deviceSize = size
			sizeInt, _ := strconv.Atoi(size)
			sizeFloat := math.Ceil(float64(sizeInt) / 1024 / 1024 / 1024)
			p.deviceName = record[5] + " [" + strconv.Itoa(int(sizeFloat)) + "GB]"
			p.diskName = `\\?\Device\Harddisk` + record[2] + `\Partition0`
			p.diskNameRaw = record[2]
			out = append(out, p)
		}
	}
	log.WithField("out", out).Debug("got drives")
	if !(len(out) > 0) {
		return nil, fmt.Errorf("[-] No removable disks found, please insert your SD card and try again.\n[-] Please remember to run this tool as an administrator.")
	}
	w.workstation.mounts = out
	return out, nil
}

// Unmounts the disk
func (w *windows) Unmount() error {
	return nil
}

// Ejects the mounted disk
func (w *windows) Eject() error {
	return nil
}

const diskSelectionTries = 3
const writeAttempts = 5

// CopyToDisk Notifies user to choose a mount, after that it tries to copy the data
func (w *windows) CopyToDisk(img string) (job *help.BackgroundJob, err error) {
	log.Debug("CopyToDisk")
	_, err = w.ListRemovableDisk()
	if err != nil {
		fmt.Println("[-] SD card is not found, please insert an unlocked SD card")
		return nil, err
	}

	var dev *MountInfo
	if len(w.Disk) == 0 {
		rng := make([]string, len(w.workstation.mounts))
		for i, e := range w.workstation.mounts {
			rng[i] = fmt.Sprintf(dialogs.PrintColored("%s")+" - "+dialogs.PrintColored("%s")+" (%s)", e.deviceName, e.diskName, e.deviceSize)
		}
		num := dialogs.SelectOneDialog("Select disk to format: ", rng)
		dev = w.workstation.mounts[num]
	} else {
		for _, e := range w.workstation.mounts {
			if e.diskName == w.Disk {
				dev = e
				break
			}
		}
		if dev == nil {
			return nil, fmt.Errorf("Disk name not recognised, try to list disks with " + dialogs.PrintColored("disks") + " argument")
		}
	}

	w.workstation.mount = dev
	fmt.Printf("[+] Writing image to %s\n", dev.diskName)
	log.WithField("image", img).WithField("mount", "N:").Debugf("Writing image to %s", dev.diskName)

	if err := w.CleanDisk(dev.diskName); err != nil {
		return nil, err
	}

	job = help.NewBackgroundJob()
	go func() {
		defer job.Close()
		job.Active(true)
		help.ExecCmd("unzip", []string{img, "-d", "N:\\"})
		fmt.Println("\r[+] Done writing image to N:")
	}()

	return job, nil
}

// WriteToDisk notifies user to choose a mount, after that it tries to write the data with `diskSelectionTries` number of retries
func (w *windows) WriteToDisk(img string) (job *help.BackgroundJob, err error) {
	for attempt := 0; attempt < diskSelectionTries; attempt++ {
		if attempt > 0 && !dialogs.YesNoDialog("Continue?") {
			break
		}

		_, err = w.ListRemovableDisk()
		if err != nil {
			fmt.Println("[-] SD card not found, please insert an unlocked SD card")
			continue
		}
		if len(w.Disk) == 0 {
			rng := make([]string, len(w.workstation.mounts))
			for i, e := range w.workstation.mounts {
				rng[i] = fmt.Sprintf(dialogs.PrintColored("%s")+" - "+dialogs.PrintColored("%s"), e.deviceName, e.diskName)
			}
			num := dialogs.SelectOneDialog("Select disk to use: ", rng)

			w.workstation.mount = w.workstation.mounts[num]
		} else {
			for _, e := range w.workstation.mounts {
				if e.diskName == w.Disk || e.diskNameRaw == w.Disk {
					w.workstation.mount = e
					break
				}
			}
			if w.workstation.mount == nil {
				return nil, fmt.Errorf("Disk name not recognised, try to list disks with " + dialogs.PrintColored("disks") + " argument")
			}
		}
		break
	}

	if err != nil {
		return nil, err
	}

	if w.ddPath == "" {
		if err := w.getDDBinary(); err != nil {
			log.Error(err)
			fmt.Println("[-] Error downloading dd binary")
			return nil, err
		}
	}

	if len(w.Disk) == 0 && !dialogs.YesNoDialog("Are you sure? ") {
		return nil, nil
	}

	fmt.Printf("[+] Writing %s to %s\n", img, w.workstation.mount.deviceName)

	job = help.NewBackgroundJob()
	go func() {
		defer job.Close()

		var err error
		for attempt := 0; attempt < writeAttempts; attempt++ {
			if attempt > 0 {
				if !dialogs.YesNoDialog("Retry flashing?") {
					break
				}
			}
			job.Active(true)
			var out []byte
			if out, err = exec.Command(w.ddPath,
				"--filter=removable",
				fmt.Sprintf("if=%s", img),
				fmt.Sprintf("of=%s", w.workstation.mount.diskName),
				"bs=1M").CombinedOutput(); err != nil {
				log.WithField("out", string(out)).Error("Error while executing: `", w.ddPath)
				job.Active(false)
				fmt.Println("\r[-] Can't write to disk.")
			} else {
				sout := string(out)
				job.Active(false)
				log.WithField("out", sout).Debug("dd finished")
				if strings.Contains(sout, "Error ") {
					if strings.Contains(sout, "Access is denied") || strings.Contains(sout, "The device is not ready") {
						fmt.Println("\n[-] Can't write to disk. Please make sure to run this tool as administrator, close all Explorer windows, try reconnecting your disk and finally reboot your computer.\n [-] You may need to run this tool with `clean` argument to clean your disk partition table before applying image.")
						if dialogs.YesNoDialog("Or we can try to clean it's partitions right now, should we proceed?") {
							if derr := w.CleanDisk(""); derr != nil {
								fmt.Println("[-] Disk cleaning failed:", derr)
								continue
							} else {
								for !dialogs.YesNoDialog("[+] Disk formatted, now please reconnect the device. Type yes once you've done it.") {
								}
							}
						}
						continue
					} else {
						fmt.Println(sout)
						continue
					}
				}
				fmt.Printf("\r[+] Done writing %s to %s \n", img, w.workstation.mount.diskName)
				return
			}
		}

		if err != nil {
			job.Error(err)
		}

		job.Error(fmt.Errorf("Image wasn't flashed"))
	}()

	return job, nil
}

func (w *windows) getDDBinary() error {
	dst := help.GetTempDir() + help.Separator()
	url := "https://cdn.isaax.io/isaax-distro/utilities/dd/ddrelease64.zip"

	if help.Exists(dst + "ddrelease64.exe") {
		w.ddPath = dst + "ddrelease64.exe"
		return nil
	}

	wg := &sync.WaitGroup{}
	fileName, bar, err := help.DownloadFromUrlWithAttemptsAsync(url, dst, 5, wg)
	if err != nil {
		return err
	}

	bar.Prefix(fmt.Sprintf("[+] Download %-15s", fileName))
	bar.Start()
	wg.Wait()
	bar.Finish()
	time.Sleep(time.Second)

	log.WithField("dst", dst).Debug("Extracting")
	if out, err := exec.Command("unzip", "-o", dst+"ddrelease64.zip", "-d", dst).CombinedOutput(); err != nil {
		return err
	} else {
		log.Debug(string(out))
	}
	w.ddPath = dst + "ddrelease64.exe"
	return nil
}

// CleanDisk cleans target disk partitions
func (w *windows) CleanDisk(disk string) error {
	fmt.Println("[+] Cleaning disk...")
	var last error
	for attempt := 0; attempt < diskSelectionTries; attempt++ {
		if attempt > 0 && !dialogs.YesNoDialog("Continue?") {
			break
		}

		if _, err := w.ListRemovableDisk(); err != nil {
			fmt.Println("[-] SD card not found, please insert an unlocked SD card")
			last = err
			continue
		}

		rng := make([]string, len(w.workstation.mounts))
		for i, e := range w.workstation.mounts {
			rng[i] = fmt.Sprintf(dialogs.PrintColored("%s")+" - "+dialogs.PrintColored("%s"), e.deviceName, e.diskName)
		}
		num := dialogs.SelectOneDialog("Select disk to clean: ", rng)

		w.workstation.mount = w.workstation.mounts[num]
		break
	}

	if last != nil {
		return last
	}

	dst := help.GetTempDir() + help.Separator() + "clean_script.txt"
	if dialogs.YesNoDialog("Are you sure you want to clean this disk? ") {
		fmt.Printf("[+] Cleaning disk %s (%s)\n", w.workstation.mount.diskNameRaw, w.workstation.mount.deviceName)
		help.CreateFile(dst)
		help.WriteFile(dst, fmt.Sprintf(cleanTemplate, w.workstation.mount.diskNameRaw))

		if help.Exists(dst) {
			if out, err := exec.Command("diskpart", "/s", dst).CombinedOutput(); err != nil {
				return err
			} else {
				log.Debug(string(out))
				fmt.Println(string(out))
				return nil
			}
		}
	}
	return nil
}

func (w *windows) PrintDisks() {
	w.workstation.printDisks(w)
}
<|endoftext|>


<commit_before>package gocd

import (
	"encoding/json"
	"fmt"
	"strings"
)

// Equal is true if the two materials are logically equivalent. Not neccesarily literally equal.
func (m Material) Equal(a *Material) (isEqual bool, err error) {
	if m.Type != a.Type {
		return
	}

	isEqual, err = m.Attributes.equal(a.Attributes)

	return
}

// UnmarshalJSON string into a Material struct
func (m *Material) UnmarshalJSON(b []byte) error {
	temp := map[string]interface{}{}
	json.Unmarshal(b, &temp)

	var rawAttributes map[string]interface{}
	for key, value := range temp {
		if value == nil {
			continue
		}
		switch key {
		case "attributes":
			rawAttributes = value.(map[string]interface{})
		case "fingerprint":
			m.Fingerprint = value.(string)
		case "description":
			m.Description = value.(string)
		case "type":
			continue
		default:
			return fmt.Errorf("Unexpected key: '%s'", key)
		}
	}

	switch m.Type = temp["type"].(string); strings.ToLower(m.Type) {
	case "git":
		mag := &MaterialAttributesGit{}
		unmarshallMaterialAttributesGit(mag, rawAttributes)
		m.Attributes = mag
	case "svn":
		mas := &MaterialAttributesSvn{}
		unmarshallMaterialAttributesSvn(mas, rawAttributes)
		m.Attributes = mas
	case "hg":
		mah := &MaterialAttributesHg{}
		unmarshallMaterialAttributesHg(mah, rawAttributes)
		m.Attributes = mah
	case "p4":
		map4 := &MaterialAttributesP4{}
		unmarshallMaterialAttributesP4(map4, rawAttributes)
		m.Attributes = map4
	case "tfs":
		mat := &MaterialAttributesTfs{}
		unmarshallMaterialAttributesTfs(mat, rawAttributes)
		m.Attributes = mat
	case "dependency":
		mad := &MaterialAttributesDependency{}
		unmarshallMaterialAttributesDependency(mad, rawAttributes)
		m.Attributes = mad
	case "package":
		mapp := &MaterialAttributesPackage{}
		unmarshallMaterialAttributesPackage(mapp, rawAttributes)
		m.Attributes = mapp
	case "plugin":
		mapl := &MaterialAttributesPlugin{}
		unmarshallMaterialAttributesPlugin(mapl, rawAttributes)
		m.Attributes = mapl
	default:
		return fmt.Errorf("Unexpected Material type: '%s'", m.Type)
	}

	return nil
}

func unmarshallMaterialFilter(i map[string]interface{}) *MaterialFilter {
	m := &MaterialFilter{}
	if ignoreI, ok := i["ignore"]; ok {
		if ignores, ok := ignoreI.([]string); ok {
			m.Ignore = ignores
		}
	}
	return m
}
<commit_msg>Split ingestion into two functions<commit_after>package gocd

import (
	"encoding/json"
	"fmt"
	"strings"
)

// Equal is true if the two materials are logically equivalent. Not neccesarily literally equal.
func (m Material) Equal(a *Material) (isEqual bool, err error) {
	if m.Type != a.Type {
		return
	}

	isEqual, err = m.Attributes.equal(a.Attributes)

	return
}

// UnmarshalJSON string into a Material struct
func (m *Material) UnmarshalJSON(b []byte) error {
	temp := map[string]interface{}{}
	json.Unmarshal(b, &temp)

	return m.Ingest(temp)
}

// Ingest an abstract structure
func (m *Material) Ingest(payload map[string]interface{}) error {
	var rawAttributes map[string]interface{}
	for key, value := range payload {
		if value == nil {
			continue
		}
		switch key {
		case "attributes":
			rawAttributes = value.(map[string]interface{})
		case "fingerprint":
			m.Fingerprint = value.(string)
		case "description":
			m.Description = value.(string)
		case "type":
			continue
		default:
			return fmt.Errorf("Unexpected key: '%s'", key)
		}
	}

	switch m.Type = temp["type"].(string); strings.ToLower(m.Type) {
	case "git":
		mag := &MaterialAttributesGit{}
		unmarshallMaterialAttributesGit(mag, rawAttributes)
		m.Attributes = mag
	case "svn":
		mas := &MaterialAttributesSvn{}
		unmarshallMaterialAttributesSvn(mas, rawAttributes)
		m.Attributes = mas
	case "hg":
		mah := &MaterialAttributesHg{}
		unmarshallMaterialAttributesHg(mah, rawAttributes)
		m.Attributes = mah
	case "p4":
		map4 := &MaterialAttributesP4{}
		unmarshallMaterialAttributesP4(map4, rawAttributes)
		m.Attributes = map4
	case "tfs":
		mat := &MaterialAttributesTfs{}
		unmarshallMaterialAttributesTfs(mat, rawAttributes)
		m.Attributes = mat
	case "dependency":
		mad := &MaterialAttributesDependency{}
		unmarshallMaterialAttributesDependency(mad, rawAttributes)
		m.Attributes = mad
	case "package":
		mapp := &MaterialAttributesPackage{}
		unmarshallMaterialAttributesPackage(mapp, rawAttributes)
		m.Attributes = mapp
	case "plugin":
		mapl := &MaterialAttributesPlugin{}
		unmarshallMaterialAttributesPlugin(mapl, rawAttributes)
		m.Attributes = mapl
	default:
		return fmt.Errorf("Unexpected Material type: '%s'", m.Type)
	}

	return nil
}

func unmarshallMaterialFilter(i map[string]interface{}) *MaterialFilter {
	m := &MaterialFilter{}
	if ignoreI, ok := i["ignore"]; ok {
		if ignores, ok := ignoreI.([]string); ok {
			m.Ignore = ignores
		}
	}
	return m
}
<|endoftext|>


<commit_before>// connect to blobstore (S3) on AWS and perform read, write, delete, list file operations

package main

import (
	"errors"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"os"
)

var sess *session.Session
var bkt *string

func init() {
	// configuration fields //TODO: crete single object to store all the required config fields for s3. i.e the list below
	accessId := os.Getenv("s3accessid")
	accessKey := os.Getenv("secretaccesskey")
	token := ""
	region := "us-east-1"
	bkt = aws.String("jenkins19")

	cred := credentials.NewStaticCredentials(accessId, accessKey, token)
	_, err := cred.Get()
	if err != nil {
		fmt.Printf("bad credentials: %s", err)
	}

	conf := aws.NewConfig()
	conf.Credentials = cred
	conf.Region = aws.String(region)

	sess = session.Must(session.NewSession(conf))

}

func main() {

	key, err := upload("file")
	if err != nil {
		fmt.Println("upload error:", err)
	}
	fmt.Printf("uploaded %v sucessfully", key)

	keys, err := list()
	if err != nil {
		fmt.Println("list error:", err)
	}
	fmt.Printf("list sucessful %v", keys)

	for _, key := range keys {
		if err = download(key); err != nil {
			fmt.Printf("download %v error: %v", key, err)
			continue
		}
		fmt.Println("download sucessfull:", key)
	}

	for _, key := range keys {
		if err = delete(key); err != nil {
			fmt.Printf("delete %v error: %v", key, err)
			continue
		}
		fmt.Println("delete sucessfull:", key)
	}

}

func upload(filePath string) (key string, err error) {
	upload := s3manager.NewUploader(sess)
	f, err := os.Open(filePath)
	if err != nil {
		return "", err
	}
	defer f.Close()

	key = filePath
	uploadInput := s3manager.UploadInput{Bucket: bkt, Body: f, Key: aws.String(key)}
	_, err = upload.Upload(&uploadInput)
	if err != nil {
		return "", err
	}
	return key, nil
}

func download(key string) error {

	return errors.New("not implemented")
}

func delete(key string) error {

	return errors.New("not implemented")
}

func list() (keys []string, err error) {

	return nil, errors.New("not implemented")
}
<commit_msg>s3 crud wip<commit_after>// connect to blobstore (S3) on AWS and perform read, write, delete, list file operations

package main

import (
	"errors"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/s3"
	"os"
	"github.com/aws/aws-sdk-go/aws/credentials"
)

var sess *session.Session
var s3Client *s3.S3

func init() {
	// configuration fields //TODO: create single object to store all the required config fields for s3. i.e the list below
	accessId := os.Getenv("s3accessid") //TODO: need to change the credentialsenv variables to AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY
	accessKey := os.Getenv("s3accesskey")
	token := ""
	region := "us-west-1"

	//fmt.Println("access ID, Key: ", accessId, accessKey)

	cred := credentials.NewStaticCredentials(accessId, accessKey, token)
	_, err := cred.Get()
	if err != nil {
		fmt.Printf("bad credentials: %s", err)
	}

	conf := aws.NewConfig()
	conf.Credentials = cred
	conf.Region = aws.String(region)

	sess = session.Must(session.NewSession(conf))

}

func main() {

	//filePath := "s3crud2TODO.go"
	bkt := "muly123"

/*	err := createBkt(bkt)
	if err != nil {
		fmt.Println("bucket create error:", err)
	}
	fmt.Printf("bucket %v created sucessfully\n", bkt)*/

	filePath := "s3crud2TODO.go"
	key, err := upload(filePath, bkt)
	if err != nil {
		fmt.Println("upload error:", err)
		return
	}
	fmt.Printf("uploaded %v sucessfully\n", key)

	keys, err := list(bkt)
	if err != nil {
		fmt.Println("list error:", err)
		return
	}
	fmt.Printf("list sucessful %v\n", keys)

	for _, key := range keys {
		if err = download(key, bkt); err != nil {
			fmt.Printf("download %v error: %v\n", key, err)
			continue
		}
		fmt.Println("download sucessfull:", key)
	}

	for _, key := range keys {
		if err = delete(key, bkt); err != nil {
			fmt.Printf("delete %v error: %v\n", key, err)
			continue
		}
		fmt.Println("delete sucessfull:", key)
	}

}

/*func createBkt(bkt string)error{

	input := &s3.CreateBucketInput{Bucket: aws.String(bkt)}
	_, err:= s3Client.CreateBucket(input)
	if err != nil{
		return err
	}

	return nil
}*/

func upload(filePath string, bkt string) (key string, err error) {
	upload := s3manager.NewUploader(sess)
	f, err := os.Open(filePath)
	if err != nil {
		fmt.Println("error to open file")
		return "", err
	}
	defer f.Close()

	key = filePath
	uploadInput := s3manager.UploadInput{Bucket: aws.String(bkt), Body: f, Key: aws.String(key)}
	_, err = upload.Upload(&uploadInput)
	if err != nil {
		fmt.Println("error uploading file")
		return "", err
	}
	return key, nil
}

func download(key string, bkt string) error {

	return errors.New("not implemented")
}

func delete(key string, bkt string) error {

	return errors.New("not implemented")
}

func list(bkt string) (keys []string, err error) {

	return nil, errors.New("not implemented")
}
<|endoftext|>


<commit_before>package cmd

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/mithrandie/csvq/lib/file"
)

type writeTest struct {
	Name     string
	Filename string
	Content  string
	Result   string
	Error    string
}

var createFileTests = []writeTest{
	{
		Name:     "Create",
		Filename: "create.txt",
		Content:  "write",
		Result:   "write",
	},
	{
		Name:     "Output to Stdout",
		Filename: "",
		Content:  "write",
		Result:   "write",
	},
	{
		Name:     "File Exists Error",
		Filename: "create.txt",
		Error:    "environment-dependent",
	},
	{
		Name:     "File Open Error",
		Filename: filepath.Join("notexistdir", "create.txt"),
		Error:    "environment-dependent",
	},
}

func TestCreateFile(t *testing.T) {
	file.LockFiles = make(file.LockFileContainer)

	for _, v := range createFileTests {
		if len(v.Filename) < 1 {
			oldStdout := os.Stdout
			r, w, _ := os.Pipe()
			os.Stdout = w

			ToStdout(v.Content)

			w.Close()
			os.Stdout = oldStdout

			buf, _ := ioutil.ReadAll(r)
			if string(buf) != v.Result {
				t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
			}
		} else {
			filename := GetTestFilePath(v.Filename)
			err := CreateFile(filename, v.Content)
			if err != nil {
				if len(v.Error) < 1 {
					t.Errorf("%s: unexpected error %q", v.Name, err)
				} else if v.Error != "environment-dependent" && err.Error() != v.Error {
					t.Errorf("%s: error %q, want error %q", v.Name, err.Error(), v.Error)
				}
				continue
			}
			if 0 < len(v.Error) {
				t.Errorf("%s: no error, want error %q", v.Name, v.Error)
				continue
			}

			fp, _ := os.Open(filename)
			buf, _ := ioutil.ReadAll(fp)
			if string(buf) != v.Result {
				t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
			}
		}
	}

	file.UnlockAll()
}

var updateFileTests = []writeTest{
	{
		Name:     "Update",
		Filename: "create.txt",
		Content:  "truncate and write",
		Result:   "truncate and write",
	},
}

func TestUpdateFile(t *testing.T) {
	file.LockFiles = make(file.LockFileContainer)

	for _, v := range updateFileTests {
		filename := GetTestFilePath(v.Filename)
		fp, _ := file.OpenToUpdate(filename)
		err := UpdateFile(fp, v.Content)
		if err != nil {
			if len(v.Error) < 1 {
				t.Errorf("%s: unexpected error %q", v.Name, err)
			} else if v.Error != "environment-dependent" && err.Error() != v.Error {
				t.Errorf("%s: error %q, want error %q", v.Name, err.Error(), v.Error)
			}
			continue
		}
		if 0 < len(v.Error) {
			t.Errorf("%s: no error, want error %q", v.Name, v.Error)
			continue
		}

		fp, _ = os.Open(filename)
		buf, _ := ioutil.ReadAll(fp)
		if string(buf) != v.Result {
			t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
		}
	}

	file.UnlockAll()
}

func TestTryCreateFile(t *testing.T) {
	err := TryCreateFile(GetTestFilePath("table1.csv"))
	if err == nil {
		t.Error("Create table1.csv: no error, want error")
	}

	err = TryCreateFile(GetTestFilePath("notexist.csv"))
	if err != nil {
		t.Errorf("Create notexist.csv: unexpected error %q", err)
	} else {
		if _, err := os.Stat(GetTestFilePath("notexist.csv")); err == nil {
			t.Errorf("Create notexist.csv: temporary file does not removed")
		}
	}
}
<commit_msg>Fix a bug of file operation test.<commit_after>package cmd

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/mithrandie/csvq/lib/file"
)

type writeTest struct {
	Name     string
	Filename string
	Content  string
	Result   string
	Error    string
}

var createFileTests = []writeTest{
	{
		Name:     "Create",
		Filename: "create.txt",
		Content:  "write",
		Result:   "write",
	},
	{
		Name:     "Output to Stdout",
		Filename: "",
		Content:  "write",
		Result:   "write",
	},
	{
		Name:     "File Exists Error",
		Filename: "create.txt",
		Error:    "environment-dependent",
	},
	{
		Name:     "File Open Error",
		Filename: filepath.Join("notexistdir", "create.txt"),
		Error:    "environment-dependent",
	},
}

func TestCreateFile(t *testing.T) {
	file.LockFiles = make(file.LockFileContainer)

	for _, v := range createFileTests {
		if len(v.Filename) < 1 {
			oldStdout := os.Stdout
			r, w, _ := os.Pipe()
			os.Stdout = w

			ToStdout(v.Content)

			w.Close()
			os.Stdout = oldStdout

			buf, _ := ioutil.ReadAll(r)
			if string(buf) != v.Result {
				t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
			}
		} else {
			filename := GetTestFilePath(v.Filename)
			err := CreateFile(filename, v.Content)
			if err != nil {
				if len(v.Error) < 1 {
					t.Errorf("%s: unexpected error %q", v.Name, err)
				} else if v.Error != "environment-dependent" && err.Error() != v.Error {
					t.Errorf("%s: error %q, want error %q", v.Name, err.Error(), v.Error)
				}
				continue
			}
			if 0 < len(v.Error) {
				t.Errorf("%s: no error, want error %q", v.Name, v.Error)
				continue
			}

			fp, _ := os.Open(filename)
			buf, _ := ioutil.ReadAll(fp)
			if string(buf) != v.Result {
				t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
			}
		}
	}

	file.UnlockAll()
}

var updateFileTests = []writeTest{
	{
		Name:     "Update",
		Filename: "create.txt",
		Content:  "truncate and write",
		Result:   "truncate and write",
	},
}

func TestUpdateFile(t *testing.T) {
	file.LockFiles = make(file.LockFileContainer)

	for _, v := range updateFileTests {
		filename := GetTestFilePath(v.Filename)
		fp, _ := file.OpenToUpdate(filename)
		err := UpdateFile(fp, v.Content)
		file.Close(fp)
		if err != nil {
			if len(v.Error) < 1 {
				t.Errorf("%s: unexpected error %q", v.Name, err)
			} else if v.Error != "environment-dependent" && err.Error() != v.Error {
				t.Errorf("%s: error %q, want error %q", v.Name, err.Error(), v.Error)
			}
			continue
		}
		if 0 < len(v.Error) {
			t.Errorf("%s: no error, want error %q", v.Name, v.Error)
			continue
		}

		fp, _ = os.Open(filename)
		buf, _ := ioutil.ReadAll(fp)
		if string(buf) != v.Result {
			t.Errorf("%s: content = %q, want %q", v.Name, string(buf), v.Result)
		}
	}

	file.UnlockAll()
}

func TestTryCreateFile(t *testing.T) {
	err := TryCreateFile(GetTestFilePath("table1.csv"))
	if err == nil {
		t.Error("Create table1.csv: no error, want error")
	}

	err = TryCreateFile(GetTestFilePath("notexist.csv"))
	if err != nil {
		t.Errorf("Create notexist.csv: unexpected error %q", err)
	} else {
		if _, err := os.Stat(GetTestFilePath("notexist.csv")); err == nil {
			t.Errorf("Create notexist.csv: temporary file does not removed")
		}
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 86.

// Rev reverses a slice.
package main

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
)

func main() {
	//!+array
	a := [...]int{0, 1, 2, 3, 4, 5}
	reverseArray(&a)
	fmt.Println(a) // "[5 4 3 2 1 0]"
	//!-array

	//!+slice
	s := []int{0, 1, 2, 3, 4, 5}
	// Rotate s left by two positions.
	reverse(s[:2])
	reverse(s[2:])
	reverse(s)
	fmt.Println(s) // "[2 3 4 5 0 1]"
	//!-slice

	// Interactive test of reverse.
	input := bufio.NewScanner(os.Stdin)
outer:
	for input.Scan() {
		var ints []int
		for _, s := range strings.Fields(input.Text()) {
			x, err := strconv.ParseInt(s, 10, 64)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				continue outer
			}
			ints = append(ints, int(x))
		}
		reverse(ints)
		fmt.Printf("%v\n", ints)
	}
	// NOTE: ignoring potential errors from input.Err()
}

//!+rev
// reverse reverses a slice of ints in place.
func reverse(s []int) {
	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
		s[i], s[j] = s[j], s[i]
	}
}

func reverseArray(a *[6]int) {
	l := len(a)
	for i := 0; i < l/2; i++ {
		j := l - i - 1
		a[i], a[j] = a[j], a[i]
	}
}

//!-rev
<commit_msg>Exercise 4.3/4.4<commit_after>// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 86.

// Rev reverses a slice.
package main

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
)

func main() {
	//!+array
	a := [...]int{0, 1, 2, 3, 4, 5}
	reverseArray(&a)
	fmt.Println(a) // "[5 4 3 2 1 0]"
	//!-array

	//!+slice
	s := []int{0, 1, 2, 3, 4, 5}
	// Rotate s left by two positions.
	reverse(s[:2])
	reverse(s[2:])
	reverse(s)
	fmt.Println(s) // "[2 3 4 5 0 1]"
	//!-slice

	// Interactive test of reverse.
	input := bufio.NewScanner(os.Stdin)
	for input.Scan() {
		var ints []int
		for _, s := range strings.Fields(input.Text()) {
			x, err := strconv.ParseInt(s, 10, 64)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				continue outer
			}
			ints = append(ints, int(x))
		}
		reverse(ints)
		fmt.Printf("%v\n", ints)
	}
	// NOTE: ignoring potential errors from input.Err()
}

//!+rev
// reverse reverses a slice of ints in place.
func reverse(s []int) {
	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
		s[i], s[j] = s[j], s[i]
	}
}

func reverseArray(a *[6]int) {
	l := len(a)
	for i := 0; i < l/2; i++ {
		j := l - i - 1
		a[i], a[j] = a[j], a[i]
	}
}

// rotate in single pass
// exercise 4.4
func rotate(s []int) {
	tmp := s[0]:
	copy(s, s[1:])
	copy(s[:len(s)-1], tmp)
}

//!-rev
<|endoftext|>


<commit_before>package pdag

import (
	"fmt"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/go/util"

	assert "github.com/stretchr/testify/require"
)

// The simulated duration of each function in ms.
const FN_DURATION_MS = 500

type extype struct {
	data  map[string]int
	mutex sync.Mutex
}

func TestSimpleTopology(t *testing.T) {
	testutils.SmallTest(t)
	rootFn := func(ctx interface{}) error {
		d := ctx.(*extype)
		d.data["val"] = 0
		return nil
	}

	sinkFn := func(ctx interface{}) error {
		d := ctx.(*extype)
		d.data["val2"] = d.data["val"] * 100
		return nil
	}

	// Create a two node topology with a source and a sink.
	root := NewNode(rootFn)
	root.Child(sinkFn)

	// Create a context and trigger in the root node.
	d := &extype{data: map[string]int{}}
	err := root.Trigger(d)

	assert.Nil(t, err)
	assert.Equal(t, 2, len(d.data))
	assert.Equal(t, d.data["val"], 0)
	assert.Equal(t, d.data["val2"], 0)
}

func TestGenericTopology(t *testing.T) {
	testutils.SmallTest(t)
	orderCh := make(chan string, 5)

	rootFn := func(ctx interface{}) error {
		d := ctx.(*extype)
		d.data["val"] = 0
		orderCh <- "a"
		return nil
	}

	bFn := incFn(1, orderCh, "b")
	cFn := incFn(10, orderCh, "c")
	dFn := incFn(100, orderCh, "d")

	sinkFn := func(ctx interface{}) error {
		d := ctx.(*extype)
		d.data["val2"] = d.data["val"] * 100
		orderCh <- "e"
		return nil
	}

	// Create a topology that fans out to aFn, bFn, cFn and
	// then collects the results in a sink function.
	root := NewNode(rootFn)
	NewNode(sinkFn,
		root.Child(bFn),
		root.Child(cFn),
		root.Child(dFn))

	// Create a context and trigger in the root node.
	d := &extype{data: map[string]int{}}
	start := time.Now()
	err := root.Trigger(d)
	delta := time.Now().Sub(start)

	assert.Nil(t, err)

	// Make sure the functions are roughly called in parallel.
	assert.True(t, delta < (2*FN_DURATION_MS*time.Millisecond))
	assert.Equal(t, len(d.data), 2)
	assert.Equal(t, len(orderCh), 5)
	assert.Equal(t, d.data["val"], 111)
	assert.Equal(t, d.data["val2"], 11100)

	// Make sure the functions are called in the right order.
	assert.Equal(t, <-orderCh, "a")
	parallel := []string{<-orderCh, <-orderCh, <-orderCh}
	sort.Strings(parallel)
	assert.Equal(t, []string{"b", "c", "d"}, parallel)
	assert.Equal(t, <-orderCh, "e")
}

func TestError(t *testing.T) {
	testutils.SmallTest(t)
	errFn := func(c interface{}) error {
		return fmt.Errorf("Not Implemented")
	}

	root := NewNode(NoOp)
	root.Child(NoOp).
		Child(NoOp).
		Child(errFn).
		Child(NoOp)

	err := root.Trigger(nil)
	assert.NotNil(t, err)
	assert.Equal(t, "Not Implemented", err.Error())
}

func TestComplexCallOrder(t *testing.T) {
	testutils.SmallTest(t)
	aFn := orderFn("a")
	bFn := orderFn("b")
	cFn := orderFn("c")
	dFn := orderFn("d")
	eFn := orderFn("e")
	fFn := orderFn("f")
	gFn := orderFn("g")

	a := NewNode(aFn).setName("a")
	b := a.Child(bFn).setName("b")
	c := a.Child(cFn).setName("c")
	b.Child(dFn).setName("d")
	e := NewNode(eFn, b, c).setName("e")
	NewNode(fFn, b, e).setName("f")
	e.Child(gFn).setName("g")

	// Create a context and trigger in the root node.
	data := make(chan string, 100)
	a.verbose = true
	assert.NoError(t, a.Trigger(data))
	close(data)
	o := ""
	for c := range data {
		o += c
	}
	bPos := strings.Index(o, "b")
	dPos := strings.Index(o, "d")
	assert.True(t, (bPos >= 0) && (dPos > bPos))

	// make sure d is called after b
	results := map[string]bool{
		"abcefg": true,
		"abcegf": true,
		"acbefg": true,
		"acbegf": true,
	}
	o = o[0:dPos] + o[dPos+1:]
	assert.True(t, results[o])

	// Make a call an node in the DAG and make the call order works.
	data = make(chan string, 100)
	b.verbose = true
	assert.NoError(t, b.Trigger(data))
	close(data)
	o = ""
	for c := range data {
		o += c
	}
	expSet := util.NewStringSet([]string{"bedfg", "bdefg", "bedgf", "bdegf"})
	assert.True(t, expSet[o])
}

func orderFn(msg string) ProcessFn {
	return func(ctx interface{}) error {
		ctx.(chan string) <- msg
		return nil
	}
}

func incFn(increment int, ch chan<- string, chVal string) ProcessFn {
	return func(ctx interface{}) error {
		d := ctx.(*extype)
		d.mutex.Lock()
		d.data["val"] += increment
		d.mutex.Unlock()
		ch <- chVal
		time.Sleep(time.Millisecond * FN_DURATION_MS)
		return nil
	}
}
<commit_msg>[gold] Fix test in pdag<commit_after>package pdag

import (
	"fmt"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/go/util"

	assert "github.com/stretchr/testify/require"
)

// The simulated duration of each function in ms.
const FN_DURATION_MS = 500

type extype struct {
	data  map[string]int
	mutex sync.Mutex
}

func TestSimpleTopology(t *testing.T) {
	testutils.SmallTest(t)
	rootFn := func(ctx interface{}) error {
		d := ctx.(*extype)
		d.data["val"] = 0
		return nil
	}

	sinkFn := func(ctx interface{}) error {
		d := ctx.(*extype)
		d.data["val2"] = d.data["val"] * 100
		return nil
	}

	// Create a two node topology with a source and a sink.
	root := NewNode(rootFn)
	root.Child(sinkFn)

	// Create a context and trigger in the root node.
	d := &extype{data: map[string]int{}}
	err := root.Trigger(d)

	assert.Nil(t, err)
	assert.Equal(t, 2, len(d.data))
	assert.Equal(t, d.data["val"], 0)
	assert.Equal(t, d.data["val2"], 0)
}

func TestGenericTopology(t *testing.T) {
	testutils.SmallTest(t)
	orderCh := make(chan string, 5)

	rootFn := func(ctx interface{}) error {
		d := ctx.(*extype)
		d.data["val"] = 0
		orderCh <- "a"
		return nil
	}

	bFn := incFn(1, orderCh, "b")
	cFn := incFn(10, orderCh, "c")
	dFn := incFn(100, orderCh, "d")

	sinkFn := func(ctx interface{}) error {
		d := ctx.(*extype)
		d.data["val2"] = d.data["val"] * 100
		orderCh <- "e"
		return nil
	}

	// Create a topology that fans out to aFn, bFn, cFn and
	// then collects the results in a sink function.
	root := NewNode(rootFn)
	NewNode(sinkFn,
		root.Child(bFn),
		root.Child(cFn),
		root.Child(dFn))

	// Create a context and trigger in the root node.
	d := &extype{data: map[string]int{}}
	start := time.Now()
	err := root.Trigger(d)
	delta := time.Now().Sub(start)

	assert.Nil(t, err)

	// Make sure the functions are roughly called in parallel.
	assert.True(t, delta < (2*FN_DURATION_MS*time.Millisecond))
	assert.Equal(t, len(d.data), 2)
	assert.Equal(t, len(orderCh), 5)
	assert.Equal(t, d.data["val"], 111)
	assert.Equal(t, d.data["val2"], 11100)

	// Make sure the functions are called in the right order.
	assert.Equal(t, <-orderCh, "a")
	parallel := []string{<-orderCh, <-orderCh, <-orderCh}
	sort.Strings(parallel)
	assert.Equal(t, []string{"b", "c", "d"}, parallel)
	assert.Equal(t, <-orderCh, "e")
}

func TestError(t *testing.T) {
	testutils.SmallTest(t)
	errFn := func(c interface{}) error {
		return fmt.Errorf("Not Implemented")
	}

	root := NewNode(NoOp)
	root.Child(NoOp).
		Child(NoOp).
		Child(errFn).
		Child(NoOp)

	err := root.Trigger(nil)
	assert.NotNil(t, err)
	assert.Equal(t, "Not Implemented", err.Error())
}

func TestComplexCallOrder(t *testing.T) {
	testutils.SmallTest(t)
	aFn := orderFn("a")
	bFn := orderFn("b")
	cFn := orderFn("c")
	dFn := orderFn("d")
	eFn := orderFn("e")
	fFn := orderFn("f")
	gFn := orderFn("g")

	a := NewNode(aFn).setName("a")
	b := a.Child(bFn).setName("b")
	c := a.Child(cFn).setName("c")
	b.Child(dFn).setName("d")
	e := NewNode(eFn, b, c).setName("e")
	NewNode(fFn, b, e).setName("f")
	e.Child(gFn).setName("g")

	// Create a context and trigger in the root node.
	data := make(chan string, 100)
	a.verbose = true
	assert.NoError(t, a.Trigger(data))
	close(data)
	o := ""
	for c := range data {
		o += c
	}
	bPos := strings.Index(o, "b")
	dPos := strings.Index(o, "d")
	assert.True(t, (bPos >= 0) && (dPos > bPos))

	// make sure d is called after b
	results := map[string]bool{
		"abcefg": true,
		"abcegf": true,
		"acbefg": true,
		"acbegf": true,
	}
	o = o[0:dPos] + o[dPos+1:]
	assert.True(t, results[o])

	// Enumerate the possible outcome and count how often each occurs.
	posOutcome := []string{"bdegf", "bdefg", "bedgf", "bedfg", "befdg", "begdf", "begfd", "befgd"}
	expSet := util.NewStringSet(posOutcome)
	assert.Equal(t, len(posOutcome), len(expSet))

	// Make a call an node in the DAG and make the call order works.
	data = make(chan string, 100)
	b.verbose = true
	assert.NoError(t, b.Trigger(data))
	close(data)
	o = ""
	for c := range data {
		o += c
	}

	assert.True(t, expSet[o], "Instead got: "+o)
}

func orderFn(msg string) ProcessFn {
	return func(ctx interface{}) error {
		ctx.(chan string) <- msg
		return nil
	}
}

func incFn(increment int, ch chan<- string, chVal string) ProcessFn {
	return func(ctx interface{}) error {
		d := ctx.(*extype)
		d.mutex.Lock()
		d.data["val"] += increment
		d.mutex.Unlock()
		ch <- chVal
		time.Sleep(time.Millisecond * FN_DURATION_MS)
		return nil
	}
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const httpPrefix = `http://`
const portSeparator = `:`
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const traefikHeatlhCheckLabel = `traefik.backend.healthcheck.path`
const traefikPortLabel = `traefik.port`
const linkSeparator = `:`
const waitTime = 10 * time.Second

var httpClient = http.Client{ Timeout: 5 * time.Second }

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices *map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (*deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(user.Username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	readBody(pull)
	log.Print(user.Username + ` ends pulling for ` + image)
	return nil
}

func healthCheckContainers(containers []*types.ContainerJSON) {
	healthCheckSuccess := make(map[string]bool)

	for len(healthCheckSuccess) != len(containers) {
		for _, container := range containers {
			if !healthCheckSuccess[container.ID] && healthCheckContainer(container) {
				healthCheckSuccess[container.ID] = true
			}
		}

		time.Sleep(waitTime)
	}
}

func healthCheckContainer(container *types.ContainerJSON) bool {
	if container.Config.Labels[traefikHeatlhCheckLabel] != `` {
		log.Printf(`Checking health of container %s`, container.Name)

		request := http.NewRequest(`GET`, httpPrefix + container.NetworkSettings.Networks[networkMode].IPAddress + portSeparator + container.Config.Labels[traefikPortLabel] + container.Config.Labels[traefikHeatlhCheckLabel], nil)
		response, err := httpClient.Do(request)
		if err != nil {
			log.Printf(`Unable to health check for container %s : %v`, container.Name, err)
			return true
		}

		defer response.Body.Close()
		if response.StatusCode != http.StatusOK {
			log.Printf(`Health check failed for container %s : HTTP/%d`, container.Name, response.StatusCode)
			return false
		}
	}

	return true
}

func cleanContainers(containers *[]types.Container, user *auth.User) {
	for _, container := range *containers {
		log.Print(user.Username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(user.Username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]deployedService) error {
	for _, service := range *containers {
		if err := docker.ContainerRename(context.Background(), service.ID, getFinalName(service.Name)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service.Name, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(services map[string]deployedService) {
	for service, container := range services {
		if err := rmContainer(container.ID); err != nil {
			log.Printf(`Error while deleting container for %s : %v`, service, err)
		}
	}
}

func startServices(services map[string]deployedService) {
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			log.Printf(`Error while starting container for %s : %v`, service, err)
		}
	}
}

func inspectServices(services map[string]deployedService) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`Error while inspecting container for %s : %v`, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func createAppHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`Error while unmarshalling compose file: %v`, err))
		return
	}

	appNameStr := string(appName)
	log.Print(user.Username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]deployedService)

	var creationError = false
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, user); err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Printf(`%s starts %s`, user.Username, serviceFullName)

		createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(&service, user, appNameStr), getHostConfig(&service), getNetworkConfig(&service, &deployedServices), serviceFullName)
		if err != nil {
			errorHandler(w, fmt.Errorf(`Error while creating container: %v`, err))
			creationError = true
			break
		}

		deployedServices[serviceName] = deployedService{ID: createdContainer.ID, Name: serviceFullName}
	}

	if creationError {
		deleteServices(deployedServices)
		return
	}

	startServices(deployedServices)

	go func() {
		log.Printf(`Waiting for new containers to start...`)

		healthCheckContainers(inspectServices(deployedServices))
		cleanContainers(&ownerContainers, user)

		if err := renameDeployedContainers(&deployedServices); err != nil {
			log.Print(err)
		}
	}()

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<commit_msg>Update compose.go<commit_after>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const httpPrefix = `http://`
const portSeparator = `:`
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const traefikHeatlhCheckLabel = `traefik.backend.healthcheck.path`
const traefikPortLabel = `traefik.port`
const linkSeparator = `:`
const waitTime = 10 * time.Second

var httpClient = http.Client{ Timeout: 5 * time.Second }

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices *map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (*deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Print(user.Username + ` starts pulling for ` + image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`Error while pulling image: %v`, err)
	}

	readBody(pull)
	log.Print(user.Username + ` ends pulling for ` + image)
	return nil
}

func healthCheckContainers(containers []*types.ContainerJSON) {
	healthCheckSuccess := make(map[string]bool)

	for len(healthCheckSuccess) != len(containers) {
		for _, container := range containers {
			if !healthCheckSuccess[container.ID] && healthCheckContainer(container) {
				healthCheckSuccess[container.ID] = true
			}
		}

		time.Sleep(waitTime)
	}
}

func healthCheckContainer(container *types.ContainerJSON) bool {
	if container.Config.Labels[traefikHeatlhCheckLabel] != `` {
		log.Printf(`Checking health of container %s`, container.Name)

		request, err := http.NewRequest(`GET`, httpPrefix + container.NetworkSettings.Networks[networkMode].IPAddress + portSeparator + container.Config.Labels[traefikPortLabel] + container.Config.Labels[traefikHeatlhCheckLabel], nil)
		if err != nil {
			log.Printf(`Unable to prepare health check request for container %s : %v`, container.Name, err)
			return true
		}

		response, err := httpClient.Do(request)
		if err != nil {
			log.Printf(`Unable to health check for container %s : %v`, container.Name, err)
			return true
		}

		defer response.Body.Close()
		if response.StatusCode != http.StatusOK {
			log.Printf(`Health check failed for container %s : HTTP/%d`, container.Name, response.StatusCode)
			return false
		}
	}

	return true
}

func cleanContainers(containers *[]types.Container, user *auth.User) {
	for _, container := range *containers {
		log.Print(user.Username + ` stops ` + strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Print(user.Username + ` rm ` + strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers *map[string]deployedService) error {
	for _, service := range *containers {
		if err := docker.ContainerRename(context.Background(), service.ID, getFinalName(service.Name)); err != nil {
			return fmt.Errorf(`Error while renaming container %s: %v`, service.Name, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(services map[string]deployedService) {
	for service, container := range services {
		if err := rmContainer(container.ID); err != nil {
			log.Printf(`Error while deleting container for %s : %v`, service, err)
		}
	}
}

func startServices(services map[string]deployedService) {
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			log.Printf(`Error while starting container for %s : %v`, service, err)
		}
	}
}

func inspectServices(services map[string]deployedService) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`Error while inspecting container for %s : %v`, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func createAppHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		http.Error(w, `An application name and a compose file are required`, http.StatusBadRequest)
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`Error while unmarshalling compose file: %v`, err))
		return
	}

	appNameStr := string(appName)
	log.Print(user.Username + ` deploys ` + appNameStr)

	ownerContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		return
	}

	deployedServices := make(map[string]deployedService)

	var creationError = false
	for serviceName, service := range compose.Services {
		if err := pullImage(service.Image, user); err != nil {
			errorHandler(w, err)
			return
		}

		serviceFullName := getServiceFullName(appNameStr, serviceName)
		log.Printf(`%s starts %s`, user.Username, serviceFullName)

		createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(&service, user, appNameStr), getHostConfig(&service), getNetworkConfig(&service, &deployedServices), serviceFullName)
		if err != nil {
			errorHandler(w, fmt.Errorf(`Error while creating container: %v`, err))
			creationError = true
			break
		}

		deployedServices[serviceName] = deployedService{ID: createdContainer.ID, Name: serviceFullName}
	}

	if creationError {
		deleteServices(deployedServices)
		return
	}

	startServices(deployedServices)

	go func() {
		log.Printf(`Waiting for new containers to start...`)

		healthCheckContainers(inspectServices(deployedServices))
		cleanContainers(&ownerContainers, user)

		if err := renameDeployedContainers(&deployedServices); err != nil {
			log.Print(err)
		}
	}()

	jsonHttp.ResponseJSON(w, results{deployedServices})
}
<|endoftext|>


<commit_before>package imgur

import (
	"errors"
	"strings"
)

// GenericInfo is returned from functions for which the final result type is not known beforehand.
// Only one pointer is != nil
type GenericInfo struct {
	Image  *ImageInfo
	Album  *AlbumInfo
	GImage *GalleryImageInfo
	GAlbum *GalleryAlbumInfo
	Limit  *RateLimit
}

// GetInfoFromURL tries to query imgur based on information identified in the URL.
// returns image/album info, status code of the request, error
func (client *Client) GetInfoFromURL(url string) (*GenericInfo, int, error) {
	url = strings.TrimSpace(url)
	var ret GenericInfo

	// https://i.imgur.com/<id>.jpg -> image
	if strings.Contains(url, "://i.imgur.com/") {
		start := strings.LastIndex(url, "/") + 1
		end := strings.LastIndex(url, ".")
		if start == -1 || end == -1 || start == end {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down i.imgur.com path.")
		}
		id := url[start:end]
		client.Log.Debugf("Detected imgur image ID %v. Was going down the i.imgur.com/ path.", id)
		gii, status, err := client.GetGalleryImageInfo(id)
		if status < 400 {
			ret.GImage = gii
		} else {
			var ii *ImageInfo
			ii, status, err = client.GetImageInfo(id)
			ret.Image = ii
		}
		return &ret, status, err
	}

	// https://imgur.com/a/<id> -> album
	if strings.Contains(url, "://imgur.com/a/") || strings.Contains(url, "://m.imgur.com/a/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/a/ path.")
		}
		end := strings.LastIndex(url, "?")
		id := url[start:end]
		client.Log.Debugf("Detected imgur album ID %v. Was going down the imgur.com/a/ path.", id)
		ai, status, err := client.GetAlbumInfo(id)
		ret.Album = ai
		return &ret, status, err
	}

	// https://imgur.com/gallery/<id> len(id) == 5 -> gallery album
	// https://imgur.com/gallery/<id> len(id) == 7 -> gallery image
	if strings.Contains(url, "://imgur.com/gallery/") || strings.Contains(url, "://m.imgur.com/gallery/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/gallery/ path.")
		}
		end := strings.LastIndex(url, "?")
		id := url[start:end]
		client.Log.Debugf("Detected imgur gallery ID %v. Was going down the imgur.com/gallery/ path.", id)
		if len(id) == 5 {
			client.Log.Debugf("Detected imgur gallery album.")
			ai, status, err := client.GetGalleryAlbumInfo(id)
			ret.GAlbum = ai
			return &ret, status, err
		}

		ii, status, err := client.GetGalleryImageInfo(id)
		ret.GImage = ii
		return &ret, status, err
	}

	// https://imgur.com/<id> -> image
	if strings.Contains(url, "://imgur.com/") || strings.Contains(url, "://m.imgur.com/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/ path.")
		}
		end := strings.LastIndex(url, "?")
		if end == -1 {
			end = len(url)
		}
		id := url[start:end]
		client.Log.Debugf("Detected imgur image ID %v. Was going down the imgur.com/ path.", id)
		ii, status, err := client.GetGalleryImageInfo(id)
		ret.GImage = ii
		return &ret, status, err
	}

	return nil, -1, errors.New("URL pattern matching for URL " + url + " failed.")
}
<commit_msg>Fixed out of bounds error in GetInfoFromURL.<commit_after>package imgur

import (
	"errors"
	"strings"
)

// GenericInfo is returned from functions for which the final result type is not known beforehand.
// Only one pointer is != nil
type GenericInfo struct {
	Image  *ImageInfo
	Album  *AlbumInfo
	GImage *GalleryImageInfo
	GAlbum *GalleryAlbumInfo
	Limit  *RateLimit
}

// GetInfoFromURL tries to query imgur based on information identified in the URL.
// returns image/album info, status code of the request, error
func (client *Client) GetInfoFromURL(url string) (*GenericInfo, int, error) {
	url = strings.TrimSpace(url)
	var ret GenericInfo

	// https://i.imgur.com/<id>.jpg -> image
	if strings.Contains(url, "://i.imgur.com/") {
		start := strings.LastIndex(url, "/") + 1
		end := strings.LastIndex(url, ".")
		if start == -1 || end == -1 || start == end {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down i.imgur.com path.")
		}
		id := url[start:end]
		client.Log.Debugf("Detected imgur image ID %v. Was going down the i.imgur.com/ path.", id)
		gii, status, err := client.GetGalleryImageInfo(id)
		if status < 400 {
			ret.GImage = gii
		} else {
			var ii *ImageInfo
			ii, status, err = client.GetImageInfo(id)
			ret.Image = ii
		}
		return &ret, status, err
	}

	// https://imgur.com/a/<id> -> album
	if strings.Contains(url, "://imgur.com/a/") || strings.Contains(url, "://m.imgur.com/a/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/a/ path.")
		}
		end := strings.LastIndex(url, "?")
		if end == -1 {
			end = len(url)
		}
		id := url[start:end]
		client.Log.Debugf("Detected imgur album ID %v. Was going down the imgur.com/a/ path.", id)
		ai, status, err := client.GetAlbumInfo(id)
		ret.Album = ai
		return &ret, status, err
	}

	// https://imgur.com/gallery/<id> len(id) == 5 -> gallery album
	// https://imgur.com/gallery/<id> len(id) == 7 -> gallery image
	if strings.Contains(url, "://imgur.com/gallery/") || strings.Contains(url, "://m.imgur.com/gallery/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/gallery/ path.")
		}
		end := strings.LastIndex(url, "?")
		if end == -1 {
			end = len(url)
		}
		id := url[start:end]
		client.Log.Debugf("Detected imgur gallery ID %v. Was going down the imgur.com/gallery/ path.", id)
		if len(id) == 5 {
			client.Log.Debugf("Detected imgur gallery album.")
			ai, status, err := client.GetGalleryAlbumInfo(id)
			ret.GAlbum = ai
			return &ret, status, err
		}

		ii, status, err := client.GetGalleryImageInfo(id)
		ret.GImage = ii
		return &ret, status, err
	}

	// https://imgur.com/<id> -> image
	if strings.Contains(url, "://imgur.com/") || strings.Contains(url, "://m.imgur.com/") {
		start := strings.LastIndex(url, "/") + 1
		if start == -1 {
			return nil, -1, errors.New("Could not find ID in URL " + url + ". I was going down imgur.com/ path.")
		}
		end := strings.LastIndex(url, "?")
		if end == -1 {
			end = len(url)
		}
		id := url[start:end]
		client.Log.Debugf("Detected imgur image ID %v. Was going down the imgur.com/ path.", id)
		ii, status, err := client.GetGalleryImageInfo(id)
		ret.GImage = ii
		return &ret, status, err
	}

	return nil, -1, errors.New("URL pattern matching for URL " + url + " failed.")
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package fs

import (
	"bytes"
	"context"
	"io"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
)

var (
	errTooManyLinks = errors.New("too many links")
)

type currentPath struct {
	path     string
	f        os.FileInfo
	fullPath string
}

func pathChange(lower, upper *currentPath) (ChangeKind, string) {
	if lower == nil {
		if upper == nil {
			panic("cannot compare nil paths")
		}
		return ChangeKindAdd, upper.path
	}
	if upper == nil {
		return ChangeKindDelete, lower.path
	}

	switch i := directoryCompare(lower.path, upper.path); {
	case i < 0:
		// File in lower that is not in upper
		return ChangeKindDelete, lower.path
	case i > 0:
		// File in upper that is not in lower
		return ChangeKindAdd, upper.path
	default:
		return ChangeKindModify, upper.path
	}
}

func directoryCompare(a, b string) int {
	l := len(a)
	if len(b) < l {
		l = len(b)
	}
	for i := 0; i < l; i++ {
		c1, c2 := a[i], b[i]
		if c1 == filepath.Separator {
			c1 = byte(0)
		}
		if c2 == filepath.Separator {
			c2 = byte(0)
		}
		if c1 < c2 {
			return -1
		}
		if c1 > c2 {
			return +1
		}
	}
	if len(a) < len(b) {
		return -1
	}
	if len(a) > len(b) {
		return +1
	}
	return 0
}

func sameFile(f1, f2 *currentPath) (bool, error) {
	if os.SameFile(f1.f, f2.f) {
		return true, nil
	}

	equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys())
	if err != nil || !equalStat {
		return equalStat, err
	}

	if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq {
		return eq, err
	}

	// If not a directory also check size, modtime, and content
	if !f1.f.IsDir() {
		if f1.f.Size() != f2.f.Size() {
			return false, nil
		}
		t1 := f1.f.ModTime()
		t2 := f2.f.ModTime()

		if t1.Unix() != t2.Unix() {
			return false, nil
		}

		// If the timestamp may have been truncated in both of the
		// files, check content of file to determine difference
		if t1.Nanosecond() == 0 && t2.Nanosecond() == 0 {
			var eq bool
			if (f1.f.Mode() & os.ModeSymlink) == os.ModeSymlink {
				eq, err = compareSymlinkTarget(f1.fullPath, f2.fullPath)
			} else if f1.f.Size() > 0 {
				eq, err = compareFileContent(f1.fullPath, f2.fullPath)
			} else {
				eq, err = true, nil
			}
			if err != nil || !eq {
				return eq, err
			}
		} else if t1.Nanosecond() != t2.Nanosecond() {
			return false, nil
		}
	}

	return true, nil
}

func compareSymlinkTarget(p1, p2 string) (bool, error) {
	t1, err := os.Readlink(p1)
	if err != nil {
		return false, err
	}
	t2, err := os.Readlink(p2)
	if err != nil {
		return false, err
	}
	return t1 == t2, nil
}

const compareChuckSize = 32 * 1024

// compareFileContent compares the content of 2 same sized files
// by comparing each byte.
func compareFileContent(p1, p2 string) (bool, error) {
	f1, err := os.Open(p1)
	if err != nil {
		return false, err
	}
	defer f1.Close()
	f2, err := os.Open(p2)
	if err != nil {
		return false, err
	}
	defer f2.Close()

	b1 := make([]byte, compareChuckSize)
	b2 := make([]byte, compareChuckSize)
	for {
		n1, err1 := f1.Read(b1)
		if err1 != nil && err1 != io.EOF {
			return false, err1
		}
		n2, err2 := f2.Read(b2)
		if err2 != nil && err2 != io.EOF {
			return false, err2
		}
		if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) {
			return false, nil
		}
		if err1 == io.EOF && err2 == io.EOF {
			return true, nil
		}
	}
}

func pathWalk(ctx context.Context, root string, pathC chan<- *currentPath) error {
	return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		path, err = filepath.Rel(root, path)
		if err != nil {
			return err
		}

		path = filepath.Join(string(os.PathSeparator), path)

		// Skip root
		if path == string(os.PathSeparator) {
			return nil
		}

		p := &currentPath{
			path:     path,
			f:        f,
			fullPath: filepath.Join(root, path),
		}

		select {
		case <-ctx.Done():
			return ctx.Err()
		case pathC <- p:
			return nil
		}
	})
}

func nextPath(ctx context.Context, pathC <-chan *currentPath) (*currentPath, error) {
	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case p := <-pathC:
		return p, nil
	}
}

// RootPath joins a path with a root, evaluating and bounding any
// symlink to the root directory.
func RootPath(root, path string) (string, error) {
	if path == "" {
		return root, nil
	}
	var linksWalked int // to protect against cycles
	for {
		i := linksWalked
		newpath, err := walkLinks(root, path, &linksWalked)
		if err != nil {
			return "", err
		}
		path = newpath
		if i == linksWalked {
			newpath = filepath.Join("/", newpath)
			if path == newpath {
				return filepath.Join(root, newpath), nil
			}
			path = newpath
		}
	}
}

func walkLink(root, path string, linksWalked *int) (newpath string, islink bool, err error) {
	if *linksWalked > 255 {
		return "", false, errTooManyLinks
	}

	path = filepath.Join("/", path)
	if path == "/" {
		return path, false, nil
	}
	realPath := filepath.Join(root, path)

	fi, err := os.Lstat(realPath)
	if err != nil {
		// If path does not yet exist, treat as non-symlink
		if os.IsNotExist(err) {
			return path, false, nil
		}
		return "", false, err
	}
	if fi.Mode()&os.ModeSymlink == 0 {
		return path, false, nil
	}
	newpath, err = os.Readlink(realPath)
	if err != nil {
		return "", false, err
	}
	*linksWalked++
	return newpath, true, nil
}

func walkLinks(root, path string, linksWalked *int) (string, error) {
	switch dir, file := filepath.Split(path); {
	case dir == "":
		newpath, _, err := walkLink(root, file, linksWalked)
		return newpath, err
	case file == "":
		if os.IsPathSeparator(dir[len(dir)-1]) {
			if dir == "/" {
				return dir, nil
			}
			return walkLinks(root, dir[:len(dir)-1], linksWalked)
		}
		newpath, _, err := walkLink(root, dir, linksWalked)
		return newpath, err
	default:
		newdir, err := walkLinks(root, dir, linksWalked)
		if err != nil {
			return "", err
		}
		newpath, islink, err := walkLink(root, filepath.Join(newdir, file), linksWalked)
		if err != nil {
			return "", err
		}
		if !islink {
			return newpath, nil
		}
		if filepath.IsAbs(newpath) {
			return newpath, nil
		}
		return filepath.Join(newdir, newpath), nil
	}
}
<commit_msg>Add a comment to clarify that we're handling the empty file case<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package fs

import (
	"bytes"
	"context"
	"io"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
)

var (
	errTooManyLinks = errors.New("too many links")
)

type currentPath struct {
	path     string
	f        os.FileInfo
	fullPath string
}

func pathChange(lower, upper *currentPath) (ChangeKind, string) {
	if lower == nil {
		if upper == nil {
			panic("cannot compare nil paths")
		}
		return ChangeKindAdd, upper.path
	}
	if upper == nil {
		return ChangeKindDelete, lower.path
	}

	switch i := directoryCompare(lower.path, upper.path); {
	case i < 0:
		// File in lower that is not in upper
		return ChangeKindDelete, lower.path
	case i > 0:
		// File in upper that is not in lower
		return ChangeKindAdd, upper.path
	default:
		return ChangeKindModify, upper.path
	}
}

func directoryCompare(a, b string) int {
	l := len(a)
	if len(b) < l {
		l = len(b)
	}
	for i := 0; i < l; i++ {
		c1, c2 := a[i], b[i]
		if c1 == filepath.Separator {
			c1 = byte(0)
		}
		if c2 == filepath.Separator {
			c2 = byte(0)
		}
		if c1 < c2 {
			return -1
		}
		if c1 > c2 {
			return +1
		}
	}
	if len(a) < len(b) {
		return -1
	}
	if len(a) > len(b) {
		return +1
	}
	return 0
}

func sameFile(f1, f2 *currentPath) (bool, error) {
	if os.SameFile(f1.f, f2.f) {
		return true, nil
	}

	equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys())
	if err != nil || !equalStat {
		return equalStat, err
	}

	if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq {
		return eq, err
	}

	// If not a directory also check size, modtime, and content
	if !f1.f.IsDir() {
		if f1.f.Size() != f2.f.Size() {
			return false, nil
		}
		t1 := f1.f.ModTime()
		t2 := f2.f.ModTime()

		if t1.Unix() != t2.Unix() {
			return false, nil
		}

		// If the timestamp may have been truncated in both of the
		// files, check content of file to determine difference
		if t1.Nanosecond() == 0 && t2.Nanosecond() == 0 {
			var eq bool
			if (f1.f.Mode() & os.ModeSymlink) == os.ModeSymlink {
				eq, err = compareSymlinkTarget(f1.fullPath, f2.fullPath)
			} else if f1.f.Size() > 0 {
				eq, err = compareFileContent(f1.fullPath, f2.fullPath)
			} else {
				eq, err = true, nil // if file sizes are zero length, the files are the same by definition
			}
			if err != nil || !eq {
				return eq, err
			}
		} else if t1.Nanosecond() != t2.Nanosecond() {
			return false, nil
		}
	}

	return true, nil
}

func compareSymlinkTarget(p1, p2 string) (bool, error) {
	t1, err := os.Readlink(p1)
	if err != nil {
		return false, err
	}
	t2, err := os.Readlink(p2)
	if err != nil {
		return false, err
	}
	return t1 == t2, nil
}

const compareChuckSize = 32 * 1024

// compareFileContent compares the content of 2 same sized files
// by comparing each byte.
func compareFileContent(p1, p2 string) (bool, error) {
	f1, err := os.Open(p1)
	if err != nil {
		return false, err
	}
	defer f1.Close()
	f2, err := os.Open(p2)
	if err != nil {
		return false, err
	}
	defer f2.Close()

	b1 := make([]byte, compareChuckSize)
	b2 := make([]byte, compareChuckSize)
	for {
		n1, err1 := f1.Read(b1)
		if err1 != nil && err1 != io.EOF {
			return false, err1
		}
		n2, err2 := f2.Read(b2)
		if err2 != nil && err2 != io.EOF {
			return false, err2
		}
		if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) {
			return false, nil
		}
		if err1 == io.EOF && err2 == io.EOF {
			return true, nil
		}
	}
}

func pathWalk(ctx context.Context, root string, pathC chan<- *currentPath) error {
	return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		path, err = filepath.Rel(root, path)
		if err != nil {
			return err
		}

		path = filepath.Join(string(os.PathSeparator), path)

		// Skip root
		if path == string(os.PathSeparator) {
			return nil
		}

		p := &currentPath{
			path:     path,
			f:        f,
			fullPath: filepath.Join(root, path),
		}

		select {
		case <-ctx.Done():
			return ctx.Err()
		case pathC <- p:
			return nil
		}
	})
}

func nextPath(ctx context.Context, pathC <-chan *currentPath) (*currentPath, error) {
	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case p := <-pathC:
		return p, nil
	}
}

// RootPath joins a path with a root, evaluating and bounding any
// symlink to the root directory.
func RootPath(root, path string) (string, error) {
	if path == "" {
		return root, nil
	}
	var linksWalked int // to protect against cycles
	for {
		i := linksWalked
		newpath, err := walkLinks(root, path, &linksWalked)
		if err != nil {
			return "", err
		}
		path = newpath
		if i == linksWalked {
			newpath = filepath.Join("/", newpath)
			if path == newpath {
				return filepath.Join(root, newpath), nil
			}
			path = newpath
		}
	}
}

func walkLink(root, path string, linksWalked *int) (newpath string, islink bool, err error) {
	if *linksWalked > 255 {
		return "", false, errTooManyLinks
	}

	path = filepath.Join("/", path)
	if path == "/" {
		return path, false, nil
	}
	realPath := filepath.Join(root, path)

	fi, err := os.Lstat(realPath)
	if err != nil {
		// If path does not yet exist, treat as non-symlink
		if os.IsNotExist(err) {
			return path, false, nil
		}
		return "", false, err
	}
	if fi.Mode()&os.ModeSymlink == 0 {
		return path, false, nil
	}
	newpath, err = os.Readlink(realPath)
	if err != nil {
		return "", false, err
	}
	*linksWalked++
	return newpath, true, nil
}

func walkLinks(root, path string, linksWalked *int) (string, error) {
	switch dir, file := filepath.Split(path); {
	case dir == "":
		newpath, _, err := walkLink(root, file, linksWalked)
		return newpath, err
	case file == "":
		if os.IsPathSeparator(dir[len(dir)-1]) {
			if dir == "/" {
				return dir, nil
			}
			return walkLinks(root, dir[:len(dir)-1], linksWalked)
		}
		newpath, _, err := walkLink(root, dir, linksWalked)
		return newpath, err
	default:
		newdir, err := walkLinks(root, dir, linksWalked)
		if err != nil {
			return "", err
		}
		newpath, islink, err := walkLink(root, filepath.Join(newdir, file), linksWalked)
		if err != nil {
			return "", err
		}
		if !islink {
			return newpath, nil
		}
		if filepath.IsAbs(newpath) {
			return newpath, nil
		}
		return filepath.Join(newdir, newpath), nil
	}
}
<|endoftext|>