<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"strings"
	"time"
)

// BASE APP CLASS

type TwitchNotifierMain struct {
	need_channels_refresh   bool
	_auth_oauth             string
	krakenInstance          *Kraken
	options                 *Options
	windows_balloon_tip_obj WindowsBalloonTipInterface
	mainEventsInterface     MainEventsInterface
	queryPageSize           uint
}

func InitTwitchNotifierMain() *TwitchNotifierMain {
	out := &TwitchNotifierMain{}

	msg("init kraken")
	out.krakenInstance = InitKraken()

	out.krakenInstance.addHeader("Accept", "application/vnd.twitchtv.v3+json")

	out.need_channels_refresh = true
	out._auth_oauth = ""
	out.queryPageSize = 25

	return out
}

func (app *TwitchNotifierMain) need_browser_auth() bool {
	msg("options.no_browser_auth %s", app.options.no_browser_auth)
	if app.options.no_browser_auth != nil {
		msg("options.no_browser_auth %s", *app.options.no_browser_auth)
	}
	msg("app._auth_oauth %s", app._auth_oauth)
	return (app.options.no_browser_auth == nil || !*app.options.no_browser_auth) && app._auth_oauth == ""
}

func (app *TwitchNotifierMain) log(msg string) {
	if app.options.debug_output != nil && *app.options.debug_output {
		log.Printf("%s TwitchNotifierMain: %s\n", time.Now(), msg)
	}
}

// API RESPONSE DATA STRUCTURES

type ChannelID float64
type StreamID float64

// Info about a channel
type ChannelInfo struct {
	Id           ChannelID `json:"_id"`
	Display_Name string
	Url          string
	Status       string
	// URL of the channel logo, if any
	Logo *string
}

// Info about a stream, which is a current live video session happening on a channel
type StreamInfo struct {
	Channel     *ChannelInfo
	Is_playlist bool
	Id          StreamID `json:"_id"`
	Created_at  string
	Game        *string
}

// Pair of stream and channel for maps
type StreamChannel struct {
	stream  *StreamInfo
	channel *ChannelInfo
}

// This is for "virtual functions" in the base app class that should go through the extended app class
type MainEventsInterface interface {
	init_channel_display(followed_channel_entries []*ChannelInfo)
	stream_state_change(channel_id ChannelID, stream_we_consider_online bool, stream *StreamInfo)
	assume_all_streams_offline()
	done_state_changes()
	_channels_reload_complete()
	log(msg string)
}

// MORE APP METHODS

// MainEventsInterface stubs
func (app *TwitchNotifierMain) init_channel_display(followed_channel_entries []*ChannelInfo) {

}

func (app *TwitchNotifierMain) stream_state_change(channel_id ChannelID, stream_we_consider_online bool, stream *StreamInfo) {

}

func (app *TwitchNotifierMain) assume_all_streams_offline() {

}

func (app *TwitchNotifierMain) done_state_changes() {

}

func (app *TwitchNotifierMain) _channels_reload_complete() {

}

func (app *TwitchNotifierMain) getEventsInterface() MainEventsInterface {
	// Get the MainEventsInterface registered in app or fall back to its own
	if app.mainEventsInterface != nil {
		return app.mainEventsInterface
	} else {
		return app
	}
}

func (app *TwitchNotifierMain) channel_display_name(channel *ChannelInfo) string {
	return fmt.Sprintf("%s (%v)", channel.Display_Name, uint32(channel.Id))
}

func (app *TwitchNotifierMain) create_show_info_suffix(stream *StreamInfo) string {
	game := stream.Game

	var show_info string
	if game == nil || *game == "" {
		show_info = ""
	} else {
		show_info = fmt.Sprintf("with %s ", *game)
	}
	return show_info
}

func (app *TwitchNotifierMain) create_online_event_message(channel_name string, stream *StreamInfo) string {
	return fmt.Sprintf("%s went online %s", channel_name, app.create_show_info_suffix(stream))
}

func (app *TwitchNotifierMain) create_offline_event_message(channel_name string) string {
	return fmt.Sprintf("%s went offline", channel_name)
}

func (app *TwitchNotifierMain) create_online_message(channel_name string, stream *StreamInfo) string {
	created_at := stream.Created_at
	start_time, err := convert_rfc3339_time(created_at)
	assert(err == nil, "Error converting time %s", created_at)
	elapsed_s := time.Now().Round(time.Second).Sub(start_time.Round(time.Second))

	show_info := app.create_show_info_suffix(stream)

	message := fmt.Sprintf("%s is now live %s(up %s)", channel_name, show_info, time_desc(elapsed_s))
	return message
}

// Create a notification for the given stream
func (app *TwitchNotifierMain) notify_for_stream(channel_name string, stream *StreamInfo) {

	message := app.create_online_message(channel_name, stream)
	msg("Showing message: '%s'", message)
	stream_browser_link := stream.Channel.Url

	// Supply a callback to handle the event where the notification was clicked
	callback := NotificationCallback{channel_name, stream_browser_link}

	popupsEnabled := true
	if app.options.no_popups != nil {
		popupsEnabled = !*app.options.no_popups
	}

	if popupsEnabled && app.windows_balloon_tip_obj != nil {
		app.windows_balloon_tip_obj.balloon_tip("twitch-notifier-go", message, callback, stream_browser_link)
	}
}

// Interface for a desktop notification provider
type WindowsBalloonTipInterface interface {
	balloon_tip(title string, message string, callback NotificationCallback, url string)
}

type OurWindowsBalloonTip struct {
	main_window *MainStatusWindowImpl
}

func NewOurWindowsBalloonTip(main_window *MainStatusWindowImpl) *OurWindowsBalloonTip {
	return &OurWindowsBalloonTip{main_window}
}

func (tip *OurWindowsBalloonTip) balloon_tip(title string, msg string, callback NotificationCallback, url string) {
	tip.main_window.enqueue_notification(title, msg, callback, url)
}

type NotificationCallback struct {
	channel_name        string
	stream_browser_link string
}

func (callback NotificationCallback) callback() error {
	fmt.Printf("notification for %s clicked\n", callback.channel_name)
	webbrowser_open(callback.stream_browser_link)
	return nil
}

func (app *TwitchNotifierMain) diag_request(parts ...string) {
	/** Do an API call and just pretty print the response contents */
	url_parts := strings.Join(parts, "/")
	msg("diag request for %s", url_parts)

	var output *map[string]interface{} = &map[string]interface{}{}
	err := app.krakenInstance.kraken(output, parts...)
	assert(err == nil, "diag request of %s failed with %s", url_parts, err)
	prettyOutput, err2 := json.MarshalIndent(output, "", "  ")
	assert(err == nil, "error remarshalling the json return value: %s", err2)
	msg("Output of request %s was %s", url_parts, prettyOutput)
}

func (app *TwitchNotifierMain) nextWithRetry(pager *KrakenPager, val interface{}, httpErrorTries uint) error {
	var err error
	for httpErrorTries > 0 {
		err = pager.Next(val)
		if err != nil {
			krakenError, wasKrakenError := err.(*KrakenError)
			if wasKrakenError && krakenError != nil {
				if krakenError.statusCode != 200 {
					httpErrorTries -= 1
					app.getEventsInterface().log(fmt.Sprintf("Got HTTP error %v while loading item; tries left %v", krakenError.statusCode, httpErrorTries))
					continue
				}
			}
		}
		break
	}
	return err
}

func (app *TwitchNotifierMain) PagedKrakenWithRetry(httpErrorTries uint, resultsListKey string, pageSize uint, addParams *url.Values, path ...string) (*KrakenPager, error) {
	var err error
	var pager *KrakenPager
	for httpErrorTries > 0 {
		pager, err = app.krakenInstance.PagedKraken(resultsListKey, pageSize, addParams, path...)
		if err != nil {
			krakenError, wasKrakenError := err.(*KrakenError)
			if wasKrakenError && krakenError != nil {
				if krakenError.statusCode != 200 {
					httpErrorTries -= 1
					app.getEventsInterface().log(fmt.Sprintf("Got HTTP error %v while doing initial pager request; tries left %v", krakenError.statusCode, httpErrorTries))
					continue
				}
			}
		}
		break
	}
	return pager, err
}

func (app *TwitchNotifierMain) get_streams_channels_following(followed_channels map[ChannelID]bool) (map[ChannelID]StreamChannel, error) {
	out := map[ChannelID]StreamChannel{}

	initialRequestHTTPTries := uint(2)

	additionalParams := make(url.Values)
	additionalParams.Add("stream_type", "live")
	pager, err := app.PagedKrakenWithRetry(initialRequestHTTPTries, "streams", app.queryPageSize, &additionalParams, "streams", "followed")

	if err != nil {
		return out, err
	}

	for pager.More() {
		var stream *StreamInfo
		httpErrorTries := uint(2)
		err := app.nextWithRetry(pager, &stream, httpErrorTries)
		if err != nil {
			return out, err
		}

		assert(err == nil, "next return error: %s", err)
		assert(stream != nil, "stream was nil")
		channel := stream.Channel
		assert(channel != nil, "stream has no channel")
		channel_id := stream.Channel.Id
		val, ok := followed_channels[channel_id]
		if val && ok {
			out[channel_id] = StreamChannel{stream, channel}
		} else {
			app.getEventsInterface().log(fmt.Sprintf("skipping channel %s because it's not a followed channel", app.channel_display_name(stream.Channel)))
		}
	}

	return out, nil
}

func (app *TwitchNotifierMain) main_loop() {

}
<commit_msg>don't show numerical stream IDs<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"strings"
	"time"
)

// BASE APP CLASS

type TwitchNotifierMain struct {
	need_channels_refresh   bool
	_auth_oauth             string
	krakenInstance          *Kraken
	options                 *Options
	windows_balloon_tip_obj WindowsBalloonTipInterface
	mainEventsInterface     MainEventsInterface
	queryPageSize           uint
}

func InitTwitchNotifierMain() *TwitchNotifierMain {
	out := &TwitchNotifierMain{}

	msg("init kraken")
	out.krakenInstance = InitKraken()

	out.krakenInstance.addHeader("Accept", "application/vnd.twitchtv.v3+json")

	out.need_channels_refresh = true
	out._auth_oauth = ""
	out.queryPageSize = 25

	return out
}

func (app *TwitchNotifierMain) need_browser_auth() bool {
	msg("options.no_browser_auth %s", app.options.no_browser_auth)
	if app.options.no_browser_auth != nil {
		msg("options.no_browser_auth %s", *app.options.no_browser_auth)
	}
	msg("app._auth_oauth %s", app._auth_oauth)
	return (app.options.no_browser_auth == nil || !*app.options.no_browser_auth) && app._auth_oauth == ""
}

func (app *TwitchNotifierMain) log(msg string) {
	if app.options.debug_output != nil && *app.options.debug_output {
		log.Printf("%s TwitchNotifierMain: %s\n", time.Now(), msg)
	}
}

// API RESPONSE DATA STRUCTURES

type ChannelID float64
type StreamID float64

// Info about a channel
type ChannelInfo struct {
	Id           ChannelID `json:"_id"`
	Display_Name string
	Url          string
	Status       string
	// URL of the channel logo, if any
	Logo *string
}

// Info about a stream, which is a current live video session happening on a channel
type StreamInfo struct {
	Channel     *ChannelInfo
	Is_playlist bool
	Id          StreamID `json:"_id"`
	Created_at  string
	Game        *string
}

// Pair of stream and channel for maps
type StreamChannel struct {
	stream  *StreamInfo
	channel *ChannelInfo
}

// This is for "virtual functions" in the base app class that should go through the extended app class
type MainEventsInterface interface {
	init_channel_display(followed_channel_entries []*ChannelInfo)
	stream_state_change(channel_id ChannelID, stream_we_consider_online bool, stream *StreamInfo)
	assume_all_streams_offline()
	done_state_changes()
	_channels_reload_complete()
	log(msg string)
}

// MORE APP METHODS

// MainEventsInterface stubs
func (app *TwitchNotifierMain) init_channel_display(followed_channel_entries []*ChannelInfo) {

}

func (app *TwitchNotifierMain) stream_state_change(channel_id ChannelID, stream_we_consider_online bool, stream *StreamInfo) {

}

func (app *TwitchNotifierMain) assume_all_streams_offline() {

}

func (app *TwitchNotifierMain) done_state_changes() {

}

func (app *TwitchNotifierMain) _channels_reload_complete() {

}

func (app *TwitchNotifierMain) getEventsInterface() MainEventsInterface {
	// Get the MainEventsInterface registered in app or fall back to its own
	if app.mainEventsInterface != nil {
		return app.mainEventsInterface
	} else {
		return app
	}
}

func (app *TwitchNotifierMain) channel_display_name(channel *ChannelInfo) string {
	return fmt.Sprintf("%s", channel.Display_Name)
}

func (app *TwitchNotifierMain) create_show_info_suffix(stream *StreamInfo) string {
	game := stream.Game

	var show_info string
	if game == nil || *game == "" {
		show_info = ""
	} else {
		show_info = fmt.Sprintf("with %s ", *game)
	}
	return show_info
}

func (app *TwitchNotifierMain) create_online_event_message(channel_name string, stream *StreamInfo) string {
	return fmt.Sprintf("%s went online %s", channel_name, app.create_show_info_suffix(stream))
}

func (app *TwitchNotifierMain) create_offline_event_message(channel_name string) string {
	return fmt.Sprintf("%s went offline", channel_name)
}

func (app *TwitchNotifierMain) create_online_message(channel_name string, stream *StreamInfo) string {
	created_at := stream.Created_at
	start_time, err := convert_rfc3339_time(created_at)
	assert(err == nil, "Error converting time %s", created_at)
	elapsed_s := time.Now().Round(time.Second).Sub(start_time.Round(time.Second))

	show_info := app.create_show_info_suffix(stream)

	message := fmt.Sprintf("%s is now live %s(up %s)", channel_name, show_info, time_desc(elapsed_s))
	return message
}

// Create a notification for the given stream
func (app *TwitchNotifierMain) notify_for_stream(channel_name string, stream *StreamInfo) {

	message := app.create_online_message(channel_name, stream)
	msg("Showing message: '%s'", message)
	stream_browser_link := stream.Channel.Url

	// Supply a callback to handle the event where the notification was clicked
	callback := NotificationCallback{channel_name, stream_browser_link}

	popupsEnabled := true
	if app.options.no_popups != nil {
		popupsEnabled = !*app.options.no_popups
	}

	if popupsEnabled && app.windows_balloon_tip_obj != nil {
		app.windows_balloon_tip_obj.balloon_tip("twitch-notifier-go", message, callback, stream_browser_link)
	}
}

// Interface for a desktop notification provider
type WindowsBalloonTipInterface interface {
	balloon_tip(title string, message string, callback NotificationCallback, url string)
}

type OurWindowsBalloonTip struct {
	main_window *MainStatusWindowImpl
}

func NewOurWindowsBalloonTip(main_window *MainStatusWindowImpl) *OurWindowsBalloonTip {
	return &OurWindowsBalloonTip{main_window}
}

func (tip *OurWindowsBalloonTip) balloon_tip(title string, msg string, callback NotificationCallback, url string) {
	tip.main_window.enqueue_notification(title, msg, callback, url)
}

type NotificationCallback struct {
	channel_name        string
	stream_browser_link string
}

func (callback NotificationCallback) callback() error {
	fmt.Printf("notification for %s clicked\n", callback.channel_name)
	webbrowser_open(callback.stream_browser_link)
	return nil
}

func (app *TwitchNotifierMain) diag_request(parts ...string) {
	/** Do an API call and just pretty print the response contents */
	url_parts := strings.Join(parts, "/")
	msg("diag request for %s", url_parts)

	var output *map[string]interface{} = &map[string]interface{}{}
	err := app.krakenInstance.kraken(output, parts...)
	assert(err == nil, "diag request of %s failed with %s", url_parts, err)
	prettyOutput, err2 := json.MarshalIndent(output, "", "  ")
	assert(err == nil, "error remarshalling the json return value: %s", err2)
	msg("Output of request %s was %s", url_parts, prettyOutput)
}

func (app *TwitchNotifierMain) nextWithRetry(pager *KrakenPager, val interface{}, httpErrorTries uint) error {
	var err error
	for httpErrorTries > 0 {
		err = pager.Next(val)
		if err != nil {
			krakenError, wasKrakenError := err.(*KrakenError)
			if wasKrakenError && krakenError != nil {
				if krakenError.statusCode != 200 {
					httpErrorTries -= 1
					app.getEventsInterface().log(fmt.Sprintf("Got HTTP error %v while loading item; tries left %v", krakenError.statusCode, httpErrorTries))
					continue
				}
			}
		}
		break
	}
	return err
}

func (app *TwitchNotifierMain) PagedKrakenWithRetry(httpErrorTries uint, resultsListKey string, pageSize uint, addParams *url.Values, path ...string) (*KrakenPager, error) {
	var err error
	var pager *KrakenPager
	for httpErrorTries > 0 {
		pager, err = app.krakenInstance.PagedKraken(resultsListKey, pageSize, addParams, path...)
		if err != nil {
			krakenError, wasKrakenError := err.(*KrakenError)
			if wasKrakenError && krakenError != nil {
				if krakenError.statusCode != 200 {
					httpErrorTries -= 1
					app.getEventsInterface().log(fmt.Sprintf("Got HTTP error %v while doing initial pager request; tries left %v", krakenError.statusCode, httpErrorTries))
					continue
				}
			}
		}
		break
	}
	return pager, err
}

func (app *TwitchNotifierMain) get_streams_channels_following(followed_channels map[ChannelID]bool) (map[ChannelID]StreamChannel, error) {
	out := map[ChannelID]StreamChannel{}

	initialRequestHTTPTries := uint(2)

	additionalParams := make(url.Values)
	additionalParams.Add("stream_type", "live")
	pager, err := app.PagedKrakenWithRetry(initialRequestHTTPTries, "streams", app.queryPageSize, &additionalParams, "streams", "followed")

	if err != nil {
		return out, err
	}

	for pager.More() {
		var stream *StreamInfo
		httpErrorTries := uint(2)
		err := app.nextWithRetry(pager, &stream, httpErrorTries)
		if err != nil {
			return out, err
		}

		assert(err == nil, "next return error: %s", err)
		assert(stream != nil, "stream was nil")
		channel := stream.Channel
		assert(channel != nil, "stream has no channel")
		channel_id := stream.Channel.Id
		val, ok := followed_channels[channel_id]
		if val && ok {
			out[channel_id] = StreamChannel{stream, channel}
		} else {
			app.getEventsInterface().log(fmt.Sprintf("skipping channel %s because it's not a followed channel", app.channel_display_name(stream.Channel)))
		}
	}

	return out, nil
}

func (app *TwitchNotifierMain) main_loop() {

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"expvar"
	"fmt"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"math/rand"
	"net/http"
	"strconv"
	"time"
)

var weathers [5]string = [5]string{"Sunny", "Rainy", "Smoggy", "Thundersnow", "Drizzly"}

type Weather struct {
	Temp string
	Type string
}

func handler(w http.ResponseWriter, r *http.Request) {
	i := rand.Intn(4)
	temp := rand.Intn(120)
	weather := Weather{Temp: strconv.Itoa(temp) + "F", Type: weathers[i]}

	w.Header().Set("Server", "WEATHER")
	w.Header().Set("Content-Type", "application/json")

	jsonData, _ := json.Marshal(weather)
	w.Write(jsonData)
	fmt.Println("Dispatched weather data: ", weather)
}

func main() {
	http.HandleFunc("/", prometheus.InstrumentHandlerFunc("Weather", handler))
	http.Handle("/metrics", promhttp.Handler())
	http.ListenAndServe(":8070", nil)
}
<commit_msg>Fixed unused imports<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"math/rand"
	"net/http"
	"strconv"
)

var weathers [5]string = [5]string{"Sunny", "Rainy", "Smoggy", "Thundersnow", "Drizzly"}

type Weather struct {
	Temp string
	Type string
}

func handler(w http.ResponseWriter, r *http.Request) {
	i := rand.Intn(4)
	temp := rand.Intn(120)
	weather := Weather{Temp: strconv.Itoa(temp) + "F", Type: weathers[i]}

	w.Header().Set("Server", "WEATHER")
	w.Header().Set("Content-Type", "application/json")

	jsonData, _ := json.Marshal(weather)
	w.Write(jsonData)
	fmt.Println("Dispatched weather data: ", weather)
}

func main() {
	http.HandleFunc("/", prometheus.InstrumentHandlerFunc("Weather", handler))
	http.Handle("/metrics", promhttp.Handler())
	http.ListenAndServe(":8070", nil)
}
<|endoftext|>


<commit_before>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

// This file holds the basic scanning routines that separate a source file
// into the various tokens

package main

import "./libgogo/_obj/libgogo"

//
// Function fetching a new character from the file that is compiled
// Adds counters for line and columns
//
func GetCharWrapped() byte {
    var singleChar byte;
    singleChar = libgogo.GetChar(fileInfo[curFileIndex].fd);
    if (singleChar == 10) {
        fileInfo[curFileIndex].charCounter = 1;
        fileInfo[curFileIndex].lineCounter = fileInfo[curFileIndex].lineCounter + 1;
    } else {
        fileInfo[curFileIndex].charCounter = fileInfo[curFileIndex].charCounter + 1;
    }
    return singleChar;
}

func GetNextTokenRaw() {
    var singleChar byte; // Byte holding the last read value
    // Flag indicating whether we are in a comment.
    // 0 for no comment
    // 1 for a single line comment
    // 2 for a multi line comment
    var inComment uint64;
    var done uint64; // Flag indicating whether a cycle (Token) is finsihed
    var spaceDone uint64; // Flag indicating whether an abolishment cycle is finished
    var numBuf string;

    // Initialize variables
    done = 0;
    spaceDone = 0;
    inComment = 0;
    tok.strValue = "";

    // If the previous cycle had to read the next char (and stored it), it is
    // now used as first read
    if tok.nextChar == 0 {
        singleChar = GetCharWrapped();
    } else {
        singleChar = tok.nextChar;
        tok.nextChar = 0;
    }

    // check if it is a valid read, or an EOF
    if singleChar == 0 {
        tok.id = TOKEN_EOS;
        done = 1;
        spaceDone = 1;
    }

    //
    // Cleaning Tasks
    // The next part strips out spaces, newlines, tabs, and comments
    // Comments can either be single line with double slashes (//) or multiline
    // using C++ syntax /* */
    //
    for ; spaceDone != 1; {

        // check whether a comment is starting
        if singleChar == '/' {
            // if we are in a comment skip the rest, get the next char otherwise
            if inComment == 0 {
                singleChar = GetCharWrapped();
                if singleChar == '/' {
                    // we are in a single line comment (until newline is found)
                    inComment = 1;
                } else {
                    if singleChar == '*' {
                        // we are in a multiline comment (until ending is found)
                        inComment = 2;
                    } else {
                        ScanErrorString("Unknown character combination for comments.");
                    }
                }
            }
        }

        // check whether a multi-line comment is ending
        if singleChar == '*' {
            singleChar = GetCharWrapped();
            if singleChar == '/' {
                if inComment == 2 {
                    inComment = 0;
                    singleChar = GetCharWrapped();
                }
            }
        }

        // if character is a newline:
        //  *) if in a singleline comment, exit the comment
        //  *) skip otherwise
        if singleChar == 10 {
            if inComment == 1 {
                inComment = 0;
            }
        }

        // handle everything that is not a space,tab,newline
        if (singleChar != ' ') && (singleChar != 9) && (singleChar != 10) {
            // if not in a comment we have our current valid char
            if inComment == 0 {
                spaceDone = 1;
            }

            // check if GetChar() returned EOF while skipping
            if singleChar == 0 {
                tok.id = TOKEN_EOS;
                spaceDone = 1;
                done = 1;
            }
        }

        // if we are not done until now, get a new character and start another abolishing cycle
        if spaceDone == 0 {
            singleChar=GetCharWrapped();
        }
    }

    //
    // Actual scanning part starts here
    //

    // Catch identifiers
    // identifier = letter { letter | digit }.
    if (done != 1) && ((singleChar >= 'A') && (singleChar <= 'Z')) || ((singleChar >= 'a') && (singleChar <= 'z')) || (singleChar == '_') { // check for letter or _
        tok.id = TOKEN_IDENTIFIER;
        // preceding characters may be letter,_, or a number
        for ; ((singleChar >= 'A') && (singleChar <= 'Z')) || ((singleChar >= 'a') && (singleChar <= 'z')) || (singleChar == '_') || ((singleChar >= '0') && (singleChar <= '9')); singleChar = GetCharWrapped() {
            tmp_TokAppendStr(singleChar);
        }
        // save the last read character for the next GetNextToken() cycle
        tok.nextChar = singleChar;
        done = 1;
    }

    // string "..."
    if (done != 1) && (singleChar == '"') {
        tok.id = TOKEN_STRING;
        for singleChar = GetCharWrapped(); (singleChar != '"') && (singleChar > 31) && (singleChar < 127);singleChar = GetCharWrapped() {
            tmp_TokAppendStr(singleChar);
        }
        if singleChar != '"' {
            ScanErrorString("String not closing.");
        }
        done = 1;
    }

    // Single Quoted Character
    if (done != 1) && singleChar == 39 {
        singleChar = GetCharWrapped();
        if (singleChar != 39) && (singleChar > 31) && (singleChar < 127) {
            tok.id = TOKEN_INTEGER;
            tok.intValue = libgogo.ToIntFromByte(singleChar);
        } else {
            ScanErrorString("Unknown character.");
        }
        singleChar = GetCharWrapped();
        if singleChar != 39 {
            ScanErrorString("Only single characters allowed. Use corresponding integer for special characters.");
        }
        done = 1;
    }

    // left brace (
    if (done != 1) && singleChar == '(' {
        tok.id = TOKEN_LBRAC;
        done = 1;
    }

    // right brace )
    if (done != 1) && singleChar == ')' {
        tok.id = TOKEN_RBRAC;
        done = 1;
    }

    // left square bracket [
    if (done != 1) && singleChar == '[' {
        tok.id = TOKEN_LSBRAC;
        done = 1;
    }

    // right square bracket ]
    if (done != 1) && singleChar == ']' {
        tok.id = TOKEN_RSBRAC;
        done = 1;
    }

    // integer
    if (done != 1) && (singleChar > 47) && (singleChar < 58) {
        numBuf = "";

        for ; (singleChar > 47) && (singleChar < 58) ; singleChar = GetCharWrapped() {
            libgogo.CharAppend(&numBuf, singleChar);
        }

        tok.nextChar = singleChar;
        tok.id = TOKEN_INTEGER;
        tok.intValue = libgogo.StringToInt(numBuf);

        done = 1;
    }

    // Left curly bracket '{'
    if (done != 1) && (singleChar == '{') {
        tok.id = TOKEN_LCBRAC;
        done = 1;
    }

    // Right curly bracket '}'
    if (done != 1) && (singleChar == '}') {
        tok.id = TOKEN_RCBRAC;
        done = 1;
    }

    // Point '.'
    if (done != 1) && (singleChar == '.') {
        tok.id = TOKEN_PT;
        done = 1;
    }

    // Not ('!') or Not Equal ('!=')
    if (done != 1) && (singleChar == '!') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_NOTEQUAL;
        } else {
            tok.id = TOKEN_NOT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // Semicolon ';'
    if (done != 1) && (singleChar == ';') {
        tok.id = TOKEN_SEMICOLON;
        done = 1;
    }

    // Colon ','
    if (done != 1) && (singleChar == ',') {
        tok.id = TOKEN_COLON;
        done = 1;
    }

    // Assignment '=' or Equals comparison '=='
    if (done != 1) && (singleChar == '=') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_EQUALS;
        } else {
            tok.id = TOKEN_ASSIGN;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // AND Relation '&&'
    if (done != 1) && (singleChar == '&') {
        singleChar = GetCharWrapped();
        if singleChar == '&' {
            tok.id = TOKEN_REL_AND;
        } else {
            tok.id = TOKEN_OP_ADR;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // OR Relation '||'
    if (done != 1) && (singleChar == '|') {
        singleChar = GetCharWrapped();
        if singleChar == '|' {
            tok.id = TOKEN_REL_OR;
        } else {
            ScanErrorString("No binary OR (|) supported. Only ||.");
        }
        done = 1;
    }

    // Greater and Greater-Than relation
    if (done != 1) && (singleChar == '>') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_REL_GTOE;
        } else {
            tok.id = TOKEN_REL_GT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // Less and Less-Than relation
    if (done != 1) && (singleChar == '<') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_REL_LTOE;
        } else {
            tok.id = TOKEN_REL_LT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    if (done != 1) && (singleChar == '+') {
        tok.id = TOKEN_ARITH_PLUS;
        done = 1;
    }

    if (done != 1) && (singleChar == '-') {
        tok.id = TOKEN_ARITH_MINUS;
        done = 1;
    }

    if (done != 1) && (singleChar == '*') {
        tok.id = TOKEN_ARITH_MUL;
        done = 1;
    }

    if (done != 1) && (singleChar == '/') {
        tok.id = TOKEN_ARITH_DIV;
        done = 1;
    }

    if (done != 1) {
        ScanErrorChar(singleChar);
    }
}

//
// GetNextToken should be called by the parser. It bascially fetches the next
// token by calling GetNextTokenRaw() and filters the identifiers for known
// keywords.
//
func GetNextToken() {
    GetNextTokenRaw();

    // Convert identifier to keyworded tokens
    if tok.id == TOKEN_IDENTIFIER {
        if libgogo.StringCompare("if",tok.strValue) == 0 {
            tok.id = TOKEN_IF;
        }
        if libgogo.StringCompare("else",tok.strValue) == 0 {
            tok.id = TOKEN_ELSE;
        }
        if libgogo.StringCompare("for",tok.strValue) == 0 {
            tok.id = TOKEN_FOR;
        }
        if libgogo.StringCompare("type",tok.strValue) == 0 {
            tok.id = TOKEN_TYPE;
        }
        if libgogo.StringCompare("const",tok.strValue) == 0 {
            tok.id = TOKEN_CONST;
        }
        if libgogo.StringCompare("var",tok.strValue) == 0 {
            tok.id = TOKEN_VAR;
        }
        if libgogo.StringCompare("struct", tok.strValue) == 0 {
            tok.id = TOKEN_STRUCT;
        }
        if libgogo.StringCompare("return", tok.strValue) == 0 {
            tok.id = TOKEN_RETURN;
        }
        if libgogo.StringCompare("func", tok.strValue) == 0 {
            tok.id = TOKEN_FUNC;
        }
        if libgogo.StringCompare("import", tok.strValue) == 0 {
            tok.id = TOKEN_IMPORT;
        }
        if libgogo.StringCompare("package", tok.strValue) == 0 {
            tok.id = TOKEN_PACKAGE;
        }
    }

    tok.nextToken = 0;
}

//
// Debugging and temporary functions
//

func debugToken(tok *Token) {
    libgogo.PrintString("---------------------\n");
    libgogo.PrintString("Token Id: ");
    libgogo.PrintNumber(tok.id);
    libgogo.PrintString("\n");
    if (tok.id == TOKEN_IDENTIFIER) || (tok.id == TOKEN_STRING) {
        libgogo.PrintString("Stored string: ");
        libgogo.PrintString(tok.strValue);
        libgogo.PrintString("\n");
    }
    if tok.id == TOKEN_INTEGER {
        libgogo.PrintString("Stored integer: ");
        libgogo.PrintNumber(tok.intValue);
        libgogo.PrintString("\n");
    }
}

func tmp_TokAppendStr(b byte) {
    libgogo.CharAppend(&tok.strValue, b);
}
<commit_msg>scanner.go: Fix ignoring '*' characters.<commit_after>// Copyright 2010 The GoGo Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

// This file holds the basic scanning routines that separate a source file
// into the various tokens

package main

import "./libgogo/_obj/libgogo"

//
// Function fetching a new character from the file that is compiled
// Adds counters for line and columns
//
func GetCharWrapped() byte {
    var singleChar byte;
    singleChar = libgogo.GetChar(fileInfo[curFileIndex].fd);
    if (singleChar == 10) {
        fileInfo[curFileIndex].charCounter = 1;
        fileInfo[curFileIndex].lineCounter = fileInfo[curFileIndex].lineCounter + 1;
    } else {
        fileInfo[curFileIndex].charCounter = fileInfo[curFileIndex].charCounter + 1;
    }
    return singleChar;
}

func GetNextTokenRaw() {
    var singleChar byte; // Byte holding the last read value
    // Flag indicating whether we are in a comment.
    // 0 for no comment
    // 1 for a single line comment
    // 2 for a multi line comment
    var inComment uint64;
    var done uint64; // Flag indicating whether a cycle (Token) is finsihed
    var spaceDone uint64; // Flag indicating whether an abolishment cycle is finished
    var numBuf string;

    // Initialize variables
    done = 0;
    spaceDone = 0;
    inComment = 0;
    tok.strValue = "";

    // If the previous cycle had to read the next char (and stored it), it is
    // now used as first read
    if tok.nextChar == 0 {
        singleChar = GetCharWrapped();
    } else {
        singleChar = tok.nextChar;
        tok.nextChar = 0;
    }

    // check if it is a valid read, or an EOF
    if singleChar == 0 {
        tok.id = TOKEN_EOS;
        done = 1;
        spaceDone = 1;
    }

    //
    // Cleaning Tasks
    // The next part strips out spaces, newlines, tabs, and comments
    // Comments can either be single line with double slashes (//) or multiline
    // using C++ syntax /* */
    //
    for ; spaceDone != 1; {

        // check whether a comment is starting
        if singleChar == '/' {
            // if we are in a comment skip the rest, get the next char otherwise
            if inComment == 0 {
                singleChar = GetCharWrapped();
                if singleChar == '/' {
                    // we are in a single line comment (until newline is found)
                    inComment = 1;
                } else {
                    if singleChar == '*' {
                        // we are in a multiline comment (until ending is found)
                        inComment = 2;
                    } else {
                        ScanErrorString("Unknown character combination for comments.");
                    }
                }
            }
        }

        if inComment == 2 {
            // check whether a multi-line comment is ending
            if singleChar == '*' {
                singleChar = GetCharWrapped();
                if singleChar == '/' {
                    if inComment == 2 {
                        inComment = 0;
                        singleChar = GetCharWrapped();
                    }
                }
            }
        }

        // if character is a newline:
        //  *) if in a singleline comment, exit the comment
        //  *) skip otherwise
        if singleChar == 10 {
            if inComment == 1 {
                inComment = 0;
            }
        }

        // handle everything that is not a space,tab,newline
        if (singleChar != ' ') && (singleChar != 9) && (singleChar != 10) {
            // if not in a comment we have our current valid char
            if inComment == 0 {
                spaceDone = 1;
            }

            // check if GetChar() returned EOF while skipping
            if singleChar == 0 {
                tok.id = TOKEN_EOS;
                spaceDone = 1;
                done = 1;
            }
        }

        // if we are not done until now, get a new character and start another abolishing cycle
        if spaceDone == 0 {
            singleChar=GetCharWrapped();
        }
    }

    //
    // Actual scanning part starts here
    //

    // Catch identifiers
    // identifier = letter { letter | digit }.
    if (done != 1) && ((singleChar >= 'A') && (singleChar <= 'Z')) || ((singleChar >= 'a') && (singleChar <= 'z')) || (singleChar == '_') { // check for letter or _
        tok.id = TOKEN_IDENTIFIER;
        // preceding characters may be letter,_, or a number
        for ; ((singleChar >= 'A') && (singleChar <= 'Z')) || ((singleChar >= 'a') && (singleChar <= 'z')) || (singleChar == '_') || ((singleChar >= '0') && (singleChar <= '9')); singleChar = GetCharWrapped() {
            tmp_TokAppendStr(singleChar);
        }
        // save the last read character for the next GetNextToken() cycle
        tok.nextChar = singleChar;
        done = 1;
    }

    // string "..."
    if (done != 1) && (singleChar == '"') {
        tok.id = TOKEN_STRING;
        for singleChar = GetCharWrapped(); (singleChar != '"') && (singleChar > 31) && (singleChar < 127);singleChar = GetCharWrapped() {
            tmp_TokAppendStr(singleChar);
        }
        if singleChar != '"' {
            ScanErrorString("String not closing.");
        }
        done = 1;
    }

    // Single Quoted Character
    if (done != 1) && singleChar == 39 {
        singleChar = GetCharWrapped();
        if (singleChar != 39) && (singleChar > 31) && (singleChar < 127) {
            tok.id = TOKEN_INTEGER;
            tok.intValue = libgogo.ToIntFromByte(singleChar);
        } else {
            ScanErrorString("Unknown character.");
        }
        singleChar = GetCharWrapped();
        if singleChar != 39 {
            ScanErrorString("Only single characters allowed. Use corresponding integer for special characters.");
        }
        done = 1;
    }

    // left brace (
    if (done != 1) && singleChar == '(' {
        tok.id = TOKEN_LBRAC;
        done = 1;
    }

    // right brace )
    if (done != 1) && singleChar == ')' {
        tok.id = TOKEN_RBRAC;
        done = 1;
    }

    // left square bracket [
    if (done != 1) && singleChar == '[' {
        tok.id = TOKEN_LSBRAC;
        done = 1;
    }

    // right square bracket ]
    if (done != 1) && singleChar == ']' {
        tok.id = TOKEN_RSBRAC;
        done = 1;
    }

    // integer
    if (done != 1) && (singleChar > 47) && (singleChar < 58) {
        numBuf = "";

        for ; (singleChar > 47) && (singleChar < 58) ; singleChar = GetCharWrapped() {
            libgogo.CharAppend(&numBuf, singleChar);
        }

        tok.nextChar = singleChar;
        tok.id = TOKEN_INTEGER;
        tok.intValue = libgogo.StringToInt(numBuf);

        done = 1;
    }

    // Left curly bracket '{'
    if (done != 1) && (singleChar == '{') {
        tok.id = TOKEN_LCBRAC;
        done = 1;
    }

    // Right curly bracket '}'
    if (done != 1) && (singleChar == '}') {
        tok.id = TOKEN_RCBRAC;
        done = 1;
    }

    // Point '.'
    if (done != 1) && (singleChar == '.') {
        tok.id = TOKEN_PT;
        done = 1;
    }

    // Not ('!') or Not Equal ('!=')
    if (done != 1) && (singleChar == '!') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_NOTEQUAL;
        } else {
            tok.id = TOKEN_NOT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // Semicolon ';'
    if (done != 1) && (singleChar == ';') {
        tok.id = TOKEN_SEMICOLON;
        done = 1;
    }

    // Colon ','
    if (done != 1) && (singleChar == ',') {
        tok.id = TOKEN_COLON;
        done = 1;
    }

    // Assignment '=' or Equals comparison '=='
    if (done != 1) && (singleChar == '=') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_EQUALS;
        } else {
            tok.id = TOKEN_ASSIGN;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // AND Relation '&&'
    if (done != 1) && (singleChar == '&') {
        singleChar = GetCharWrapped();
        if singleChar == '&' {
            tok.id = TOKEN_REL_AND;
        } else {
            tok.id = TOKEN_OP_ADR;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // OR Relation '||'
    if (done != 1) && (singleChar == '|') {
        singleChar = GetCharWrapped();
        if singleChar == '|' {
            tok.id = TOKEN_REL_OR;
        } else {
            ScanErrorString("No binary OR (|) supported. Only ||.");
        }
        done = 1;
    }

    // Greater and Greater-Than relation
    if (done != 1) && (singleChar == '>') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_REL_GTOE;
        } else {
            tok.id = TOKEN_REL_GT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    // Less and Less-Than relation
    if (done != 1) && (singleChar == '<') {
        singleChar = GetCharWrapped();
        if singleChar == '=' {
            tok.id = TOKEN_REL_LTOE;
        } else {
            tok.id = TOKEN_REL_LT;
            tok.nextChar = singleChar;
        }
        done = 1;
    }

    if (done != 1) && (singleChar == '+') {
        tok.id = TOKEN_ARITH_PLUS;
        done = 1;
    }

    if (done != 1) && (singleChar == '-') {
        tok.id = TOKEN_ARITH_MINUS;
        done = 1;
    }

    if (done != 1) && (singleChar == '*') {
        tok.id = TOKEN_ARITH_MUL;
        done = 1;
    }

    if (done != 1) && (singleChar == '/') {
        tok.id = TOKEN_ARITH_DIV;
        done = 1;
    }

    if (done != 1) {
        ScanErrorChar(singleChar);
    }
}

//
// GetNextToken should be called by the parser. It bascially fetches the next
// token by calling GetNextTokenRaw() and filters the identifiers for known
// keywords.
//
func GetNextToken() {
    GetNextTokenRaw();

    // Convert identifier to keyworded tokens
    if tok.id == TOKEN_IDENTIFIER {
        if libgogo.StringCompare("if",tok.strValue) == 0 {
            tok.id = TOKEN_IF;
        }
        if libgogo.StringCompare("else",tok.strValue) == 0 {
            tok.id = TOKEN_ELSE;
        }
        if libgogo.StringCompare("for",tok.strValue) == 0 {
            tok.id = TOKEN_FOR;
        }
        if libgogo.StringCompare("type",tok.strValue) == 0 {
            tok.id = TOKEN_TYPE;
        }
        if libgogo.StringCompare("const",tok.strValue) == 0 {
            tok.id = TOKEN_CONST;
        }
        if libgogo.StringCompare("var",tok.strValue) == 0 {
            tok.id = TOKEN_VAR;
        }
        if libgogo.StringCompare("struct", tok.strValue) == 0 {
            tok.id = TOKEN_STRUCT;
        }
        if libgogo.StringCompare("return", tok.strValue) == 0 {
            tok.id = TOKEN_RETURN;
        }
        if libgogo.StringCompare("func", tok.strValue) == 0 {
            tok.id = TOKEN_FUNC;
        }
        if libgogo.StringCompare("import", tok.strValue) == 0 {
            tok.id = TOKEN_IMPORT;
        }
        if libgogo.StringCompare("package", tok.strValue) == 0 {
            tok.id = TOKEN_PACKAGE;
        }
    }

    tok.nextToken = 0;
}

//
// Debugging and temporary functions
//

func debugToken(tok *Token) {
    libgogo.PrintString("---------------------\n");
    libgogo.PrintString("Token Id: ");
    libgogo.PrintNumber(tok.id);
    libgogo.PrintString("\n");
    if (tok.id == TOKEN_IDENTIFIER) || (tok.id == TOKEN_STRING) {
        libgogo.PrintString("Stored string: ");
        libgogo.PrintString(tok.strValue);
        libgogo.PrintString("\n");
    }
    if tok.id == TOKEN_INTEGER {
        libgogo.PrintString("Stored integer: ");
        libgogo.PrintNumber(tok.intValue);
        libgogo.PrintString("\n");
    }
}

func tmp_TokAppendStr(b byte) {
    libgogo.CharAppend(&tok.strValue, b);
}
<|endoftext|>


<commit_before>package extensions

import (
	"bytes"
	"xd/lib/common"
	"xd/lib/version"

	"github.com/zeebo/bencode"
)

// Extension is a bittorrent extenension string
type Extension string

var extensionDefaults = map[Extension]uint8{
	I2PDHT:       1,
	PeerExchange: 2,
}

func (ex Extension) String() string {
	return string(ex)
}

// ExtendedOptions is a serializable BitTorrent extended options message
type ExtendedOptions struct {
	ID         uint8               `bencode:"-"`
	Version    string              `bencode:"v"`
	Extensions map[Extension]uint8 `bencode:"m"`
}

func (opts *ExtendedOptions) SetSupported(ext Extension) {
	// TODO: this will error if we do not support this extension
	opts.Extensions[ext] = extensionDefaults[ext]
}

func (opts *ExtendedOptions) IsSupported(ext Extension) (has bool) {
	_, has = opts.Extensions[ext]
	return
}

// Copy makes a copy of this ExtendedOptions
func (opts *ExtendedOptions) Copy() *ExtendedOptions {
	ext := make(map[Extension]uint8)
	for k, v := range opts.Extensions {
		ext[k] = v
	}
	return &ExtendedOptions{
		ID:         opts.ID,
		Version:    opts.Version,
		Extensions: ext,
	}
}

// ToWireMessage serializes this ExtendedOptions to a BitTorrent wire message
func (opts *ExtendedOptions) ToWireMessage() *common.WireMessage {
	b := new(bytes.Buffer)
	b.Write([]byte{opts.ID})
	if opts.ID == 0 {
		bencode.NewEncoder(b).Encode(opts)
	}
	return common.NewWireMessage(common.Extended, b.Bytes())
}

// New creates new valid ExtendedOptions instance
func New() *ExtendedOptions {
	return &ExtendedOptions{
		Version:    version.Version(),
		Extensions: make(map[Extension]uint8),
	}
}

// FromWireMessage loads an ExtendedOptions messgae from a BitTorrent wire message
func FromWireMessage(msg *common.WireMessage) (opts *ExtendedOptions) {
	if msg.MessageID() == common.Extended {
		payload := msg.Payload()
		if len(payload) > 0 {
			opts = &ExtendedOptions{
				ID: payload[0],
			}
			bencode.DecodeBytes(payload[1:], opts)
		}
	}
	return
}
<commit_msg>try fixing extensions<commit_after>package extensions

import (
	"bytes"
	"xd/lib/common"
	"xd/lib/version"

	"github.com/zeebo/bencode"
)

// Extension is a bittorrent extenension string
type Extension string

var extensionDefaults = map[Extension]uint8{
	I2PDHT:       1,
	PeerExchange: 2,
}

func (ex Extension) String() string {
	return string(ex)
}

// ExtendedOptions is a serializable BitTorrent extended options message
type ExtendedOptions struct {
	ID         uint8            `bencode:"-"`
	Version    string           `bencode:"v"`
	Extensions map[string]uint8 `bencode:"m"`
}

func (opts *ExtendedOptions) SetSupported(ext Extension) {
	// TODO: this will error if we do not support this extension
	opts.Extensions[ext.String()] = extensionDefaults[ext]
}

func (opts *ExtendedOptions) IsSupported(ext string) (has bool) {
	_, has = opts.Extensions[ext]
	return
}

// Copy makes a copy of this ExtendedOptions
func (opts *ExtendedOptions) Copy() *ExtendedOptions {
	ext := make(map[string]uint8)
	for k, v := range opts.Extensions {
		ext[k] = v
	}
	return &ExtendedOptions{
		ID:         opts.ID,
		Version:    opts.Version,
		Extensions: ext,
	}
}

// ToWireMessage serializes this ExtendedOptions to a BitTorrent wire message
func (opts *ExtendedOptions) ToWireMessage() *common.WireMessage {
	b := new(bytes.Buffer)
	b.Write([]byte{opts.ID})
	if opts.ID == 0 {
		bencode.NewEncoder(b).Encode(opts)
	}
	return common.NewWireMessage(common.Extended, b.Bytes())
}

// New creates new valid ExtendedOptions instance
func New() *ExtendedOptions {
	return &ExtendedOptions{
		Version:    version.Version(),
		Extensions: make(map[string]uint8),
	}
}

// FromWireMessage loads an ExtendedOptions messgae from a BitTorrent wire message
func FromWireMessage(msg *common.WireMessage) (opts *ExtendedOptions) {
	if msg.MessageID() == common.Extended {
		payload := msg.Payload()
		if len(payload) > 0 {
			opts = &ExtendedOptions{
				ID: payload[0],
			}
			bencode.DecodeBytes(payload[1:], opts)
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"os"
	"strings"

	"golang.org/x/crypto/ssh"
)

// DEBUG ALERT --> Uma instalação contemém métodos relacionados aos comandos.
// e.g. adicionar comando, executar commando, ...
// Yii é um tipo de Programa que contém commandos e implementa a interface
// Installer.
// Wp é outro tipo de Programa que contém commandos e implementa a interface
// Installer.
// Quaisquer novos sotfwares a serem instalados por este programas deverão ser
// do tipo Programa e implementar a interface Installer.
type ProjectField struct {
	name, label, inputQuestion, errorMsg, validationMsg string
}

type Program struct {
	ProjectField
	commands []string
}

type Project struct {
	projectname, hostname, pwd, port ProjectField
	typ Program
}

type Installer interface {
	connect()
}

func main() {

	// Initialization
	project := new(Project)

	project.assemblyLine()

	// SSH connection config
	config := &ssh.ClientConfig {
		User: project.projectname.name,
		Auth: []ssh.AuthMethod{
			ssh.Password(project.pwd.name),
		},
	}

	// commands := []string{
	// 	"cd private && echo 'hello world' > hello.txt",
	// 	}
	// project.connect(commands, config)

}

func (p *Project) assemblyLine() {

	// Project name
	p.projectname.inputQuestion = "project name: "
	p.projectname.label = "projectname"
	p.projectname.errorMsg = "error getting the project's name: "
	p.projectname.validationMsg = "make sure you type a valid name for your project (3 to 20 characters)."
	ask4Input(&p.projectname)

	// Hostname
	p.hostname.inputQuestion = "hostname: "
	p.hostname.label = "hostname"
	p.hostname.errorMsg = "error getting the project's hostname: "
	p.hostname.validationMsg = "make sure you type a valid hostname for your project. it must contain '.com', '.pt' or '.org', for example.)."
	ask4Input(&p.hostname)

	// Password
	p.pwd.inputQuestion = "password: "
	p.pwd.label = "pwd"
	p.pwd.errorMsg = "error getting the project's password: "
	p.pwd.validationMsg = "type a valid password. It must contain at least 6 digits"
	ask4Input(&p.pwd)

	// Port
	p.port.inputQuestion = "port (default 22): "
	p.port.label = "port"
	p.port.errorMsg = "error getting the project's port"
	p.port.validationMsg = "only digits allowed. min 0, max 999."
	ask4Input(&p.port)

	// Type
	p.typ.inputQuestion = "project type [1]yii [2]wp or gohugo: "
	p.typ.label = "type"
	p.typ.errorMsg = "error getting the project's type"
	p.typ.validationMsg = "pay attention to the options"
	ask4Input(&p.typ)
}

// Takes the assemblyLine's data and mount the prompt for the user.
func ask4Input(field *ProjectField) {
	consolereader := bufio.NewReader(os.Stdin)
	fmt.Print(field.inputQuestion)
	input, err := consolereader.ReadString('\n')
	checkError(field.errorMsg, err)
	checkInput(field, input)
}

// Right on hand error checker.
func checkError(msg string, err error) {
	if err != nil {
		log.Fatal(msg, err.Error())
	}
}

func checkInput(field *ProjectField, input string) {

	var cleanInput string
	var inputLength int

	if len(input) > 1 {
		cleanInput = strings.Fields(input)[0]
		inputLength = len(cleanInput)
	}

	switch field.label {
		case "projectname":

			if inputLength < 3 || inputLength > 20 {
				fmt.Println(field.validationMsg)
				ask4Input(field)
			}
		case "hostname":
			if inputLength <= 5 {
				fmt.Println(field.validationMsg)
				ask4Input(field)
			}
		case "pwd":
			if inputLength <= 6 {
				fmt.Println(field.validationMsg)
				ask4Input(field)
			}
		case "port":

			if inputLength == 0 {
				cleanInput = "22"
			} else if inputLength > 3 {
				fmt.Println(field.validationMsg)
				ask4Input(field)
			}
		case "type":
			if cleanInput != "1" && cleanInput != "2" {
				fmt.Println(field.validationMsg)
				ask4Input(field)
			} else if cleanInput == "1" {
				cleanInput = "Yii"
			} else if cleanInput == "2" {
				cleanInput = "WP"
			}
	}

	field.name = cleanInput
}

func (p *Project) connect(commands []string, config *ssh.ClientConfig) {

	log.Printf("Trying connection...\n")

	conn, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", p.hostname.name, p.port.name), config)
	checkError("Failed to dial: ", err)
	log.Printf("Connection established.\n")

	session, err := conn.NewSession()
	checkError("Failed to build the session: ", err)

	defer session.Close()

	log.Printf("Session created.\n")

	var stdoutBuf bytes.Buffer
	session.Stdout = &stdoutBuf

	// log.Printf("Preparing to execute command: %s", commands[0])
	// if err := session.Run(commands[0]); err != nil {
	// 	log.Fatal("Error on command execution", err.Error())
	// }

	log.Printf("Preparing to execute command: %s", p.typ.command[0])
}
<commit_msg>identation ssh<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"log"
	"os"
	"strings"

	"golang.org/x/crypto/ssh"
)

// DEBUG ALERT --> Uma instalação contemém métodos relacionados aos comandos.
// e.g. adicionar comando, executar commando, ...
// Yii é um tipo de Programa que contém commandos e implementa a interface
// Installer.
// Wp é outro tipo de Programa que contém commandos e implementa a interface
// Installer.
// Quaisquer novos sotfwares a serem instalados por este programas deverão ser
// do tipo Programa e implementar a interface Installer.
type ProjectField struct {
	name, label, inputQuestion, errorMsg, validationMsg string
}

type Program struct {
	ProjectField
	commands []string
}

type Project struct {
	projectname, hostname, pwd, port ProjectField
	typ                              Program
}

type Installer interface {
	connect()
}

func main() {

	// Initialization
	project := new(Project)

	project.assemblyLine()

	// SSH connection config
	config := &ssh.ClientConfig{
		User: project.projectname.name,
		Auth: []ssh.AuthMethod{
			ssh.Password(project.pwd.name),
		},
	}

	// commands := []string{
	// 	"cd private && echo 'hello world' > hello.txt",
	// 	}
	// project.connect(commands, config)

}

func (p *Project) assemblyLine() {

	// Project name
	p.projectname.inputQuestion = "project name: "
	p.projectname.label = "projectname"
	p.projectname.errorMsg = "error getting the project's name: "
	p.projectname.validationMsg = "make sure you type a valid name for your project (3 to 20 characters)."
	ask4Input(&p.projectname)

	// Hostname
	p.hostname.inputQuestion = "hostname: "
	p.hostname.label = "hostname"
	p.hostname.errorMsg = "error getting the project's hostname: "
	p.hostname.validationMsg = "make sure you type a valid hostname for your project. it must contain '.com', '.pt' or '.org', for example.)."
	ask4Input(&p.hostname)

	// Password
	p.pwd.inputQuestion = "password: "
	p.pwd.label = "pwd"
	p.pwd.errorMsg = "error getting the project's password: "
	p.pwd.validationMsg = "type a valid password. It must contain at least 6 digits"
	ask4Input(&p.pwd)

	// Port
	p.port.inputQuestion = "port (default 22): "
	p.port.label = "port"
	p.port.errorMsg = "error getting the project's port"
	p.port.validationMsg = "only digits allowed. min 0, max 999."
	ask4Input(&p.port)

	// Type
	p.typ.inputQuestion = "project type [1]yii [2]wp or gohugo: "
	p.typ.label = "type"
	p.typ.errorMsg = "error getting the project's type"
	p.typ.validationMsg = "pay attention to the options"
	ask4Input(&p.typ)
}

// Takes the assemblyLine's data and mount the prompt for the user.
func ask4Input(field *ProjectField) {
	consolereader := bufio.NewReader(os.Stdin)
	fmt.Print(field.inputQuestion)
	input, err := consolereader.ReadString('\n')
	checkError(field.errorMsg, err)
	checkInput(field, input)
}

// Right on hand error checker.
func checkError(msg string, err error) {
	if err != nil {
		log.Fatal(msg, err.Error())
	}
}

func checkInput(field *ProjectField, input string) {

	var cleanInput string
	var inputLength int

	if len(input) > 1 {
		cleanInput = strings.Fields(input)[0]
		inputLength = len(cleanInput)
	}

	switch field.label {
	case "projectname":

		if inputLength < 3 || inputLength > 20 {
			fmt.Println(field.validationMsg)
			ask4Input(field)
		}
	case "hostname":
		if inputLength <= 5 {
			fmt.Println(field.validationMsg)
			ask4Input(field)
		}
	case "pwd":
		if inputLength <= 6 {
			fmt.Println(field.validationMsg)
			ask4Input(field)
		}
	case "port":

		if inputLength == 0 {
			cleanInput = "22"
		} else if inputLength > 3 {
			fmt.Println(field.validationMsg)
			ask4Input(field)
		}
	case "type":
		if cleanInput != "1" && cleanInput != "2" {
			fmt.Println(field.validationMsg)
			ask4Input(field)
		} else if cleanInput == "1" {
			cleanInput = "Yii"
		} else if cleanInput == "2" {
			cleanInput = "WP"
		}
	}

	field.name = cleanInput
}

func (p *Project) connect(commands []string, config *ssh.ClientConfig) {

	log.Printf("Trying connection...\n")

	conn, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", p.hostname.name, p.port.name), config)
	checkError("Failed to dial: ", err)
	log.Printf("Connection established.\n")

	session, err := conn.NewSession()
	checkError("Failed to build the session: ", err)

	defer session.Close()

	log.Printf("Session created.\n")

	var stdoutBuf bytes.Buffer
	session.Stdout = &stdoutBuf

	// log.Printf("Preparing to execute command: %s", commands[0])
	// if err := session.Run(commands[0]); err != nil {
	// 	log.Fatal("Error on command execution", err.Error())
	// }

	log.Printf("Preparing to execute command: %s", p.typ.command[0])
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charmrevisionupdater_test

import (
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/charmrevisionupdater"
	"launchpad.net/juju-core/state/apiserver/charmrevisionupdater/testing"
	"launchpad.net/juju-core/utils"
)

type versionUpdaterSuite struct {
	testing.CharmSuite

	updater *charmrevisionupdater.State
}

var _ = gc.Suite(&versionUpdaterSuite{})

func (s *versionUpdaterSuite) SetUpTest(c *gc.C) {
	s.CharmSuite.SetUpTest(c)

	machine, err := s.State.AddMachine("quantal", state.JobManageEnviron)
	c.Assert(err, gc.IsNil)
	password, err := utils.RandomPassword()
	c.Assert(err, gc.IsNil)
	err = machine.SetPassword(password)
	c.Assert(err, gc.IsNil)
	err = machine.SetProvisioned("i-manager", "fake_nonce", nil)
	c.Assert(err, gc.IsNil)
	st := s.OpenAPIAsMachine(c, machine.Tag(), password, "fake_nonce")
	c.Assert(st, gc.NotNil)

	s.updater = charmrevisionupdater.NewState(st)
	c.Assert(s.updater, gc.NotNil)
}

func (s *versionUpdaterSuite) TestUpdateRevisions(c *gc.C) {
	s.SetupScenario(c)
	err := s.updater.UpdateLatestRevisions()
	c.Assert(err, gc.IsNil)

	curl := charm.MustParseURL("cs:quantal/mysql")
	pending, err := s.State.LatestPlaceholderCharm(curl)
	c.Assert(err, gc.IsNil)
	c.Assert(pending.String(), gc.Equals, "cs:quantal/mysql-23")
}
<commit_msg>state/api/charmrevisionupdater: fix tests<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charmrevisionupdater_test

import (
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/charm"
	jujutesting "launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/charmrevisionupdater"
	"launchpad.net/juju-core/state/apiserver/charmrevisionupdater/testing"
	"launchpad.net/juju-core/utils"
)

type versionUpdaterSuite struct {
	jujutesting.JujuConnSuite
	testing.CharmSuite

	updater *charmrevisionupdater.State
}

var _ = gc.Suite(&versionUpdaterSuite{})

func (s *versionUpdaterSuite) SetUpSuite(c *gc.C) {
	s.JujuConnSuite.SetUpSuite(c)
	s.CharmSuite.SetUpSuite(c, &s.JujuConnSuite)
}

func (s *versionUpdaterSuite) TearDownSuite(c *gc.C) {
	s.CharmSuite.TearDownSuite(c)
	s.JujuConnSuite.TearDownSuite(c)
}

func (s *versionUpdaterSuite) SetUpTest(c *gc.C) {
	s.JujuConnSuite.SetUpTest(c)
	s.CharmSuite.SetUpTest(c)

	machine, err := s.State.AddMachine("quantal", state.JobManageEnviron)
	c.Assert(err, gc.IsNil)
	password, err := utils.RandomPassword()
	c.Assert(err, gc.IsNil)
	err = machine.SetPassword(password)
	c.Assert(err, gc.IsNil)
	err = machine.SetProvisioned("i-manager", "fake_nonce", nil)
	c.Assert(err, gc.IsNil)
	st := s.OpenAPIAsMachine(c, machine.Tag(), password, "fake_nonce")
	c.Assert(st, gc.NotNil)

	s.updater = charmrevisionupdater.NewState(st)
	c.Assert(s.updater, gc.NotNil)
}

func (s *versionUpdaterSuite) TearDownTest(c *gc.C) {
	s.CharmSuite.TearDownTest(c)
	s.JujuConnSuite.TearDownTest(c)
}

func (s *versionUpdaterSuite) TestUpdateRevisions(c *gc.C) {
	s.SetupScenario(c)
	err := s.updater.UpdateLatestRevisions()
	c.Assert(err, gc.IsNil)

	curl := charm.MustParseURL("cs:quantal/mysql")
	pending, err := s.State.LatestPlaceholderCharm(curl)
	c.Assert(err, gc.IsNil)
	c.Assert(pending.String(), gc.Equals, "cs:quantal/mysql-23")
}
<|endoftext|>


<commit_before>// Copyright 2021 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package checkpoints

import (
	"container/heap"
	"context"
	"encoding/json"
	"os"
	"sync"

	"github.com/pingcap/tidb-tools/sync_diff_inspector/config"
	"github.com/pingcap/tidb-tools/sync_diff_inspector/report"

	"github.com/pingcap/tidb-tools/sync_diff_inspector/chunk"
	"github.com/siddontang/go/ioutil2"

	"github.com/pingcap/errors"
	"github.com/pingcap/log"
	"go.uber.org/zap"
)

const (
	// SuccessState
	// for chunk: means this chunk's data is equal
	// for table: means this all chunk in this table is equal(except ignore chunk)
	SuccessState = "success"

	// FailedState
	// for chunk: means this chunk's data is not equal
	// for table: means some chunks' data is not equal or some chunk check failed in this table
	FailedState = "failed"

	// IgnoreState
	// for chunk: this chunk is ignored. if sample is not 100% or it is Empty chunk, will ignore some chunk
	// for table: don't have this state
	IgnoreState = "ignore"
)

type Node struct {
	State string `json:"state"` // indicate the state ("success" or "failed") of the chunk

	ChunkRange *chunk.Range `json:"chunk-range"`
	IndexID    int64        `json:"index-id"`
}

func (n *Node) GetID() *chunk.ChunkID { return n.ChunkRange.Index }

func (n *Node) GetState() string { return n.State }

func (n *Node) GetTableIndex() int { return n.ChunkRange.Index.TableIndex }

func (n *Node) GetBucketIndexLeft() int { return n.ChunkRange.Index.BucketIndexLeft }

func (n *Node) GetBucketIndexRight() int { return n.ChunkRange.Index.BucketIndexRight }

func (n *Node) GetChunkIndex() int { return n.ChunkRange.Index.ChunkIndex }

// IsAdjacent represents whether the next node is adjacent node.
// it's the important logic for checkpoint update.
// we need keep this node save to checkpoint in global order.
func (n *Node) IsAdjacent(next *Node) bool {
	if n.GetTableIndex() == next.GetTableIndex()-1 {
		if n.ChunkRange.IsLastChunkForTable() && next.ChunkRange.IsFirstChunkForTable() {
			return true
		}
		return false
	}
	if n.GetTableIndex() == next.GetTableIndex() {
		// same table
		if n.GetBucketIndexRight() == next.GetBucketIndexLeft()-1 {
			if n.ChunkRange.IsLastChunkForBucket() && next.ChunkRange.IsFirstChunkForBucket() {
				return true
			}
			return false
		}
		if n.GetBucketIndexLeft() == next.GetBucketIndexLeft() {
			return n.GetChunkIndex() == next.GetChunkIndex()-1
		}
		return false
	}
	return false
}

// IsLess represents whether the cur node is less than next node.
func (n *Node) IsLess(next *Node) bool {
	if n.GetTableIndex() < next.GetTableIndex() {
		return true
	}
	if n.GetTableIndex() == next.GetTableIndex() {
		if n.GetBucketIndexLeft() <= next.GetBucketIndexLeft()-1 {
			return true
		}
		if n.GetBucketIndexLeft() == next.GetBucketIndexLeft() {
			return n.GetChunkIndex() < next.GetChunkIndex()
		}
		return false
	}
	return false
}

// heap maintain a Min Heap, which can be accessed by multiple threads and protected by mutex.
type nodeHeap struct {
	Nodes            []*Node
	CurrentSavedNode *Node       // CurrentSavedNode save the minimum checker chunk, updated by `GetChunkSnapshot` method
	mu               *sync.Mutex // protect critical section
}

// Checkpoint provide the ability to restart the sync-diff process from the
// latest previous exit point (due to error or intention).
type Checkpoint struct {
	hp *nodeHeap
}

// SaveState contains the information of the latest checked chunk and state of `report`
// When sync-diff start from the checkpoint, it will load this information and continue running
type SavedState struct {
	Chunk  *Node          `json:"chunk-info"`
	Report *report.Report `json:"report-info"`
}

// SetCurrentSavedID the method is unsynchronized, be cautious
func (cp *Checkpoint) SetCurrentSavedID(n *Node) {
	cp.hp.CurrentSavedNode = n
}

func (cp *Checkpoint) GetCurrentSavedID() *Node {
	cp.hp.mu.Lock()
	defer cp.hp.mu.Unlock()
	return cp.hp.CurrentSavedNode
}

func (cp *Checkpoint) Insert(node *Node) {
	cp.hp.mu.Lock()
	heap.Push(cp.hp, node)
	cp.hp.mu.Unlock()
}

// Len - get the length of the heap
func (hp *nodeHeap) Len() int { return len(hp.Nodes) }

// Less - determine which is more priority than another
func (hp *nodeHeap) Less(i, j int) bool {
	return hp.Nodes[i].IsLess(hp.Nodes[j])
}

// Swap - implementation of swap for the heap interface
func (hp *nodeHeap) Swap(i, j int) {
	hp.Nodes[i], hp.Nodes[j] = hp.Nodes[j], hp.Nodes[i]
}

// Push - implementation of push for the heap interface
func (hp *nodeHeap) Push(x interface{}) {
	hp.Nodes = append(hp.Nodes, x.(*Node))
}

// Pop - implementation of pop for heap interface
func (hp *nodeHeap) Pop() (item interface{}) {
	if len(hp.Nodes) == 0 {
		return
	}

	hp.Nodes, item = hp.Nodes[:len(hp.Nodes)-1], hp.Nodes[len(hp.Nodes)-1]
	return
}

func (cp *Checkpoint) Init() {
	hp := &nodeHeap{
		mu:    &sync.Mutex{},
		Nodes: make([]*Node, 0),
		CurrentSavedNode: &Node{
			ChunkRange: &chunk.Range{
				Index:   chunk.GetInitChunkID(),
				IsFirst: true,
				IsLast:  true,
			},
		},
	}
	heap.Init(hp)
	cp.hp = hp
}

// GetChunkSnapshot get the snapshot of the minimum continuous checked chunk
func (cp *Checkpoint) GetChunkSnapshot() (cur *Node) {
	cp.hp.mu.Lock()
	defer cp.hp.mu.Unlock()
	for cp.hp.Len() != 0 && cp.hp.CurrentSavedNode.IsAdjacent(cp.hp.Nodes[0]) {
		cp.hp.CurrentSavedNode = heap.Pop(cp.hp).(*Node)
		cur = cp.hp.CurrentSavedNode
	}
	// wait for next 10s to check
	return cur
}

// SaveChunk saves the chunk to file.
func (cp *Checkpoint) SaveChunk(ctx context.Context, fileName string, cur *Node, reportInfo *report.Report) (*chunk.ChunkID, error) {
	if cur == nil {
		return nil, nil
	}

	savedState := &SavedState{
		Chunk:  cur,
		Report: reportInfo,
	}
	checkpointData, err := json.Marshal(savedState)
	if err != nil {
		log.Warn("fail to save the chunk to the file", zap.Any("chunk index", cur.GetID()), zap.Error(err))
		return nil, errors.Trace(err)
	}

	if err = ioutil2.WriteFileAtomic(fileName, checkpointData, config.LocalFilePerm); err != nil {
		return nil, err
	}
	log.Info("save checkpoint",
		zap.Any("chunk", cur),
		zap.String("state", cur.GetState()))
	return cur.GetID(), nil
}

// LoadChunk loads chunk info from file `chunk`
func (cp *Checkpoint) LoadChunk(fileName string) (*Node, *report.Report, error) {
	bytes, err := os.ReadFile(fileName)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	n := &SavedState{}
	err = json.Unmarshal(bytes, n)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	return n.Chunk, n.Report, nil
}

// CheckConfig stores the information about the config of the current run.
// If we restart a sync-diff from a different config, the checkpoint is disable.
type CheckConfig struct {
	Table     string `json:"tables"`
	Fields    string `json:"fields"`
	Range     string `json:"range"`
	Snapshot  string `json:"snapshot"`
	Collation string `json:"collation"`
}
<commit_msg>Update sync_diff_inspector/checkpoints/checkpoints.go<commit_after>// Copyright 2021 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package checkpoints

import (
	"container/heap"
	"context"
	"encoding/json"
	"os"
	"sync"

	"github.com/pingcap/tidb-tools/sync_diff_inspector/config"
	"github.com/pingcap/tidb-tools/sync_diff_inspector/report"

	"github.com/pingcap/tidb-tools/sync_diff_inspector/chunk"
	"github.com/siddontang/go/ioutil2"

	"github.com/pingcap/errors"
	"github.com/pingcap/log"
	"go.uber.org/zap"
)

const (
	// SuccessState
	// for chunk: means this chunk's data is equal
	// for table: means this all chunk in this table is equal(except ignore chunk)
	SuccessState = "success"

	// FailedState
	// for chunk: means this chunk's data is not equal
	// for table: means some chunks' data is not equal or some chunk check failed in this table
	FailedState = "failed"

	// IgnoreState
	// for chunk: this chunk is ignored. if sample is not 100% or it is Empty chunk, will ignore some chunk
	// for table: don't have this state
	IgnoreState = "ignore"
)

type Node struct {
	State string `json:"state"` // indicate the state ("success" or "failed") of the chunk

	ChunkRange *chunk.Range `json:"chunk-range"`
	IndexID    int64        `json:"index-id"`
}

func (n *Node) GetID() *chunk.ChunkID { return n.ChunkRange.Index }

func (n *Node) GetState() string { return n.State }

func (n *Node) GetTableIndex() int { return n.ChunkRange.Index.TableIndex }

func (n *Node) GetBucketIndexLeft() int { return n.ChunkRange.Index.BucketIndexLeft }

func (n *Node) GetBucketIndexRight() int { return n.ChunkRange.Index.BucketIndexRight }

func (n *Node) GetChunkIndex() int { return n.ChunkRange.Index.ChunkIndex }

// IsAdjacent represents whether the next node is adjacent node.
// it's the important logic for checkpoint update.
// we need keep this node save to checkpoint in global order.
func (n *Node) IsAdjacent(next *Node) bool {
	if n.GetTableIndex() == next.GetTableIndex()-1 {
		if n.ChunkRange.IsLastChunkForTable() && next.ChunkRange.IsFirstChunkForTable() {
			return true
		}
		return false
	}
	if n.GetTableIndex() == next.GetTableIndex() {
		// same table
		if n.GetBucketIndexRight() == next.GetBucketIndexLeft()-1 {
			if n.ChunkRange.IsLastChunkForBucket() && next.ChunkRange.IsFirstChunkForBucket() {
				return true
			}
			return false
		}
		if n.GetBucketIndexLeft() == next.GetBucketIndexLeft() {
			return n.GetChunkIndex() == next.GetChunkIndex()-1
		}
		return false
	}
	return false
}

// IsLess represents whether the cur node is less than next node.
func (n *Node) IsLess(next *Node) bool {
	if n.GetTableIndex() < next.GetTableIndex() {
		return true
	}
	if n.GetTableIndex() == next.GetTableIndex() {
		if n.GetBucketIndexLeft() <= next.GetBucketIndexLeft()-1 {
			return true
		}
		if n.GetBucketIndexLeft() == next.GetBucketIndexLeft() {
			return n.GetChunkIndex() < next.GetChunkIndex()
		}
		return false
	}
	return false
}

// heap maintain a Min Heap, which can be accessed by multiple threads and protected by mutex.
type nodeHeap struct {
	Nodes            []*Node
	CurrentSavedNode *Node       // CurrentSavedNode save the minimum checker chunk, updated by `GetChunkSnapshot` method
	mu               *sync.Mutex // protect critical section
}

// Checkpoint provide the ability to restart the sync-diff process from the
// latest previous exit point (due to error or intention).
type Checkpoint struct {
	hp *nodeHeap
}

// SaveState contains the information of the latest checked chunk and state of `report`
// When sync-diff start from the checkpoint, it will load this information and continue running
type SavedState struct {
	Chunk  *Node          `json:"chunk-info"`
	Report *report.Report `json:"report-info"`
}

// SetCurrentSavedID the method is unsynchronized, be cautious
func (cp *Checkpoint) SetCurrentSavedID(n *Node) {
	cp.hp.CurrentSavedNode = n
}

func (cp *Checkpoint) GetCurrentSavedID() *Node {
	cp.hp.mu.Lock()
	defer cp.hp.mu.Unlock()
	return cp.hp.CurrentSavedNode
}

func (cp *Checkpoint) Insert(node *Node) {
	cp.hp.mu.Lock()
	heap.Push(cp.hp, node)
	cp.hp.mu.Unlock()
}

// Len - get the length of the heap
func (hp *nodeHeap) Len() int { return len(hp.Nodes) }

// Less - determine which is more priority than another
func (hp *nodeHeap) Less(i, j int) bool {
	return hp.Nodes[i].IsLess(hp.Nodes[j])
}

// Swap - implementation of swap for the heap interface
func (hp *nodeHeap) Swap(i, j int) {
	hp.Nodes[i], hp.Nodes[j] = hp.Nodes[j], hp.Nodes[i]
}

// Push - implementation of push for the heap interface
func (hp *nodeHeap) Push(x interface{}) {
	hp.Nodes = append(hp.Nodes, x.(*Node))
}

// Pop - implementation of pop for heap interface
func (hp *nodeHeap) Pop() (item interface{}) {
	if len(hp.Nodes) == 0 {
		return
	}

	hp.Nodes, item = hp.Nodes[:len(hp.Nodes)-1], hp.Nodes[len(hp.Nodes)-1]
	return
}

func (cp *Checkpoint) Init() {
	hp := &nodeHeap{
		mu:    &sync.Mutex{},
		Nodes: make([]*Node, 0),
		CurrentSavedNode: &Node{
			ChunkRange: &chunk.Range{
				Index:   chunk.GetInitChunkID(),
				IsFirst: true,
				IsLast:  true,
			},
		},
	}
	heap.Init(hp)
	cp.hp = hp
}

// GetChunkSnapshot get the snapshot of the minimum continuous checked chunk
func (cp *Checkpoint) GetChunkSnapshot() (cur *Node) {
	cp.hp.mu.Lock()
	defer cp.hp.mu.Unlock()
	for cp.hp.Len() != 0 && cp.hp.CurrentSavedNode.IsAdjacent(cp.hp.Nodes[0]) {
		cp.hp.CurrentSavedNode = heap.Pop(cp.hp).(*Node)
		cur = cp.hp.CurrentSavedNode
	}
	// wait for next 10s to check
	return cur
}

// SaveChunk saves the chunk to file.
func (cp *Checkpoint) SaveChunk(ctx context.Context, fileName string, cur *Node, reportInfo *report.Report) (*chunk.ChunkID, error) {
	if cur == nil {
		return nil, nil
	}

	savedState := &SavedState{
		Chunk:  cur,
		Report: reportInfo,
	}
	checkpointData, err := json.Marshal(savedState)
	if err != nil {
		log.Warn("fail to save the chunk to the file", zap.Any("chunk index", cur.GetID()), zap.Error(err))
		return nil, errors.Trace(err)
	}

	if err = ioutil2.WriteFileAtomic(fileName, checkpointData, config.LocalFilePerm); err != nil {
		return nil, err
	}
	log.Info("save checkpoint",
		zap.Any("chunk", cur),
		zap.String("state", cur.GetState()))
	return cur.GetID(), nil
}

// LoadChunk loads chunk info from file `chunk`
func (cp *Checkpoint) LoadChunk(fileName string) (*Node, *report.Report, error) {
	bytes, err := os.ReadFile(fileName)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	n := &SavedState{}
	err = json.Unmarshal(bytes, n)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	return n.Chunk, n.Report, nil
}

// CheckConfig stores the information about the config of the current run.
// If we restart a sync-diff from a different config, the checkpoint is disable.
type CheckConfig struct {
	Table     string `json:"table"`
	Fields    string `json:"fields"`
	Range     string `json:"range"`
	Snapshot  string `json:"snapshot"`
	Collation string `json:"collation"`
}
<|endoftext|>


<commit_before>package scheduling

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	"path"
	"strconv"
	"strings"
	"time"

	swarming_api "go.chromium.org/luci/common/api/swarming/swarming/v1"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/isolate"
	"go.skia.org/infra/go/swarming"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/task_scheduler/go/db"
	"go.skia.org/infra/task_scheduler/go/specs"
)

// taskCandidate is a struct used for determining which tasks to schedule.
type taskCandidate struct {
	Attempt int `json:"attempt"`
	// NB: Because multiple Jobs may share a Task, the BuildbucketBuildId
	// could be inherited from any matching Job. Therefore, this should be
	// used for non-critical, informational purposes only.
	BuildbucketBuildId int64     `json:"buildbucketBuildId"`
	Commits            []string  `json:"commits"`
	IsolatedInput      string    `json:"isolatedInput"`
	IsolatedHashes     []string  `json:"isolatedHashes"`
	JobCreated         time.Time `json:"jobCreated"`
	Jobs               []string  `json:"jobs"`
	ParentTaskIds      []string  `json:"parentTaskIds"`
	RetryOf            string    `json:"retryOf"`
	Score              float64   `json:"score"`
	StealingFromId     string    `json:"stealingFromId"`
	db.TaskKey
	TaskSpec *specs.TaskSpec `json:"taskSpec"`
}

// Copy returns a copy of the taskCandidate.
func (c *taskCandidate) Copy() *taskCandidate {
	return &taskCandidate{
		Attempt:            c.Attempt,
		BuildbucketBuildId: c.BuildbucketBuildId,
		Commits:            util.CopyStringSlice(c.Commits),
		IsolatedInput:      c.IsolatedInput,
		IsolatedHashes:     util.CopyStringSlice(c.IsolatedHashes),
		JobCreated:         c.JobCreated,
		Jobs:               util.CopyStringSlice(c.Jobs),
		ParentTaskIds:      util.CopyStringSlice(c.ParentTaskIds),
		RetryOf:            c.RetryOf,
		Score:              c.Score,
		StealingFromId:     c.StealingFromId,
		TaskKey:            c.TaskKey.Copy(),
		TaskSpec:           c.TaskSpec.Copy(),
	}
}

// MakeId generates a string ID for the taskCandidate.
func (c *taskCandidate) MakeId() string {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(&c.TaskKey); err != nil {
		panic(fmt.Sprintf("Failed to GOB encode TaskKey: %s", err))
	}
	b64 := base64.StdEncoding.EncodeToString(buf.Bytes())
	return fmt.Sprintf("taskCandidate|%s", b64)
}

// parseId generates taskCandidate information from the ID.
func parseId(id string) (db.TaskKey, error) {
	var rv db.TaskKey
	split := strings.Split(id, "|")
	if len(split) != 2 {
		return rv, fmt.Errorf("Invalid ID, not enough parts: %q", id)
	}
	if split[0] != "taskCandidate" {
		return rv, fmt.Errorf("Invalid ID, no 'taskCandidate' prefix: %q", id)
	}
	b, err := base64.StdEncoding.DecodeString(split[1])
	if err != nil {
		return rv, fmt.Errorf("Failed to base64 decode ID: %s", err)
	}
	if err := gob.NewDecoder(bytes.NewBuffer(b)).Decode(&rv); err != nil {
		return rv, fmt.Errorf("Failed to GOB decode ID: %s", err)
	}
	return rv, nil
}

// MakeTask instantiates a db.Task from the taskCandidate.
func (c *taskCandidate) MakeTask() *db.Task {
	commits := make([]string, len(c.Commits))
	copy(commits, c.Commits)
	jobs := make([]string, len(c.Jobs))
	copy(jobs, c.Jobs)
	parentTaskIds := make([]string, len(c.ParentTaskIds))
	copy(parentTaskIds, c.ParentTaskIds)
	maxAttempts := c.TaskSpec.MaxAttempts
	if maxAttempts == 0 {
		maxAttempts = specs.DEFAULT_TASK_SPEC_MAX_ATTEMPTS
	}
	return &db.Task{
		Attempt:       c.Attempt,
		Commits:       commits,
		Id:            "", // Filled in when the task is inserted into the DB.
		Jobs:          jobs,
		MaxAttempts:   maxAttempts,
		ParentTaskIds: parentTaskIds,
		RetryOf:       c.RetryOf,
		TaskKey:       c.TaskKey.Copy(),
	}
}

// MakeIsolateTask creates an isolate.Task from this taskCandidate.
func (c *taskCandidate) MakeIsolateTask(infraBotsDir, baseDir string) *isolate.Task {
	os := "linux"
	for _, d := range c.TaskSpec.Dimensions {
		if strings.HasPrefix(d, "os:") {
			os = d[len("os:"):]
			break
		}
	}
	return &isolate.Task{
		BaseDir:     baseDir,
		Blacklist:   isolate.DEFAULT_BLACKLIST,
		Deps:        c.IsolatedHashes,
		IsolateFile: path.Join(infraBotsDir, c.TaskSpec.Isolate),
		OsType:      os,
	}
}

// getPatchStorage returns "gerrit" or "" based on the Server URL.
func getPatchStorage(server string) string {
	if server == "" {
		return ""
	}
	return "gerrit"
}

// replaceVars replaces variable names with their values in a given string.
func replaceVars(c *taskCandidate, s, taskId string) string {
	issueShort := ""
	if len(c.Issue) < specs.ISSUE_SHORT_LENGTH {
		issueShort = c.Issue
	} else {
		issueShort = c.Issue[len(c.Issue)-specs.ISSUE_SHORT_LENGTH:]
	}
	replacements := map[string]string{
		specs.VARIABLE_BUILDBUCKET_BUILD_ID: strconv.FormatInt(c.BuildbucketBuildId, 10),
		specs.VARIABLE_CODEREVIEW_SERVER:    c.Server,
		specs.VARIABLE_ISSUE:                c.Issue,
		specs.VARIABLE_ISSUE_SHORT:          issueShort,
		specs.VARIABLE_PATCH_REPO:           c.PatchRepo,
		specs.VARIABLE_PATCH_STORAGE:        getPatchStorage(c.Server),
		specs.VARIABLE_PATCHSET:             c.Patchset,
		specs.VARIABLE_REPO:                 c.Repo,
		specs.VARIABLE_REVISION:             c.Revision,
		specs.VARIABLE_TASK_ID:              taskId,
		specs.VARIABLE_TASK_NAME:            c.Name,
	}
	for k, v := range replacements {
		s = strings.Replace(s, fmt.Sprintf(specs.VARIABLE_SYNTAX, k), v, -1)
	}
	return s
}

// MakeTaskRequest creates a SwarmingRpcsNewTaskRequest object from the taskCandidate.
func (c *taskCandidate) MakeTaskRequest(id, isolateServer, pubSubTopic string) (*swarming_api.SwarmingRpcsNewTaskRequest, error) {
	var cipdInput *swarming_api.SwarmingRpcsCipdInput
	if len(c.TaskSpec.CipdPackages) > 0 {
		cipdInput = &swarming_api.SwarmingRpcsCipdInput{
			Packages: make([]*swarming_api.SwarmingRpcsCipdPackage, 0, len(c.TaskSpec.CipdPackages)),
		}
		for _, p := range c.TaskSpec.CipdPackages {
			cipdInput.Packages = append(cipdInput.Packages, &swarming_api.SwarmingRpcsCipdPackage{
				PackageName: p.Name,
				Path:        p.Path,
				Version:     p.Version,
			})
		}
	}

	dims := make([]*swarming_api.SwarmingRpcsStringPair, 0, len(c.TaskSpec.Dimensions))
	dimsMap := make(map[string]string, len(c.TaskSpec.Dimensions))
	for _, d := range c.TaskSpec.Dimensions {
		split := strings.SplitN(d, ":", 2)
		key := split[0]
		val := split[1]
		dims = append(dims, &swarming_api.SwarmingRpcsStringPair{
			Key:   key,
			Value: val,
		})
		dimsMap[key] = val
	}

	var env []*swarming_api.SwarmingRpcsStringPair
	if len(c.TaskSpec.Environment) > 0 {
		env = make([]*swarming_api.SwarmingRpcsStringPair, 0, len(c.TaskSpec.Environment))
		for k, v := range c.TaskSpec.Environment {
			env = append(env, &swarming_api.SwarmingRpcsStringPair{
				Key:   k,
				Value: v,
			})
		}
	}

	extraArgs := make([]string, 0, len(c.TaskSpec.ExtraArgs))
	for _, arg := range c.TaskSpec.ExtraArgs {
		extraArgs = append(extraArgs, replaceVars(c, arg, id))
	}

	expirationSecs := int64(c.TaskSpec.Expiration.Seconds())
	if expirationSecs == int64(0) {
		expirationSecs = int64(swarming.RECOMMENDED_EXPIRATION.Seconds())
	}
	executionTimeoutSecs := int64(c.TaskSpec.ExecutionTimeout.Seconds())
	if executionTimeoutSecs == int64(0) {
		executionTimeoutSecs = int64(swarming.RECOMMENDED_HARD_TIMEOUT.Seconds())
	}
	ioTimeoutSecs := int64(c.TaskSpec.IoTimeout.Seconds())
	if ioTimeoutSecs == int64(0) {
		ioTimeoutSecs = int64(swarming.RECOMMENDED_IO_TIMEOUT.Seconds())
	}
	return &swarming_api.SwarmingRpcsNewTaskRequest{
		ExpirationSecs: expirationSecs,
		Name:           c.Name,
		Priority:       int64(100.0 * c.TaskSpec.Priority),
		Properties: &swarming_api.SwarmingRpcsTaskProperties{
			CipdInput:            cipdInput,
			Dimensions:           dims,
			Env:                  env,
			ExecutionTimeoutSecs: executionTimeoutSecs,
			ExtraArgs:            extraArgs,
			InputsRef: &swarming_api.SwarmingRpcsFilesRef{
				Isolated:       c.IsolatedInput,
				Isolatedserver: isolateServer,
				Namespace:      isolate.DEFAULT_NAMESPACE,
			},
			IoTimeoutSecs: ioTimeoutSecs,
		},
		PubsubTopic:    fmt.Sprintf(swarming.PUBSUB_FULLY_QUALIFIED_TOPIC_TMPL, common.PROJECT_ID, pubSubTopic),
		ServiceAccount: swarming.GetServiceAccountFromTaskDims(dimsMap),
		Tags:           db.TagsForTask(c.Name, id, c.Attempt, c.TaskSpec.Priority, c.RepoState, c.RetryOf, dimsMap, c.ForcedJobId, c.ParentTaskIds),
		User:           "skiabot@google.com",
	}, nil
}

// allDepsMet determines whether all dependencies for the given task candidate
// have been satisfied, and if so, returns a map of whose keys are task IDs and
// values are their isolated outputs.
func (c *taskCandidate) allDepsMet(cache db.TaskCache) (bool, map[string]string, error) {
	rv := make(map[string]string, len(c.TaskSpec.Dependencies))
	for _, depName := range c.TaskSpec.Dependencies {
		key := c.TaskKey.Copy()
		key.Name = depName
		byKey, err := cache.GetTasksByKey(&key)
		if err != nil {
			return false, nil, err
		}
		ok := false
		for _, t := range byKey {
			if t.Done() && t.Success() && t.IsolatedOutput != "" {
				rv[t.Id] = t.IsolatedOutput
				ok = true
				break
			}
		}
		if !ok {
			return false, nil, nil
		}
	}
	return true, rv, nil
}

// taskCandidateSlice is an alias used for sorting a slice of taskCandidates.
type taskCandidateSlice []*taskCandidate

func (s taskCandidateSlice) Len() int { return len(s) }
func (s taskCandidateSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s taskCandidateSlice) Less(i, j int) bool {
	return s[i].Score > s[j].Score // candidates sort in decreasing order.
}
<commit_msg>[task scheduler] Always set idempotent=true for non-forced tasks<commit_after>package scheduling

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	"path"
	"strconv"
	"strings"
	"time"

	swarming_api "go.chromium.org/luci/common/api/swarming/swarming/v1"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/isolate"
	"go.skia.org/infra/go/swarming"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/task_scheduler/go/db"
	"go.skia.org/infra/task_scheduler/go/specs"
)

// taskCandidate is a struct used for determining which tasks to schedule.
type taskCandidate struct {
	Attempt int `json:"attempt"`
	// NB: Because multiple Jobs may share a Task, the BuildbucketBuildId
	// could be inherited from any matching Job. Therefore, this should be
	// used for non-critical, informational purposes only.
	BuildbucketBuildId int64     `json:"buildbucketBuildId"`
	Commits            []string  `json:"commits"`
	IsolatedInput      string    `json:"isolatedInput"`
	IsolatedHashes     []string  `json:"isolatedHashes"`
	JobCreated         time.Time `json:"jobCreated"`
	Jobs               []string  `json:"jobs"`
	ParentTaskIds      []string  `json:"parentTaskIds"`
	RetryOf            string    `json:"retryOf"`
	Score              float64   `json:"score"`
	StealingFromId     string    `json:"stealingFromId"`
	db.TaskKey
	TaskSpec *specs.TaskSpec `json:"taskSpec"`
}

// Copy returns a copy of the taskCandidate.
func (c *taskCandidate) Copy() *taskCandidate {
	return &taskCandidate{
		Attempt:            c.Attempt,
		BuildbucketBuildId: c.BuildbucketBuildId,
		Commits:            util.CopyStringSlice(c.Commits),
		IsolatedInput:      c.IsolatedInput,
		IsolatedHashes:     util.CopyStringSlice(c.IsolatedHashes),
		JobCreated:         c.JobCreated,
		Jobs:               util.CopyStringSlice(c.Jobs),
		ParentTaskIds:      util.CopyStringSlice(c.ParentTaskIds),
		RetryOf:            c.RetryOf,
		Score:              c.Score,
		StealingFromId:     c.StealingFromId,
		TaskKey:            c.TaskKey.Copy(),
		TaskSpec:           c.TaskSpec.Copy(),
	}
}

// MakeId generates a string ID for the taskCandidate.
func (c *taskCandidate) MakeId() string {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(&c.TaskKey); err != nil {
		panic(fmt.Sprintf("Failed to GOB encode TaskKey: %s", err))
	}
	b64 := base64.StdEncoding.EncodeToString(buf.Bytes())
	return fmt.Sprintf("taskCandidate|%s", b64)
}

// parseId generates taskCandidate information from the ID.
func parseId(id string) (db.TaskKey, error) {
	var rv db.TaskKey
	split := strings.Split(id, "|")
	if len(split) != 2 {
		return rv, fmt.Errorf("Invalid ID, not enough parts: %q", id)
	}
	if split[0] != "taskCandidate" {
		return rv, fmt.Errorf("Invalid ID, no 'taskCandidate' prefix: %q", id)
	}
	b, err := base64.StdEncoding.DecodeString(split[1])
	if err != nil {
		return rv, fmt.Errorf("Failed to base64 decode ID: %s", err)
	}
	if err := gob.NewDecoder(bytes.NewBuffer(b)).Decode(&rv); err != nil {
		return rv, fmt.Errorf("Failed to GOB decode ID: %s", err)
	}
	return rv, nil
}

// MakeTask instantiates a db.Task from the taskCandidate.
func (c *taskCandidate) MakeTask() *db.Task {
	commits := make([]string, len(c.Commits))
	copy(commits, c.Commits)
	jobs := make([]string, len(c.Jobs))
	copy(jobs, c.Jobs)
	parentTaskIds := make([]string, len(c.ParentTaskIds))
	copy(parentTaskIds, c.ParentTaskIds)
	maxAttempts := c.TaskSpec.MaxAttempts
	if maxAttempts == 0 {
		maxAttempts = specs.DEFAULT_TASK_SPEC_MAX_ATTEMPTS
	}
	return &db.Task{
		Attempt:       c.Attempt,
		Commits:       commits,
		Id:            "", // Filled in when the task is inserted into the DB.
		Jobs:          jobs,
		MaxAttempts:   maxAttempts,
		ParentTaskIds: parentTaskIds,
		RetryOf:       c.RetryOf,
		TaskKey:       c.TaskKey.Copy(),
	}
}

// MakeIsolateTask creates an isolate.Task from this taskCandidate.
func (c *taskCandidate) MakeIsolateTask(infraBotsDir, baseDir string) *isolate.Task {
	os := "linux"
	for _, d := range c.TaskSpec.Dimensions {
		if strings.HasPrefix(d, "os:") {
			os = d[len("os:"):]
			break
		}
	}
	return &isolate.Task{
		BaseDir:     baseDir,
		Blacklist:   isolate.DEFAULT_BLACKLIST,
		Deps:        c.IsolatedHashes,
		IsolateFile: path.Join(infraBotsDir, c.TaskSpec.Isolate),
		OsType:      os,
	}
}

// getPatchStorage returns "gerrit" or "" based on the Server URL.
func getPatchStorage(server string) string {
	if server == "" {
		return ""
	}
	return "gerrit"
}

// replaceVars replaces variable names with their values in a given string.
func replaceVars(c *taskCandidate, s, taskId string) string {
	issueShort := ""
	if len(c.Issue) < specs.ISSUE_SHORT_LENGTH {
		issueShort = c.Issue
	} else {
		issueShort = c.Issue[len(c.Issue)-specs.ISSUE_SHORT_LENGTH:]
	}
	replacements := map[string]string{
		specs.VARIABLE_BUILDBUCKET_BUILD_ID: strconv.FormatInt(c.BuildbucketBuildId, 10),
		specs.VARIABLE_CODEREVIEW_SERVER:    c.Server,
		specs.VARIABLE_ISSUE:                c.Issue,
		specs.VARIABLE_ISSUE_SHORT:          issueShort,
		specs.VARIABLE_PATCH_REPO:           c.PatchRepo,
		specs.VARIABLE_PATCH_STORAGE:        getPatchStorage(c.Server),
		specs.VARIABLE_PATCHSET:             c.Patchset,
		specs.VARIABLE_REPO:                 c.Repo,
		specs.VARIABLE_REVISION:             c.Revision,
		specs.VARIABLE_TASK_ID:              taskId,
		specs.VARIABLE_TASK_NAME:            c.Name,
	}
	for k, v := range replacements {
		s = strings.Replace(s, fmt.Sprintf(specs.VARIABLE_SYNTAX, k), v, -1)
	}
	return s
}

// MakeTaskRequest creates a SwarmingRpcsNewTaskRequest object from the taskCandidate.
func (c *taskCandidate) MakeTaskRequest(id, isolateServer, pubSubTopic string) (*swarming_api.SwarmingRpcsNewTaskRequest, error) {
	var cipdInput *swarming_api.SwarmingRpcsCipdInput
	if len(c.TaskSpec.CipdPackages) > 0 {
		cipdInput = &swarming_api.SwarmingRpcsCipdInput{
			Packages: make([]*swarming_api.SwarmingRpcsCipdPackage, 0, len(c.TaskSpec.CipdPackages)),
		}
		for _, p := range c.TaskSpec.CipdPackages {
			cipdInput.Packages = append(cipdInput.Packages, &swarming_api.SwarmingRpcsCipdPackage{
				PackageName: p.Name,
				Path:        p.Path,
				Version:     p.Version,
			})
		}
	}

	dims := make([]*swarming_api.SwarmingRpcsStringPair, 0, len(c.TaskSpec.Dimensions))
	dimsMap := make(map[string]string, len(c.TaskSpec.Dimensions))
	for _, d := range c.TaskSpec.Dimensions {
		split := strings.SplitN(d, ":", 2)
		key := split[0]
		val := split[1]
		dims = append(dims, &swarming_api.SwarmingRpcsStringPair{
			Key:   key,
			Value: val,
		})
		dimsMap[key] = val
	}

	var env []*swarming_api.SwarmingRpcsStringPair
	if len(c.TaskSpec.Environment) > 0 {
		env = make([]*swarming_api.SwarmingRpcsStringPair, 0, len(c.TaskSpec.Environment))
		for k, v := range c.TaskSpec.Environment {
			env = append(env, &swarming_api.SwarmingRpcsStringPair{
				Key:   k,
				Value: v,
			})
		}
	}

	extraArgs := make([]string, 0, len(c.TaskSpec.ExtraArgs))
	for _, arg := range c.TaskSpec.ExtraArgs {
		extraArgs = append(extraArgs, replaceVars(c, arg, id))
	}

	expirationSecs := int64(c.TaskSpec.Expiration.Seconds())
	if expirationSecs == int64(0) {
		expirationSecs = int64(swarming.RECOMMENDED_EXPIRATION.Seconds())
	}
	executionTimeoutSecs := int64(c.TaskSpec.ExecutionTimeout.Seconds())
	if executionTimeoutSecs == int64(0) {
		executionTimeoutSecs = int64(swarming.RECOMMENDED_HARD_TIMEOUT.Seconds())
	}
	ioTimeoutSecs := int64(c.TaskSpec.IoTimeout.Seconds())
	if ioTimeoutSecs == int64(0) {
		ioTimeoutSecs = int64(swarming.RECOMMENDED_IO_TIMEOUT.Seconds())
	}
	return &swarming_api.SwarmingRpcsNewTaskRequest{
		ExpirationSecs: expirationSecs,
		Name:           c.Name,
		Priority:       int64(100.0 * c.TaskSpec.Priority),
		Properties: &swarming_api.SwarmingRpcsTaskProperties{
			CipdInput:            cipdInput,
			Dimensions:           dims,
			Env:                  env,
			ExecutionTimeoutSecs: executionTimeoutSecs,
			ExtraArgs:            extraArgs,
			Idempotent:           c.ForcedJobId == "",
			InputsRef: &swarming_api.SwarmingRpcsFilesRef{
				Isolated:       c.IsolatedInput,
				Isolatedserver: isolateServer,
				Namespace:      isolate.DEFAULT_NAMESPACE,
			},
			IoTimeoutSecs: ioTimeoutSecs,
		},
		PubsubTopic:    fmt.Sprintf(swarming.PUBSUB_FULLY_QUALIFIED_TOPIC_TMPL, common.PROJECT_ID, pubSubTopic),
		ServiceAccount: swarming.GetServiceAccountFromTaskDims(dimsMap),
		Tags:           db.TagsForTask(c.Name, id, c.Attempt, c.TaskSpec.Priority, c.RepoState, c.RetryOf, dimsMap, c.ForcedJobId, c.ParentTaskIds),
		User:           "skiabot@google.com",
	}, nil
}

// allDepsMet determines whether all dependencies for the given task candidate
// have been satisfied, and if so, returns a map of whose keys are task IDs and
// values are their isolated outputs.
func (c *taskCandidate) allDepsMet(cache db.TaskCache) (bool, map[string]string, error) {
	rv := make(map[string]string, len(c.TaskSpec.Dependencies))
	for _, depName := range c.TaskSpec.Dependencies {
		key := c.TaskKey.Copy()
		key.Name = depName
		byKey, err := cache.GetTasksByKey(&key)
		if err != nil {
			return false, nil, err
		}
		ok := false
		for _, t := range byKey {
			if t.Done() && t.Success() && t.IsolatedOutput != "" {
				rv[t.Id] = t.IsolatedOutput
				ok = true
				break
			}
		}
		if !ok {
			return false, nil, nil
		}
	}
	return true, rv, nil
}

// taskCandidateSlice is an alias used for sorting a slice of taskCandidates.
type taskCandidateSlice []*taskCandidate

func (s taskCandidateSlice) Len() int { return len(s) }
func (s taskCandidateSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s taskCandidateSlice) Less(i, j int) bool {
	return s[i].Score > s[j].Score // candidates sort in decreasing order.
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
)

func main() {
	filename := os.Args[1]
	colnum, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Println("Please supply the column number as the second argument")
		panic(err)
	}
	outfilename := os.Args[3]
	outplainwords := os.Args[4]
	plainwords := false
	if outplainwords != "-" {
		plainwords = true
	}

	//regular users will start counting cols from 1
	colnum--

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Println("Could not read file")
		panic(err)
	}

	datastring := string(data)
	rows := strings.Split(datastring, "\n")

	count := make(map[string]int)
	plainwordsstring := ""

	for _, row := range rows {
		//ignore XML lines
		if !strings.HasPrefix(row, "<") && !strings.HasPrefix(row, "#") && row != "" && row != " " {
			fields := strings.Split(row, "\t")
			if fields[colnum] != "." && fields[colnum] != "..." && fields[colnum] != "?" && fields[colnum] != "!" && fields[colnum] != ":" &&
				fields[colnum] != ";" && fields[colnum] != "_" && fields[colnum] != "," && fields[colnum] != "?!" && fields[colnum] != "!?" &&
				fields[colnum] != "???" && fields[colnum] != "!!!" && fields[colnum] != "!!" && fields[colnum] != "??" && fields[colnum] != "&" &&
				fields[colnum] != "(" && fields[colnum] != ")" && fields[colnum] != "*" && fields[colnum] != "'" && fields[colnum] != "\"" &&
				fields[colnum] != "\\" && fields[colnum] != "/" && fields[colnum] != "|" && fields[colnum] != "+" && fields[colnum] != "=" &&
				fields[colnum] != "[" && fields[colnum] != "]" && fields[colnum] != "{" && fields[colnum] != "}" && fields[colnum] != "<" &&
				fields[colnum] != ">" && fields[colnum] != "-" {
				token := strings.Trim(strings.ToLower(fields[colnum]), " ")
				if len(token) < 1 {
					continue
				}
				count[token]++
				if plainwords {
					plainwordsstring += token + "\t"
				}
			}
		}
	}

	var result bytes.Buffer
	filetotal := 0

	for key, value := range count {
		result.WriteString(key)
		result.WriteString("\t")
		valuestring := strconv.Itoa(value)
		result.WriteString(valuestring)
		result.WriteString("\n")
		filetotal += value
	}
	result.WriteString("total.snelslim")
	result.WriteString("\t")
	valuestring := strconv.Itoa(filetotal)
	result.WriteString(valuestring)
	result.WriteString("\n")

	err = ioutil.WriteFile(outfilename, result.Bytes(), 0644)
	if err != nil {
		fmt.Println("Could not write result")
		panic(err)
	}

	if plainwords {
		err = ioutil.WriteFile(outplainwords, []byte(plainwordsstring), 0644)
		if err != nil {
			fmt.Println("Could not write plainwords for collocational analysis")
			panic(err)
		}
	}

	fmt.Println(filetotal)
	fmt.Println("OK")
}
<commit_msg>Replace checks for punctuation and symbols in CoNLL parser with unicode category system from XPath parser<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"unicode"
)

func main() {
	filename := os.Args[1]
	colnum, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Println("Please supply the column number as the second argument")
		panic(err)
	}
	outfilename := os.Args[3]
	outplainwords := os.Args[4]
	plainwords := false
	if outplainwords != "-" {
		plainwords = true
	}

	//regular users will start counting cols from 1
	colnum--

	data, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Println("Could not read file")
		panic(err)
	}

	datastring := string(data)
	rows := strings.Split(datastring, "\n")

	count := make(map[string]int)
	plainwordsstring := ""

	for _, row := range rows {
		//ignore XML lines
		if !strings.HasPrefix(row, "<") && !strings.HasPrefix(row, "#") && row != "" && row != " " {
			fields := strings.Split(row, "\t")
			token := strings.Trim(strings.ToLower(fields[colnum]), " ")
			if len(token) < 1 {
				continue
			}
			if isIrrelevantToken(token) {
				continue
			}
			count[token]++
			if plainwords {
				plainwordsstring += token + "\t"
			}
		}
	}

	var result bytes.Buffer
	filetotal := 0

	for key, value := range count {
		result.WriteString(key)
		result.WriteString("\t")
		valuestring := strconv.Itoa(value)
		result.WriteString(valuestring)
		result.WriteString("\n")
		filetotal += value
	}
	result.WriteString("total.snelslim")
	result.WriteString("\t")
	valuestring := strconv.Itoa(filetotal)
	result.WriteString(valuestring)
	result.WriteString("\n")

	err = ioutil.WriteFile(outfilename, result.Bytes(), 0644)
	if err != nil {
		fmt.Println("Could not write result")
		panic(err)
	}

	if plainwords {
		err = ioutil.WriteFile(outplainwords, []byte(plainwordsstring), 0644)
		if err != nil {
			fmt.Println("Could not write plainwords for collocational analysis")
			panic(err)
		}
	}

	fmt.Println(filetotal)
	fmt.Println("OK")
}

func isIrrelevantToken(token string) bool {
	for _, rune := range token {
		if !unicode.In(rune, unicode.Cf, unicode.Punct, unicode.Sk, unicode.Sm, unicode.So, unicode.Z) {
			return false
		}
	}
	return true
}
<|endoftext|>