repo_description: HTTP, HTTP2, HTTPS, Websocket debugging proxy
repo_topics: charles debug fiddler hosts node nodejs proxy web weinre
code: var net = require('net');
  var util = require('./index');
  var logger = require('./logger');
  var config = require('../config');
  var socketMgr = require('../socket-mgr');
  
  var version = config.version;
  var nodeVersion = process.version.substring(1);
  
  if (!(config.reqCacheSize > 0) || config.reqCacheSize < 600) {
    config.reqCacheSize = 600;
  }
  
  if (!(config.frameCacheSize > 0) || config.frameCacheSize < 720) {
    config.frameCacheSize = 600;
  }
  
  var CLEAR_INTERVAL = 6000;
  var CACHE_TIME = 1000 * 60 * 2;
  var MAX_CACHE_TIME = 1000 * 60 * 6;
  var MAX_LENGTH = config.reqCacheSize;
  var OVERFLOW_LENGTH = MAX_LENGTH * 3;
  var MAX_CACHE_SIZE = MAX_LENGTH * 2;
  var PRESERVE_LEN = 360;
  var MAX_FRAMES_LENGTH = config.frameCacheSize;
  var COUNT = 100;
  var count = 0;
  var ids = [];
  var reqData = {};
  var framesCache = [];
  var framesMap = {};
  var proxy, binded;
  var clearCount = 0;
  
  function enable() {
    if (binded) {
      return;
    }
    binded = true;
    proxy.on('request', handleRequest);
    proxy.on('frame', handleFrame);
    setInterval(clearCache, CLEAR_INTERVAL);
  }
  
  /**
   * 如果超过最大缓存数，清理如下请求数据：
   * 1. 已经请求结束且结束时间超过10秒
   * 2. 请求#1前面的未结束且未被ui读取过的请求
   */
  
  var MAX_BUF_LEN1 = 1024 * 512;
  var MAX_BUF_LEN2 = 1024 * 256;
  var MAX_BUF_LEN3 = 1024 * 128;
  var MAX_BUF_LEN4 = 1024 * 64;
  var MIN1 = 1000 * 60;
  var MIN2 = MIN1 * 2;
  var MIN3 = MIN1 * 6;
  var MIN4 = MIN1 * 12;
  
  function reduceFrameSize(frame, len, interval, now) {
    var id = frame.frameId;
    if (now - id.substring(0, id.indexOf('-')) < interval) {
      return;
    }
    frame.len = len;
    var bin = frame.bin;
    var base64 = frame.base64;
    if (base64) {
      frame.base64 = null;
      bin = Buffer.from(base64, 'base64');
    }
    if (bin) {
      frame.bin = bin.slice(0, len);
    }
  }
  
  var clearFrames = function (frame, now) {
    var len = frame.len || frame.length;
    if (!len || len <= MAX_BUF_LEN4) {
      return;
    }
    if (len > MAX_BUF_LEN1) {
      return reduceFrameSize(frame, MAX_BUF_LEN1, MIN1, now);
    }
    if (len > MAX_BUF_LEN2) {
      return reduceFrameSize(frame, MAX_BUF_LEN2, MIN2, now);
    }
  
    if (len > MAX_BUF_LEN3) {
      return reduceFrameSize(frame, MAX_BUF_LEN3, MIN3, now);
    }
    reduceFrameSize(frame, MAX_BUF_LEN4, MIN4, now);
  };
  
  function clearCache() {
    var overflow = framesCache.length - MAX_FRAMES_LENGTH;
    var now = Date.now();
    // 1 分钟触发一次
    ++clearCount;
    if (clearCount > 10) {
      clearCount = 0;
    }
  
    if (overflow > 0) {
      framesCache.splice(0, overflow + 60);
      framesMap = {};
      framesCache.forEach(function (frame) {
        framesMap[frame.reqId] = frame;
        !clearCount && clearFrames(frame, now);
      });
    } else if (!clearCount) {
      framesCache.forEach(function (frame) {
        clearFrames(frame, now);
      });
    }
    var len = ids.length;
    if (len <= MAX_LENGTH) {
      return;
    }
  
    var _ids = [];
    var preserveLen = len;
    overflow = -1;
    if (len >= OVERFLOW_LENGTH) {
      overflow = len - MAX_CACHE_SIZE;
      preserveLen = len - PRESERVE_LEN;
    }
    var isTimeout = function (curData, i) {
      if (i < overflow) {
        return true;
      }
      return (
        curData.endTime &&
        now - curData.endTime > (i >= preserveLen ? MAX_CACHE_TIME : CACHE_TIME)
      );
    };
    for (var i = 0; i < len; i++) {
      var id = ids[i];
      var curData = reqData[id];
      if (isTimeout(curData, i)) {
        curData.abort && curData.abort(true);
        delete reqData[id];
      } else {
        if (curData.abort && now - curData.startTime > MAX_CACHE_TIME) {
          curData.abort(true);
        }
        _ids.push(id);
      }
    }
    ids = _ids;
  }
  // 不存在startTime相等的id
  function getIndex(startTime, start, end) {
    var midIndex = Math.floor((start + end) / 2);
    if (midIndex == start) {
      return end;
    }
    if (ids[midIndex] < startTime) {
      return getIndex(startTime, midIndex, end);
    }
    return getIndex(startTime, start, midIndex);
  }
  
  function toBase64String(data) {
    if (Buffer.isBuffer(data.body)) {
      data.base64 = data.body.toString('base64');
      data.body = '';
    }
  }
  
  function getIds(startTime, count, lastRowId) {
    startTime = startTime || lastRowId;
    if (!startTime) {
      return ids.slice(-count);
    }
    var index = 0;
    if (startTime !== '0') {
      index = ids.indexOf(startTime) + 1;
      if (!index && startTime.length > 6) {
        var startId = ids[0];
        if (startId && startId < startTime) {
          var end = ids.length - 1;
          if (ids[end] < startTime) {
            return [];
          }
          index = getIndex(startTime, 0, end);
        }
      }
    }
    return ids.slice(index, index + count);
  }
  
  function removeBase64(data) {
    var result = {};
    Object.keys(data).forEach(function(key) {
      if (key !== 'headers' && key !== 'base64' && key !== 'rawHeaderNames') {
        result[key] = data[key];
      }
    });
    return result;
  }
  
  function getBase64Len(base64) {
    if (!base64) {
      return 0;
    }
    var len = base64.length;
    if (base64[len - 1] === '=') {
      len -= 2;
      if (base64[len] === '=') {
        --len;
      }
    }
    return len;
  }
  
  function setBody(item, len) {
    if (!(len > 0)) {
      return item;
    }
    var base64 = item.base64;
    var curLen = getBase64Len(base64);
    item = removeBase64(item);
    if (len < curLen) {
      item.preLen = len;
      item.base64 = base64.substring(len);
    }
    return item;
  }
  
  function getList(ids, status) {
    if (!Array.isArray(ids)) {
      return [];
    }
    return ids.map(function (id, i) {
      var data = reqData[id];
      var opts = status && status[i];
      if (data) {
        toBase64String(data.req);
        toBase64String(data.res);
      }
      if (!data || typeof opts !== 'string') {
        return data;
      }
      opts = opts.split('-');
      var result = {};
      Object.keys(data).forEach(function(key) {
        if (key === 'url' || key === 'rulesHeaders' || (key === 'rules' && opts[1] != '0')) {
          return;
        }
        var item = data[key];
        if (key === 'req') {
          if (!opts[0]) {
            item = removeBase64(item);
          } else {
            item = setBody(item, +opts[0]);
          }
        } else if (key === 'res') {
          item = setBody(item, +opts[1]);
        }
        result[key] = item;
      });
      return result;
    });
  }
  
  function handleRequest(req, data) {
    var id = (data.id = data.id || data.startTime + '-' + ++count);
    var removeAbort = function () {
      if (data.abort) {
        delete data.abort;
      }
    };
    req.on('end', removeAbort);
    req.on('error', removeAbort);
    req.on('abort', removeAbort);
    data.version = version;
    data.nodeVersion = nodeVersion;
    reqData[id] = data;
    ids.indexOf(id) === -1 && ids.push(id);
  }
  
  function decodeData(frame) {
    if (frame.base64 == null) {
      frame.base64 = frame.bin ? frame.bin.toString('base64') : '';
      frame.bin = '';
    }
    return frame;
  }
  function handleFrame(data) {
    framesCache.push(data);
    framesMap[data.reqId] = data;
  }
  
  function getFrames(curReqId, lastFrameId) {
    if (!curReqId) {
      return;
    }
    var result = [];
    var lastFrame = framesMap[curReqId];
    if (lastFrame && (!lastFrameId || lastFrame.frameId > lastFrameId)) {
      var count = 16;
      for (var i = 0, len = framesCache.length; i < len; i++) {
        var frame = framesCache[i];
        if (
          frame.reqId === curReqId &&
          (!lastFrameId || frame.frameId > lastFrameId)
        ) {
          result.push(decodeData(frame));
          if (--count <= 0) {
            return result;
          }
        }
      }
    }
    return result;
  }
  
  function getLastFrame(curReqId) {
    var frame = framesMap[curReqId];
    if (frame && frame.reqId === curReqId) {
      return decodeData(frame);
    }
  }
  
  function getDomain(item) {
    var url = item && item.url;
    if (!url) {
      return '';
    }
    var index = url.indexOf('://');
    if (index === -1) {
      return '';
    }
    index += 3;
    url = url.substring(index);
    index = url.indexOf('/');
    if (index !== -1) {
      url = url.substring(0, index);
    }
    index = url.indexOf(':');
    if (index !== -1) {
      url = url.substring(0, index);
    }
    return url;
  }
  
  
  module.exports = function init(_proxy) {
    proxy = _proxy;
    enable();
    /**
     * options: {
     * 		startTime: timestamp || timestamp + '-' + count
     * 		count: 获取新数据的数量
     * 		ids: 请未结束的id列表
     * }
     *
     * @param options
     */
    function formatFilter(filter, clientIp, clientId) {
      if (!filter.url && !filter.name && !filter.value && !filter.ip) {
        return;
      }
      var url = util.trimStr(filter.url).toLowerCase();
      var ip = util.trimStr(filter.ip);
      var list = [];
      var cid;
      var result;
      if (ip === 'self') {
        ip = clientIp;
        cid = clientId;
      } else if (ip === 'clientIp') {
        ip = clientIp;
      }
      if (ip === 'clientId') {
        if (clientId) {
          cid = clientId;
        } else {
          result = { clientIp: clientIp };
        }
        ip = null;
      } else if (ip && !net.isIP(ip)) {
        ip.split(',').forEach(function (item) {
          item = item.trim();
          if (item === 'clientId') {
            cid = clientId;
          } else {
            if (item === 'self') {
              cid = clientId;
              item = clientIp;
            } else if (item === 'clientIp') {
              item = clientIp;
            }
            if (list.indexOf(item) === -1) {
              list.push(item);
            }
          }
        });
        ip = null;
      }
      if (url) {
        result = result || {};
        result.url = url;
      }
      var headers;
      for (var i = 0; i < 6; i++) {
        var key = 'name' + (i || '');
        var name = util.trimStr(filter[key]).toLowerCase();
        if (name) {
          result = result || {};
          var value = util.trimStr(filter['value' + (i || '')]).toLowerCase();
          if (i) {
            headers = headers || [];
            result.headers = headers;
            headers.push({
              name: name,
              value: value
            });
          } else {
            result.name = name;
            result.value = value;
          }
        }
      }
      if (ip) {
        result = result || {};
        result.ip = ip;
      }
      if (cid) {
        result = result || {};
        result.clientId = cid;
      }
      if (list.length) {
        result = result || {};
        result.ipList = result.idList = list.slice(0, 16);
      }
      if (result && (result.name || headers) && filter.mtype == 1) {
        result.exact = 1;
      }
      return result;
    }
    function checkClientIp(item, filter) {
      var clientIp = item.req.ip;
      if (filter.clientIp) {
        return clientIp === filter.clientIp;
      }
      if (filter.ip && clientIp === filter.ip) {
        return true;
      }
      // 有 clientId 过滤条件时，必须匹配 clientId
      var clientId = getClientId(item);
      var ipList = filter.ipList;
      if (filter.clientId) {
        if (clientId === filter.clientId) {
          return true;
        }
        if (!ipList) {
          return false;
        }
      } else if (!ipList) {
        return true;
      }
      var len = ipList.length;
      if (len < 3) {
        if (ipList[0] === clientIp || ipList[0] === clientId) {
          return true;
        }
        if (ipList[1] && (ipList[1] === clientIp || ipList[1] === clientId)) {
          return true;
        }
      } else {
        for (var i = 0; i < len; i++) {
          var ip = ipList[i];
          if (ip === clientIp || ip === clientId) {
            return true;
          }
        }
      }
      return false;
    }
  
    function checkHeader(text, keyword, exact) {
      if (!keyword) {
        return text != null;
      }
      if (!text || typeof text !== 'string') {
        if (!Array.isArray(text)) {
          return false;
        }
        text = text.join('\n');
      }
      text = text.toLowerCase();
      if (exact) {
        return (
          text === keyword ||
          text === util.encodeURIComponent(keyword).toLowerCase()
        );
      }
      return (
        text.indexOf(keyword) !== -1 ||
        text.indexOf(util.encodeURIComponent(keyword).toLowerCase()) !== -1
      );
    }
  
    function getClientId(item) {
      return (
        item._clientId ||
        item.req.headers[config.CLIENT_ID_HEADER] ||
        item.clientId
      );
    }
  
    function checkItem(item, filter) {
      if (!item || !checkClientIp(item, filter)) {
        return false;
      }
      var h = item.req.headers;
      if (filter.filterKey && h[filter.filterKey] != filter.filterValue) {
        return false;
      }
      if (filter.filterClientId && getClientId(item) != filter.filterClientId) {
        return false;
      }
      if (
        filter.name &&
        !checkHeader(h[filter.name], filter.value, filter.exact)
      ) {
        return false;
      }
      if (
        filter.url &&
        !checkHeader((item.isHttps ? 'tunnel://' : '') + item.url, filter.url)
      ) {
        return false;
      }
      var headers = filter.headers;
      if (headers) {
        for (var i = 0, len = headers.length; i < len; i++) {
          var header = headers[i];
          if (!checkHeader(h[header.name], header.value, filter.exact)) {
            return false;
          }
        }
      }
      return true;
    }
    proxy.getLastDataId = function () {
      return ids[ids.length - 1];
    };
  
    proxy.getCookiesByDomain = function(domain) {
      var result = [];
      if (!domain || typeof domain !== 'string') {
        return result;
      }
      for (var i = ids.length - 1; i >= 0; i--) {
        var item = reqData[ids[i]];
        if (getDomain(item) === domain) {
          var cookie = item.req.headers.cookie;
          if (cookie && result.indexOf(cookie) === -1) {
            result.push(cookie);
            if (result.length >= 30) {
              return result;
            }
          }
        }
      }
      return result;
    };
  
    proxy.getItem = function (id) {
      var item = reqData[id];
      if (item) {
        toBase64String(item.req);
        toBase64String(item.res);
      }
      return item;
    };
    proxy.abortRequest = function (id) {
      var item = id && reqData[id];
      item && item.abort && item.abort();
    };
    proxy.getFrames = function (options) {
      return getFrames(options.curReqId, options.lastFrameId);
    };
  
    function getNetworkTime(id) {
      var item = id && reqData[id];
      return item && {
        startTime: item.startTime,
        dnsTime: item.dnsTime,
        requestTime: item.requestTime,
        responseTime: item.responseTime,
        endTime: item.endTime
      };
    }
  
    proxy.getData = function (
      options,
      clientIp,
      key,
      value,
      filterClientId,
      clientId
    ) {
      options = options || {};
      var filter = formatFilter(options, clientIp, clientId);
      var data = {};
      var count = options.count;
      var startTime = options.startTime;
      var clearNetwork =
        !(options.dumpCount > 0) && (count == 0 || startTime == -2);
      if (!clearNetwork) {
        count = count > 0 && count < COUNT ? +count : COUNT;
        if (options.dumpCount > 0) {
          var len = ids.length;
          startTime = ids[len > options.dumpCount ? len - options.dumpCount : 0];
        }
      }
      if (key && value) {
        filter = filter || {};
        filter.filterKey = key;
        filter.filterValue = value;
      }
      if (filterClientId) {
        filter = filter || {};
        filter.filterClientId = filterClientId;
      }
      var newIds =
        clearNetwork || startTime == -1
          ? []
          : getIds(startTime, count, options.lastRowId);
      var setData = function (item) {
        if (item) {
          if (config.secureFilter) {
            try {
              item = config.secureFilter(item, clientIp, filter) || item;
            } catch (e) {
              if (config.debugMode) {
                /* eslint-disable no-console */
                console.log(e);
              }
              logger.error(e);
            }
          }
          data[item.id] = item;
        }
      };
      if (newIds.length > 0) {
        if (filter) {
          var id = newIds[0];
          var index = ids.indexOf(newIds[0]);
          newIds = [];
          while (id && count > 0) {
            var item = reqData[id];
            if (checkItem(item, filter)) {
              toBase64String(item.req);
              toBase64String(item.res);
              setData(item);
              newIds.push(id);
              --count;
            }
            id = ids[++index];
          }
        } else if (!filter) {
          getList(newIds).forEach(setData);
        }
      }
      getList(options.ids, options.status).forEach(setData);
      var endId = ids[ids.length - 1];
      var lastFrameId, frames;
      if (options.lastFrameId == -3) {
        var lastFrame = getLastFrame(options.curReqId);
        if (lastFrame) {
          if (lastFrame.closed || lastFrame.err) {
            frames = [lastFrame];
          } else {
            lastFrameId = lastFrame.frameId;
          }
        }
      } else {
        frames = getFrames(options.curReqId, options.lastFrameId);
      }
      var lastNewId = newIds[newIds.length - 1];
      var lastNewFrameId = frames && frames[frames.length - 1];
      var hasNew =
        (lastNewId && endId !== lastNewId) ||
        (lastNewFrameId && lastNewFrameId !== lastFrameId);
      var tunnelIds = options.tunnelIds;
      var tunnelIps = {};
      if (Array.isArray(tunnelIds) && tunnelIds.length > 0) {
        tunnelIds.forEach(function (id) {
          tunnelIps[id] = proxy.getTunnelIp(id);
        });
      }
  
      return {
        ids: options.ids || [],
        composerTime: getNetworkTime(options.composerReqId),
        tunnelIps: tunnelIps,
        newIds: newIds,
        data: data,
        hasNew: hasNew,
        lastId: clearNetwork ? endId : lastNewId,
        endId: endId,
        frames: frames,
        lastFrameId: lastFrameId,
        socketStatus: socketMgr.getStatus(options.curReqId)
      };
    };
  };
  