core/src/common/http.ts

import http, { RequestOptions } from 'http';
import https from 'https';
import fs from 'fs-extra';
import { createGunzip, createInflate } from 'zlib';
import { URL } from 'url';
import { Readable } from 'stream';

/**
 * Create an HTTP GET request.
 * @param link Request URL
 * @param headers Request headers
 * @param restrictToText Only allow textual responses
 * @param saveTo Save to a file
 * @returns Response data
 */
export function httpGET(
  link: string,
  headers: any = {},
  restrictToText = true,
  saveTo?: string,
  lback?: number
): Promise<any> {
  let parsed: URL;
  try {
    parsed = new URL(link);
  } catch (e: any) {
    return Promise.reject(e);
  }

  const opts = {
    headers: {
      Accept: '*/*',
      'Accept-Language': 'en-US',
      'User-Agent': 'Squeebot/Commons-3.0.0 squeebot.visit@lunasqu.ee',
    },
    host: parsed.hostname,
    path: `${parsed.pathname}${parsed.search}`,
    port: parsed.port || null,
  };

  if (headers) {
    opts.headers = Object.assign(opts.headers, headers);
  }

  let reqTimeOut: any;
  let data: string | null = '';

  const httpModule = parsed.protocol === 'https:' ? https : http;
  return new Promise((resolve, reject) => {
    if (lback && lback >= 5) {
      return reject(new Error('infinite loop!'));
    }

    const req = httpModule.get(opts, (res) => {
      if (res.statusCode === 302 || res.statusCode === 301) {
        if (!lback) {
          lback = 1;
        } else {
          lback += 1;
        }

        return httpGET(res.headers.location as string,
          headers,
          restrictToText,
          saveTo,
          lback,
        ).then(resolve, reject);
      }

      if (saveTo) {
        const file = fs.createWriteStream(saveTo);
        res.pipe(file);
        file.on('close', () => resolve(saveTo));
        file.on('error', (e) => reject(e));
        return;
      }

      if (restrictToText) {
        const cType = res.headers['content-type'];
        if (cType && cType.indexOf('text') === -1 && cType.indexOf('json') === -1) {
          req.destroy();
          return reject(new Error('Response type is not supported by httpGET!'));
        }
      }

      reqTimeOut = setTimeout(() => {
        req.destroy();
        data = null;
        reject(new Error('Request took too long!'));
      }, 5000);

      let output: Readable = res;

      // Decompress GZip
      if (res.headers['content-encoding'] === 'gzip') {
        const gzip = createGunzip();
        res.pipe(gzip);
        output = gzip;
      }

      // Deflate
      if (res.headers['content-encoding'] === 'deflate') {
        const inflate = createInflate();
        res.pipe(inflate);
        output = inflate;
      }

      output.on('data', (chunk) => {
        data += chunk;
      });

      output.on('end', () => {
        clearTimeout(reqTimeOut);

        resolve(data || saveTo);
      });
    }).on('error', (e) => {
      reject(e);
    });

    req.setTimeout(10000);
  });
}

/**
 * Create an HTTP POST request.
 *
 * Note: Content-Type defaults to `application/x-www-form-urlencoded`.
 * Set the `Content-Type` to `application/json` to post and parse JSON.
 * @param link Request URL
 * @param headers Request headers
 * @param data Submit data
 * @returns Response data
 */
export function httpPOST(
  link: string,
  headers: any = {},
  data: any
): Promise<any> {
  let parsed: URL;
  let postData: string | URLSearchParams;

  try {
    parsed = new URL(link);
    postData = new URLSearchParams(data);
  } catch(e: any) {
    return Promise.reject(e);
  }

  const opts: RequestOptions = {
    headers: {
      'Content-Type': 'application/x-www-form-urlencoded',
      'User-Agent': 'Squeebot/Commons-3.0.0 squeebot.visit@lunasqu.ee',
    },
    host: parsed.host,
    method: 'POST',
    path: `${parsed.pathname}${parsed.search}`,
    port: parsed.port || null,
  };

  // Assign provided headers
  if (headers) {
    opts.headers = Object.assign({}, opts.headers, headers);
  }

  // Ensure headers list exists
  if (!opts.headers) {
    opts.headers = {};
  }

  // If content type is JSON, add it to body
  if (opts.headers['Content-Type'] === 'application/json') {
    postData = JSON.stringify(data);
  }

  // Set content length accordingly
  opts.headers['Content-Length'] = Buffer.byteLength(postData.toString());

  return new Promise((resolve, reject) => {
    const httpModule = parsed.protocol === 'https:' ? https : http;
    const req = httpModule.request(opts, (res) => {
      res.setEncoding('utf8');
      let resp = '';

      res.on('data', (chunk) => {
        resp += chunk;
      });

      res.on('end', () => {
        resolve(resp);
      });
    }).on('error', (e) => {
      reject(e);
    });

    req.write(postData);
    req.end();
  });
}