import http, { RequestOptions } from 'http'; import https from 'https'; import fs from 'fs-extra'; import { createGunzip, createInflate } from 'zlib'; import { URL } from 'url'; import { Readable } from 'stream'; /** * Create an HTTP GET request. * @param link Request URL * @param headers Request headers * @param restrictToText Only allow textual responses * @param saveTo Save to a file * @returns Response data */ export function httpGET( link: string, headers: any = {}, restrictToText = true, saveTo?: string, lback?: number ): Promise { let parsed: URL; try { parsed = new URL(link); } catch (e: any) { return Promise.reject(e); } const opts = { headers: { Accept: '*/*', 'Accept-Language': 'en-US', 'User-Agent': 'Squeebot/Commons-3.0.0 squeebot.visit@lunasqu.ee', }, host: parsed.hostname, path: `${parsed.pathname}${parsed.search}`, port: parsed.port || null, }; if (headers) { opts.headers = Object.assign(opts.headers, headers); } let reqTimeOut: any; let data: string | null = ''; const httpModule = parsed.protocol === 'https:' ? https : http; return new Promise((resolve, reject) => { if (lback && lback >= 5) { return reject(new Error('infinite loop!')); } const req = httpModule.get(opts, (res) => { if (res.statusCode === 302 || res.statusCode === 301) { if (!lback) { lback = 1; } else { lback += 1; } return httpGET(res.headers.location as string, headers, restrictToText, saveTo, lback, ).then(resolve, reject); } if (saveTo) { const file = fs.createWriteStream(saveTo); res.pipe(file); file.on('close', () => resolve(saveTo)); file.on('error', (e) => reject(e)); return; } if (restrictToText) { const cType = res.headers['content-type']; if (cType && cType.indexOf('text') === -1 && cType.indexOf('json') === -1) { req.destroy(); return reject(new Error('Response type is not supported by httpGET!')); } } reqTimeOut = setTimeout(() => { req.destroy(); data = null; reject(new Error('Request took too long!')); }, 5000); let output: Readable = res; // Decompress GZip if (res.headers['content-encoding'] === 'gzip') { const gzip = createGunzip(); res.pipe(gzip); output = gzip; } // Deflate if (res.headers['content-encoding'] === 'deflate') { const inflate = createInflate(); res.pipe(inflate); output = inflate; } output.on('data', (chunk) => { data += chunk; }); output.on('end', () => { clearTimeout(reqTimeOut); resolve(data || saveTo); }); }).on('error', (e) => { reject(e); }); req.setTimeout(10000); }); } /** * Create an HTTP POST request. * * Note: Content-Type defaults to `application/x-www-form-urlencoded`. * Set the `Content-Type` to `application/json` to post and parse JSON. * @param link Request URL * @param headers Request headers * @param data Submit data * @returns Response data */ export function httpPOST( link: string, headers: any = {}, data: any ): Promise { let parsed: URL; let postData: string | URLSearchParams; try { parsed = new URL(link); postData = new URLSearchParams(data); } catch(e: any) { return Promise.reject(e); } const opts: RequestOptions = { headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'Squeebot/Commons-3.0.0 squeebot.visit@lunasqu.ee', }, host: parsed.host, method: 'POST', path: `${parsed.pathname}${parsed.search}`, port: parsed.port || null, }; // Assign provided headers if (headers) { opts.headers = Object.assign({}, opts.headers, headers); } // Ensure headers list exists if (!opts.headers) { opts.headers = {}; } // If content type is JSON, add it to body if (opts.headers['Content-Type'] === 'application/json') { postData = JSON.stringify(data); } // Set content length accordingly opts.headers['Content-Length'] = Buffer.byteLength(postData.toString()); return new Promise((resolve, reject) => { const httpModule = parsed.protocol === 'https:' ? https : http; const req = httpModule.request(opts, (res) => { res.setEncoding('utf8'); let resp = ''; res.on('data', (chunk) => { resp += chunk; }); res.on('end', () => { resolve(resp); }); }).on('error', (e) => { reject(e); }); req.write(postData); req.end(); }); }