const urrl = require('url'); import { readFileSync } from 'fs'; import * as tls from 'tls'; import { QueryError, QueryResult, QueryService, cleanSensitiveData, redactHeaders, User, App, OAuthUnauthorizedClientError, getRefreshedToken, isEmpty, validateAndSetRequestOptionsBasedOnAuthType, sanitizeHeaders, sanitizeCookies, cookiesToString, sanitizeSearchParams, getAuthUrl, } from '@tooljet-plugins/common'; const FormData = require('form-data'); const JSON5 = require('json5'); import got, { HTTPError, OptionsOfTextResponseBody } from 'got'; import { SourceOptions } from './types'; function isFileObject(value) { const keys = Object.keys(value); return ( typeof value === 'object' && keys.length > 0 && keys.includes('name') && // example.zip keys.includes('type') && // application/zip keys.includes('content') && // raw'ish bytes (contains new lines - \n) keys.includes('dataURL') && // data url representation keys.includes('base64Data') && // data in base64 keys.includes('filePath') ); } interface RestAPIResult extends QueryResult { metadata?: Array | object; } export default class RestapiQueryService implements QueryService { async run( sourceOptions: any, queryOptions: any, dataSourceId: string, dataSourceUpdatedAt: string, context?: { user?: User; app?: App } ): Promise { const hasDataSource = dataSourceId !== undefined; const url = this.constructUrl(sourceOptions, queryOptions, hasDataSource); const _requestOptions = await this.constructValidatedRequestOptions( context, sourceOptions, queryOptions, hasDataSource, url ); if (_requestOptions.status === 'needs_oauth') return _requestOptions; const requestOptions = _requestOptions.data as OptionsOfTextResponseBody; try { const response = await got(url, requestOptions); const { result, requestObject, responseObject } = this.handleResponse(response); return { status: 'ok', data: result, metadata: { request: requestObject, response: responseObject, }, }; } catch (error) { throw this.handleError(error, sourceOptions); } } private async constructValidatedRequestOptions( context: { user?: User; app?: App }, sourceOptions: any, queryOptions: any, hasDataSource: boolean, url: string ) { const headers = sanitizeHeaders(sourceOptions, queryOptions, hasDataSource); const method = queryOptions['method']; const searchParams = this.buildSearchParams(sourceOptions, queryOptions, hasDataSource, url); const _requestOptions: OptionsOfTextResponseBody = { method, ...this.fetchHttpsCertsForCustomCA(sourceOptions), headers, searchParams, ...(queryOptions['retry_network_errors'] === true ? {} : { retry: 0 }), }; this.addCookiesToRequest(sourceOptions, queryOptions, hasDataSource, _requestOptions); const body = this.constructRequestBody(sourceOptions, queryOptions, hasDataSource); this.addBodyToRequest(_requestOptions, body); const authValidatedRequestOptions = await validateAndSetRequestOptionsBasedOnAuthType( sourceOptions, context, _requestOptions ); return authValidatedRequestOptions; } constructRequestBody( sourceOptions: any, queryOptions: any, hasDataSource: boolean ): undefined | string | Record { if (queryOptions.method === 'get') return undefined; if (queryOptions['body_toggle']) { // FIXME: Remove json_body usage with data migration // For backward compatibility, check if JSON body was previously used queryOptions['raw_body'] ||= queryOptions['json_body']; return queryOptions['raw_body']; } const _body = (queryOptions.body || []).filter((o) => { return o.some((e) => !isEmpty(e)); }); // Body params of the source will be overridden by body params of the query if (!hasDataSource) return Object.fromEntries(_body); const bodyParams = _body.concat(sourceOptions.body || []); return Object.fromEntries(bodyParams); } isJson(str: string) { try { JSON.parse(str); } catch (e) { return false; } return true; } private maybeParseJson(body: string): any { try { return JSON5.parse(body); } catch { return body; } } private constructUrl(sourceOptions: any, queryOptions: any, hasDataSource: boolean): string { return hasDataSource ? `${sourceOptions.url || ''}${queryOptions.url || ''}` : queryOptions.url; } private buildSearchParams( sourceOptions: any, queryOptions: any, hasDataSource: boolean, url: string ): URLSearchParams { const searchParams = new URLSearchParams(); // Add URL parameters from source options for (const param of sourceOptions.url_parameters || []) { const [key, value] = param; if (key && value) { searchParams.append(key, value); } } const paramsFromUrl = urrl.parse(url, true).query; // Append parameters to preserve duplicates for (const [key, value] of Object.entries(paramsFromUrl)) { if (Array.isArray(value)) { value.forEach((val) => searchParams.append(key, val)); } else { searchParams.append(key, String(value)); } } // Sanitize and append search parameters for (const [key, value] of sanitizeSearchParams(sourceOptions, queryOptions, hasDataSource)) { searchParams.append(key, String(value)); } return searchParams; } private addCookiesToRequest( sourceOptions: any, queryOptions: any, hasDataSource: boolean, requestOptions: OptionsOfTextResponseBody ) { const sanitizedCookies = sanitizeCookies(sourceOptions, queryOptions, hasDataSource); const cookieString = cookiesToString(sanitizedCookies); if (cookieString) { requestOptions.headers['Cookie'] = cookieString; } } private addBodyToRequest(requestOptions: OptionsOfTextResponseBody, body: any) { const headers = requestOptions.headers as Record; const contentType = this.getContentType(headers); switch (contentType) { case 'application/json': requestOptions.json = this.maybeParseJson(body); break; case 'application/x-www-form-urlencoded': this.setFormUrlencodedBody(requestOptions, body); break; case 'multipart/form-data': this.setMultipartFormDataBody(requestOptions, body); break; default: requestOptions.body = body; break; } } private getContentType(headers: Record): string { const contentTypeKey = Object.keys(headers).find((key) => key.toLowerCase() === 'content-type'); return contentTypeKey ? headers[contentTypeKey].toLowerCase() : 'application/json'; } private setFormUrlencodedBody(requestOptions: OptionsOfTextResponseBody, body: any) { typeof body === 'object' ? (requestOptions.form = body) : (requestOptions.body = body); } private setMultipartFormDataBody(requestOptions: OptionsOfTextResponseBody, body: any) { if (body && Object.values(body).some(isFileObject)) { const form = new FormData(); Object.entries(body).forEach(([key, value]: [string, Record]) => { if (isFileObject(value)) { const fileBuffer = Buffer.from(value.base64Data || '', 'base64'); form.append(key, fileBuffer, { filename: value?.name || '', contentType: value?.type || '', knownLength: fileBuffer.length, }); } else if (value != null) { form.append(key, value); } }); requestOptions.body = form; requestOptions.headers = { ...requestOptions.headers, ...form.getHeaders() }; } } private handleResponse(response: any) { const result = this.getResponse(response); const requestUrl = response?.request?.options?.url?.origin + response?.request?.options?.url?.pathname; const requestObject = { requestUrl, url: response.requestUrl, method: response.request.options.method, headers: redactHeaders(response.request.options.headers), params: urrl.parse(response.request.requestUrl, true).query, }; const responseObject = { statusCode: response.statusCode, headers: redactHeaders(response.headers), }; return { result, requestObject, responseObject }; } private handleError(error: any, sourceOptions: any): Error { console.error( `Error while calling REST API endpoint. Status code: ${error?.response?.statusCode}, Message: ${error?.response?.body}` ); let result = {}; let metadata = {}; if (error instanceof HTTPError) { const requestUrl = error?.request?.options?.url?.origin + error?.request?.options?.url?.pathname; const requestHeaders = cleanSensitiveData(error?.request?.options?.headers, ['authorization']); const requestObject = { requestUrl: requestUrl, requestHeaders: requestHeaders, requestParams: urrl.parse(error.request.requestUrl, true).query, }; const responseObject = { statusCode: error.response.statusCode, responseBody: error.response.body, headers: redactHeaders(error.response.headers), }; metadata = { request: requestObject, response: responseObject, }; // TODO: Need to remove the request/response related information in result in next MAJOR release. // This is now shared in `metadata` key. Keeping this here for backward compatibility. result = { requestObject: requestObject, responseObject: responseObject, responseHeaders: error.response.headers, }; } if (sourceOptions['auth_type'] === 'oauth2' && error?.response?.statusCode === 401) { throw new OAuthUnauthorizedClientError('Unauthorized status from API server', error.message, result); } throw new QueryError('Query could not be completed', error.message, result, metadata); } fetchHttpsCertsForCustomCA(sourceOptions: any) { let httpsParams: any = {}; switch (sourceOptions.ssl_certificate) { case 'ca_certificate': httpsParams = { https: { certificateAuthority: [sourceOptions.ca_cert], }, }; break; case 'client_certificate': httpsParams = { https: { certificateAuthority: [sourceOptions.ca_cert], key: [sourceOptions.client_key], certificate: [sourceOptions.client_cert], }, }; break; default: break; } if (process.env.NODE_EXTRA_CA_CERTS) { 'https' in httpsParams ? (httpsParams.https.certificateAuthority = httpsParams.https?.certificateAuthority.concat([ ...tls.rootCertificates, readFileSync(process.env.NODE_EXTRA_CA_CERTS), ])) : (httpsParams = { https: { certificateAuthority: [...tls.rootCertificates, readFileSync(process.env.NODE_EXTRA_CA_CERTS)].join('\n'), }, }); } return httpsParams; } private getResponse(response) { const contentType: string = response.headers?.['content-type'] ?? ''; try { if (this.isJson(response.body)) { return JSON.parse(response.body); } if (response.rawBody && this.isBinary(contentType)) { return Buffer.from(response.rawBody, 'binary').toString('base64'); } } catch (error) { console.error('Error while parsing response', error); } return response.body; } private isBinary(contentType: string) { const binaryPrefixes = ['application/', 'image/']; const binaryApplicationTypes = ['application/pdf', 'application/zip']; for (const binaryPrefix of binaryPrefixes) { if (contentType?.startsWith(binaryPrefix)) { if (binaryPrefix === 'application/') return binaryApplicationTypes.includes(contentType); return true; } } return false; } authUrl(sourceOptions: SourceOptions): string { return getAuthUrl(sourceOptions); } async refreshToken(sourceOptions: any, error: any, userId: string, isAppPublic: boolean) { return getRefreshedToken(sourceOptions, error, userId, isAppPublic); } }