From 8d9d25c2c1f3edc77850c228aab59fb014e73ab0 Mon Sep 17 00:00:00 2001 From: Zahrun <10415894+Zahrun@users.noreply.github.com> Date: Tue, 13 Dec 2022 12:22:18 +0100 Subject: [PATCH 1/5] feat: support liquid trades import --- src/parsers/trades/index.ts | 2 + src/parsers/trades/liquid/index.ts | 93 ++++++++++++++++++++++++++++++ src/types/locations.ts | 3 + 3 files changed, 98 insertions(+) create mode 100644 src/parsers/trades/liquid/index.ts diff --git a/src/parsers/trades/index.ts b/src/parsers/trades/index.ts index 30b720d..387b6ad 100644 --- a/src/parsers/trades/index.ts +++ b/src/parsers/trades/index.ts @@ -4,6 +4,7 @@ import binanceParser from './binance'; import bittrexParser from './bittrex'; import geminiParser from './gemini'; import krakenParser from './kraken'; +import liquidParser from './liquid'; import poloniexParser from './poloniex'; import revolutParser from './revolut'; @@ -12,6 +13,7 @@ const parserMapping: {[key in EXCHANGES]: any} = { [EXCHANGES.Bittrex]: bittrexParser, [EXCHANGES.Gemini]: geminiParser, [EXCHANGES.Kraken]: krakenParser, + [EXCHANGES.Liquid]: liquidParser, [EXCHANGES.Poloniex]: poloniexParser, [EXCHANGES.Revolut]: revolutParser, } diff --git a/src/parsers/trades/liquid/index.ts b/src/parsers/trades/liquid/index.ts new file mode 100644 index 0000000..f3c77a6 --- /dev/null +++ b/src/parsers/trades/liquid/index.ts @@ -0,0 +1,93 @@ +import { getCSVData } from '../../'; +import { EXCHANGES, IImport, IPartialTrade, ITrade } from '../../../types'; +import { createDateAsUTC, createID } from '../../utils'; + +enum LiquidOrderDirection { + PAY = 'PAY', + RECEIVE = 'RECEIVE', +} + +interface ILiquid { + exchange: string; + currency_type: string; + direction: string; + transaction_id: string; + transaction_type: string; + gross_amount: string; + currency: string; + execution_id: string; + generated_for_type: string; + generated_for_id: string; + transaction_hash: string; + from_address: string; + to_address: string; + state: string; + created_at_jst: string; + updated_at_jst: string; + created_at_utc: string; + updated_at: string; + notes: string; + used_id: string; + account_id: string; +} + +export default async function processData(importDetails: IImport): Promise { + const data: ILiquid[] = await getCSVData(importDetails.data) as ILiquid[]; + const internalFormat: ITrade[] = []; + if (data.length < 1) { + return internalFormat; + } + let splitTrade = data[0]; + let lineContinuity = 0; + for (const trade of data) { + const tradeToAdd: IPartialTrade = { + date : createDateAsUTC(new Date(trade.updated_at)).getTime(), + exchange : EXCHANGES.Liquid, + exchangeID : trade.execution_id, + }; + switch (trade.transaction_type) { + case 'rebate_trade_fee': + case 'trade_fee': + case 'quick_exchange': + case 'trade': { + switch (lineContinuity) { + case 0: { + splitTrade = trade; + lineContinuity = 1; + continue; + } + case 1: { + lineContinuity = 0; + if (trade.directiontoUpperCase() === LiquidOrderDirection.PAY && splitTrade.direction === LiquidOrderDirection.RECEIVE) { + tradeToAdd.boughtCurrency = splitTrade.currency; + tradeToAdd.soldCurrency = trade.currency; + tradeToAdd.amountSold = Math.abs(parseFloat(trade.gross_amount)); + tradeToAdd.rate = Math.abs(parseFloat(trade.gross_amount) / parseFloat(splitTrade.gross_amount)); + } else if (trade.direction === LiquidOrderDirection.RECEIVE && splitTrade.direction === LiquidOrderDirection.PAY) { + tradeToAdd.soldCurrency = splitTrade.currency; + tradeToAdd.boughtCurrency = trade.currency; + tradeToAdd.amountSold = Math.abs(parseFloat(splitTrade.gross_amount)); + tradeToAdd.rate = Math.abs(parseFloat(splitTrade.gross_amount) / parseFloat(trade.gross_amount)); + } else { + console.error(`Error parsing ${trade.exchange} trade splitTrade.direction=${splitTrade.direction} and trade.direction=${trade.direction}`); + break; + } + tradeToAdd.ID = createID(tradeToAdd); + internalFormat.push(tradeToAdd as ITrade); + continue; + } + default: { + console.error(`Error parsing ${trade.exchange} trade lineContinuity=${lineContinuity}`); + break; + } + } + break; + } + default: { + console.log(`Ignored ${trade.exchange} trade of type ${trade.transaction_type}`); + continue; + } + } + } + return internalFormat; +} diff --git a/src/types/locations.ts b/src/types/locations.ts index c0053d4..1783cf7 100644 --- a/src/types/locations.ts +++ b/src/types/locations.ts @@ -3,6 +3,8 @@ export type Location = EXCHANGES | string; export enum EXCHANGES { Bittrex = 'BITTREX', Gemini= 'GEMINI', + Kraken = 'KRAKEN', + Liquid = 'LIQUID', Poloniex = 'POLONIEX', Kraken = 'KRAKEN', Binance = 'BINANCE', @@ -16,6 +18,7 @@ export enum IncomeImportTypes { export enum ExchangesTradeHeaders { BITTREX = '07230399aaa8d1f15e88e38bd43a01c5ef1af6c1f9131668d346e196ff090d80', GEMINI = '996edee25db7f3d1dd16c83c164c6cff8c6d0f5d6b3aafe6d1700f2a830f6c9e', + LIQUID = '12c125c9080e41e087ff0955826bba94568d637214b166a296447e53ae469abe', POLONIEX = 'd7484d726e014edaa059c0137ac91183a7eaa9ee5d52713aa48bb4104b01afb0', KRAKEN = '85bf27e799cc0a30fe5b201cd6a4724e4a52feb433f41a1e8b046924e3bf8dc5', BINANCE = '4d0d5df894fe488872e513f6148dfa14ff29272e759b7fb3c86d264687a7cf99', From 4de086d20c24ab5e89003d70d6b78c284c86f54c Mon Sep 17 00:00:00 2001 From: Zahrun <10415894+Zahrun@users.noreply.github.com> Date: Wed, 14 Dec 2022 01:36:42 +0100 Subject: [PATCH 2/5] feat: support liquid trades import --- src/parsers/trades/liquid/index.ts | 162 ++++++++++++++++++++++------- src/types/locations.ts | 1 - 2 files changed, 122 insertions(+), 41 deletions(-) diff --git a/src/parsers/trades/liquid/index.ts b/src/parsers/trades/liquid/index.ts index f3c77a6..2df3eac 100644 --- a/src/parsers/trades/liquid/index.ts +++ b/src/parsers/trades/liquid/index.ts @@ -31,63 +31,145 @@ interface ILiquid { account_id: string; } +function groupByExecutionID(group: any, line: any) { + group[line.execution_id] = group[line.execution_id] ?? []; + group[line.execution_id].push(line); + return group; +} + export default async function processData(importDetails: IImport): Promise { const data: ILiquid[] = await getCSVData(importDetails.data) as ILiquid[]; const internalFormat: ITrade[] = []; - if (data.length < 1) { - return internalFormat; - } - let splitTrade = data[0]; - let lineContinuity = 0; - for (const trade of data) { + const sorted = data.reduce(groupByExecutionID, {}); + for (const execution of Object.keys(sorted)) { + const trades = sorted[execution]; const tradeToAdd: IPartialTrade = { - date : createDateAsUTC(new Date(trade.updated_at)).getTime(), + date : createDateAsUTC(new Date(trades[0].updated_at)).getTime(), exchange : EXCHANGES.Liquid, - exchangeID : trade.execution_id, + exchangeID : execution, }; - switch (trade.transaction_type) { - case 'rebate_trade_fee': - case 'trade_fee': - case 'quick_exchange': + if (execution === '') { + for (const line of trades){ + console.log(`Ignored ${tradeToAdd.exchange} trade of type ${line.transaction_type}`); + continue; + } + } + switch (trades[0].transaction_type) { case 'trade': { - switch (lineContinuity) { - case 0: { - splitTrade = trade; - lineContinuity = 1; - continue; - } + switch (trades.length) { case 1: { - lineContinuity = 0; - if (trade.directiontoUpperCase() === LiquidOrderDirection.PAY && splitTrade.direction === LiquidOrderDirection.RECEIVE) { - tradeToAdd.boughtCurrency = splitTrade.currency; - tradeToAdd.soldCurrency = trade.currency; - tradeToAdd.amountSold = Math.abs(parseFloat(trade.gross_amount)); - tradeToAdd.rate = Math.abs(parseFloat(trade.gross_amount) / parseFloat(splitTrade.gross_amount)); - } else if (trade.direction === LiquidOrderDirection.RECEIVE && splitTrade.direction === LiquidOrderDirection.PAY) { - tradeToAdd.soldCurrency = splitTrade.currency; - tradeToAdd.boughtCurrency = trade.currency; - tradeToAdd.amountSold = Math.abs(parseFloat(splitTrade.gross_amount)); - tradeToAdd.rate = Math.abs(parseFloat(splitTrade.gross_amount) / parseFloat(trade.gross_amount)); - } else { - console.error(`Error parsing ${trade.exchange} trade splitTrade.direction=${splitTrade.direction} and trade.direction=${trade.direction}`); - break; - } - tradeToAdd.ID = createID(tradeToAdd); - internalFormat.push(tradeToAdd as ITrade); - continue; + internalFormat.push(addSingleLineTrade(tradeToAdd, trades[0])); + break; } - default: { - console.error(`Error parsing ${trade.exchange} trade lineContinuity=${lineContinuity}`); + case 2: { + internalFormat.push(addTrade(tradeToAdd, trades[0], trades[1])); + break; + } + case 3: { + let feeTrade = tradeToAdd; + internalFormat.push(addTrade(tradeToAdd, trades[0], trades[1])); + internalFormat.push(addFeeTrade(feeTrade, trades[2])); + break; + } + case 4: { + let feeTrade = tradeToAdd; + internalFormat.push(addTrade(tradeToAdd, trades[0], trades[1])); + internalFormat.push(addFeeTrade(feeTrade, trades[2], trades[3])); + break; + } + case 6: { + internalFormat.push(addFeeTrade(tradeToAdd, trades[4], trades[5])); + break; + } + case 8: { + let secondTrade = tradeToAdd; + internalFormat.push(addFeeTrade(tradeToAdd, trades[4], trades[5])); + internalFormat.push(addFeeTrade(secondTrade, trades[6], trades[7])); break; } + default: { + console.warn(`Error parsing ${tradeToAdd.exchange} trade. It extends over ${trades.length} lines`); + console.info(trades); + } } break; } + case 'rebate_trade_fee': + case 'trade_fee': { + console.error(`Error parsing ${tradeToAdd.exchange} trade. First line should not be of type ${trades[0].transaction_type}`); + break; + } default: { - console.log(`Ignored ${trade.exchange} trade of type ${trade.transaction_type}`); - continue; + console.log(`Ignored ${tradeToAdd.exchange} trade of type ${trades[0].transaction_type}`); } } } return internalFormat; } + +function addSingleLineTrade( + tradeToAdd: IPartialTrade, + trade: ILiquid, +) : ITrade { + if (trade.direction.toUpperCase() === LiquidOrderDirection.PAY) { + tradeToAdd.boughtCurrency = 'USDT'; + tradeToAdd.soldCurrency = trade.currency; + tradeToAdd.amountSold = parseFloat(trade.gross_amount); + tradeToAdd.rate = parseFloat(trade.gross_amount) / 0; + } else if (trade.direction.toUpperCase() === LiquidOrderDirection.RECEIVE) { + tradeToAdd.soldCurrency = 'USDT'; + tradeToAdd.boughtCurrency = trade.currency; + tradeToAdd.amountSold = 0; + tradeToAdd.rate = 0 / parseFloat(trade.gross_amount); + // This case here above does not work. We cannot, with current structure, create a trade with nothing sold but something bought. It should be an income (airdrop). Should we process the incomes here with the trades or import the same file again as income data file? + } + else { + console.info(trade); + throw new Error(`Error parsing ${tradeToAdd.exchange} trade.direction=${trade.direction}`); + } + tradeToAdd.ID = createID(tradeToAdd); + return tradeToAdd as ITrade; +} + +function addFeeTrade( + tradeToAdd: IPartialTrade, + feeTrade: ILiquid, + rebateFeeTrade?: ILiquid, +) : ITrade { + let amount = parseFloat(feeTrade.gross_amount); + if (rebateFeeTrade !== undefined) { + amount -= parseFloat(rebateFeeTrade.gross_amount); + } + tradeToAdd.boughtCurrency = 'USDT'; + tradeToAdd.soldCurrency = feeTrade.currency; + tradeToAdd.amountSold = amount; + tradeToAdd.rate = amount / 0; + tradeToAdd.ID = createID(tradeToAdd); + return tradeToAdd as ITrade; +} + +function addTrade( + tradeToAdd: IPartialTrade, + firstHalf: ILiquid, + secondHalf: ILiquid, +): ITrade { + let firstHalfDirection = firstHalf.direction.toUpperCase(); + let secondHalfDirection = secondHalf.direction.toUpperCase(); + if (firstHalfDirection === LiquidOrderDirection.PAY && secondHalfDirection === LiquidOrderDirection.RECEIVE) { + tradeToAdd.boughtCurrency = secondHalf.currency; + tradeToAdd.soldCurrency = firstHalf.currency; + tradeToAdd.amountSold = Math.abs(parseFloat(firstHalf.gross_amount)); + tradeToAdd.rate = Math.abs(parseFloat(firstHalf.gross_amount) / parseFloat(secondHalf.gross_amount)); + } else if (firstHalfDirection === LiquidOrderDirection.RECEIVE && secondHalfDirection === LiquidOrderDirection.PAY) { + tradeToAdd.soldCurrency = secondHalf.currency; + tradeToAdd.boughtCurrency = firstHalf.currency; + tradeToAdd.amountSold = Math.abs(parseFloat(secondHalf.gross_amount)); + tradeToAdd.rate = Math.abs(parseFloat(secondHalf.gross_amount) / parseFloat(firstHalf.gross_amount)); + } else { + console.info(firstHalf); + console.info(secondHalf); + throw new Error(`Error parsing ${tradeToAdd.exchange} firstHalf.direction=${firstHalf.direction} and secondHalf.direction=${secondHalf.direction}`); + } + tradeToAdd.ID = createID(tradeToAdd); + return tradeToAdd as ITrade; +} diff --git a/src/types/locations.ts b/src/types/locations.ts index 1783cf7..047d9ef 100644 --- a/src/types/locations.ts +++ b/src/types/locations.ts @@ -3,7 +3,6 @@ export type Location = EXCHANGES | string; export enum EXCHANGES { Bittrex = 'BITTREX', Gemini= 'GEMINI', - Kraken = 'KRAKEN', Liquid = 'LIQUID', Poloniex = 'POLONIEX', Kraken = 'KRAKEN', From 27e645900ae0c45311a147998e9f28b323d13b17 Mon Sep 17 00:00:00 2001 From: Zahrun <10415894+Zahrun@users.noreply.github.com> Date: Wed, 14 Dec 2022 16:13:03 +0100 Subject: [PATCH 3/5] fix: define types --- src/parsers/trades/liquid/index.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/parsers/trades/liquid/index.ts b/src/parsers/trades/liquid/index.ts index 2df3eac..da45381 100644 --- a/src/parsers/trades/liquid/index.ts +++ b/src/parsers/trades/liquid/index.ts @@ -31,7 +31,11 @@ interface ILiquid { account_id: string; } -function groupByExecutionID(group: any, line: any) { +interface ILiquidGroup { + [key: string]: ILiquid[]; +} + +function groupByExecutionID(group: ILiquidGroup, line: ILiquid) { group[line.execution_id] = group[line.execution_id] ?? []; group[line.execution_id].push(line); return group; From 412cc91c316fc68cf94ba4f83e3581c29f5819e2 Mon Sep 17 00:00:00 2001 From: Zahrun <10415894+Zahrun@users.noreply.github.com> Date: Thu, 15 Dec 2022 12:09:26 +0100 Subject: [PATCH 4/5] fix: better for loop --- src/parsers/trades/liquid/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parsers/trades/liquid/index.ts b/src/parsers/trades/liquid/index.ts index da45381..7419d38 100644 --- a/src/parsers/trades/liquid/index.ts +++ b/src/parsers/trades/liquid/index.ts @@ -45,7 +45,7 @@ export default async function processData(importDetails: IImport): Promise Date: Thu, 15 Dec 2022 15:47:42 +0100 Subject: [PATCH 5/5] feat: better support liquid trades import --- components/TradesTable/index.tsx | 2 + pages/import.tsx | 15 +++- src/parsers/index.ts | 6 +- src/parsers/trades/index.ts | 7 +- src/parsers/trades/liquid/index.ts | 127 +++++++++++++++++++---------- src/types/import.ts | 1 + src/types/locations.ts | 3 +- src/types/trade.ts | 2 + 8 files changed, 113 insertions(+), 50 deletions(-) diff --git a/components/TradesTable/index.tsx b/components/TradesTable/index.tsx index c92de88..d244c28 100644 --- a/components/TradesTable/index.tsx +++ b/components/TradesTable/index.tsx @@ -55,6 +55,7 @@ export class TradesTable extends React.Component [ @@ -66,6 +67,7 @@ export class TradesTable extends React.Component{trade.boughtCurrency}, {(trade.amountSold / trade.rate).toFixed(8)}, {`${trade.transactionFee} ${trade.transactionFeeCurrency}`}, + {`${trade.tradeFee} ${trade.tradeFeeCurrency}`}, , ])} /> diff --git a/pages/import.tsx b/pages/import.tsx index aa6c494..27e4ada 100644 --- a/pages/import.tsx +++ b/pages/import.tsx @@ -125,6 +125,7 @@ const Import = () => { const [showNewIncome, setShowNewIncome] = useState(false); const [processing, setProcessing] = useState(false); const [fileBrowseOpen, setFileBrowseOpen] = useState(false); + const [secondFileData, setSecondFileData] = useState(''); const [processedData, setProcessedData] = useState([]); const [duplicateData, setDuplicateData] = useState([]); const [alertData, setAlertData] = useState(undefined); @@ -179,6 +180,8 @@ const Import = () => { setDuplicateData, setAlertData, setProcessedData, + secondFileData, + setSecondFileData, )} browse={fileBrowseOpen} /> @@ -305,6 +308,8 @@ const readFile = ( setDuplicateData: (duplicateData: DuplicateDataTypes) => void, setAlertData: (alertData: IAlertData) => void, setProcessedData: (processedData: ProcessedDataTypes) => void, + secondFileData: string, + setSecondFileData: (secondFileData: string) => void, ) => async ( fileData: string, input: React.RefObject @@ -318,8 +323,16 @@ const readFile = ( const processedData = await processData({ ...importDetails, data: fileData, + data2: secondFileData, }); - if (processedData && processedData.length) { + if (typeof processedData === 'string') { + setSecondFileData(fileData); + alert(`To import ${processedData} completly, \ +please provide both the crypto transactions and the fiat transactions files.\n +Please now select the other one.`); + setFileBrowseOpen(true); + } else if (processedData && processedData.length) { + setSecondFileData(''); const duplicateData = duplicateCheck( importDetails, savedData, processedData, ); diff --git a/src/parsers/index.ts b/src/parsers/index.ts index 6fdeec3..d36f986 100644 --- a/src/parsers/index.ts +++ b/src/parsers/index.ts @@ -36,7 +36,7 @@ export async function getCSVData(fileData: string): Promise { }); } -export async function processData(importDetails: IImport): Promise { +export async function processData(importDetails: IImport): Promise { switch (importDetails.type) { case ImportType.TRADES: return await processTradesImport(importDetails); @@ -48,7 +48,3 @@ export async function processData(importDetails: IImport): Promise { +export default async function processTradesImport(importDetails: IImport): Promise { if (importDetails.location in parserMapping) { const parser = parserMapping[importDetails.location]; return await parser(importDetails); @@ -26,7 +26,10 @@ export default async function processTradesImport(importDetails: IImport): Promi const headers = importDetails.data.substr(0, importDetails.data.indexOf('\n')); const headersHash = crypto.createHash('sha256').update(headers).digest('hex'); for (const key in ExchangesTradeHeaders) { - if (ExchangesTradeHeaders[key] === headersHash) { + if (ExchangesTradeHeaders[key].split(';').includes(headersHash)) { + if (key === EXCHANGES.Liquid && importDetails.data2 === '') { + return 'Liquid trades'; + } return processTradesImport({ ...importDetails, location: key, diff --git a/src/parsers/trades/liquid/index.ts b/src/parsers/trades/liquid/index.ts index 7419d38..4a17c6e 100644 --- a/src/parsers/trades/liquid/index.ts +++ b/src/parsers/trades/liquid/index.ts @@ -41,22 +41,61 @@ function groupByExecutionID(group: ILiquidGroup, line: ILiquid) { return group; } +function groupByCreatedAtUTC(group: ILiquidGroup, line: ILiquid) { + group[line.created_at_utc] = group[line.created_at_utc] ?? []; + group[line.created_at_utc].push(line); + return group; +} + export default async function processData(importDetails: IImport): Promise { - const data: ILiquid[] = await getCSVData(importDetails.data) as ILiquid[]; + let data2: ILiquid[] = []; + if (importDetails.data2 !== undefined ) { + data2 = await getCSVData(importDetails.data2) as ILiquid[]; + } + const data: ILiquid[] = (await getCSVData(importDetails.data) as ILiquid[]).concat(data2); const internalFormat: ITrade[] = []; - const sorted = data.reduce(groupByExecutionID, {}); - for (const execution in sorted) { - const trades = sorted[execution]; + const sorted = data.sort(function(a, b){ + const dateA = createDateAsUTC(new Date(a.created_at_utc)).getTime(); + const dateB = createDateAsUTC(new Date(b.created_at_utc)).getTime(); + return dateA - dateB; + }); + const grouped = sorted.reduce(groupByExecutionID, {}); + for (const execution in grouped) { + const trades = grouped[execution]; const tradeToAdd: IPartialTrade = { - date : createDateAsUTC(new Date(trades[0].updated_at)).getTime(), + date : createDateAsUTC(new Date(trades[0].created_at_utc)).getTime(), exchange : EXCHANGES.Liquid, exchangeID : execution, }; if (execution === '') { - for (const line of trades){ - console.log(`Ignored ${tradeToAdd.exchange} trade of type ${line.transaction_type}`); - continue; + const dateGrouped = trades.reduce(groupByCreatedAtUTC, {}); + for (const date in dateGrouped) { + const groupedTrades = dateGrouped[date]; + const firstLine = groupedTrades[0]; + switch (firstLine.transaction_type) { + /*case 'funding': { + // TODO: create a deposit transaction + break; + } + case 'withdrawal': { + // TODO: create a withdrawal transaction + break; + }*/ + case 'quick_exchange': { + if (groupedTrades.length == 2) { + internalFormat.push(addTrade(tradeToAdd, groupedTrades[0], groupedTrades[1])); + } else { + console.error(`Error parsing ${tradeToAdd.exchange} quick exchange. + It extends over ${groupedTrades.length} lines`); + } + break; + } + default: { + console.log(`Ignored ${tradeToAdd.exchange} trade of type ${firstLine.transaction_type}`); + } + } } + continue; } switch (trades[0].transaction_type) { case 'trade': { @@ -70,29 +109,38 @@ export default async function processData(importDetails: IImport): Promise