From 6519136ad8a77778da742c12f026c3e41fd50cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=B6zkan=20gezmi=C5=9F?= Date: Sat, 22 Nov 2025 15:55:59 +0300 Subject: [PATCH 1/3] Replace fromtimestamp with epoch-offset to support negative Unix timestamps on Windows and restore full historical XAUUSD data. --- README.md | 14 ++++++++++++++ tvDatafeed/main.py | 20 +++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 66c5d8f..1f15a09 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ # **NOTE** +> **This is an improved fork of TvDatafeed.** +> +> - Windows compatibility: Dates before 1970 now work correctly when fetching historical data. +> - Nologin method: Can fetch more than 5000 bars for some symbols (typically daily timeframe, e.g., XAUUSD:OANDA). +> - See below for more details and usage notes. + +--- + +# **NOTE** + This is a fork of the original [TvDatafeed](https://github.com/rongardF/tvdatafeed.git) project by StreamAlpha. This fork has live data retrieving feature implemented. More information about this will be found in the TvDatafeedLive section down below in the README. @@ -57,6 +67,10 @@ tv = TvDatafeed() when using without login, following warning will be shown `you are using nologin method, data you access may be limited` +**Note:** The nologin method now works for more than 5000 bars for some symbols (e.g., `XAUUSD:OANDA`). Try your symbol to check availability. + +This extended limit is typically available for the daily timeframe. Other intervals may still be restricted. + --- ## Getting Data diff --git a/tvDatafeed/main.py b/tvDatafeed/main.py index de77ebc..6e669f8 100644 --- a/tvDatafeed/main.py +++ b/tvDatafeed/main.py @@ -133,14 +133,28 @@ def __send_message(self, func, args): @staticmethod def __create_df(raw_data, symbol): try: - out = re.search('"s":\[(.+?)\}\]', raw_data).group(1) + out = re.search(r'"s":\[(.+?)\}\]', raw_data).group(1) x = out.split(',{"') data = list() volume_data = True + epoch = datetime.datetime(1970, 1, 1) + for xi in x: - xi = re.split("\[|:|,|\]", xi) - ts = datetime.datetime.fromtimestamp(float(xi[4])) + xi = re.split(r"\[|:|,|\]", xi) + + try: + ts_raw = float(xi[4]) + except ValueError: + # malformed row, skip + continue + + try: + # works for negative timestamps too (pre-1970) + ts = epoch + datetime.timedelta(0, ts_raw) + except OverflowError: + # insane timestamp, skip + continue row = [ts] From 12e3fe9ca79ecd2e247842d779c53a1345ca479e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=B6zkan=20gezmi=C5=9F?= Date: Sat, 27 Dec 2025 14:49:50 +0300 Subject: [PATCH 2/3] search symbol function is fixed and find exchange function is added --- test_exchange.py | 14 ++++++++++++++ test_search.py | 13 +++++++++++++ tvDatafeed/main.py | 27 +++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 test_exchange.py create mode 100644 test_search.py diff --git a/test_exchange.py b/test_exchange.py new file mode 100644 index 0000000..024aa6e --- /dev/null +++ b/test_exchange.py @@ -0,0 +1,14 @@ +import sys +import os +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from tvDatafeed import TvDatafeed + +tv = TvDatafeed() + +print("Testing find_exchange:") +symbols = ['AAPL', 'MSFT', 'BTCUSDT', 'UNKNOWNKEYWORD123', "THYAO"] + +for s in symbols: + ex = tv.find_exchange(s) + print(f"Symbol: {s}, Exchange: {ex}") diff --git a/test_search.py b/test_search.py new file mode 100644 index 0000000..9423f7f --- /dev/null +++ b/test_search.py @@ -0,0 +1,13 @@ +import sys +import os +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from tvDatafeed import TvDatafeed +tv = TvDatafeed() + +# Check valid ticker +matches = tv.search_symbol(symbol='AAPL') +if matches: + print("Found:", matches[0]['symbol']) +else: + print("Ticker not found") \ No newline at end of file diff --git a/tvDatafeed/main.py b/tvDatafeed/main.py index 6e669f8..d6656be 100644 --- a/tvDatafeed/main.py +++ b/tvDatafeed/main.py @@ -303,20 +303,39 @@ def get_hist( return self.__create_df(raw_data, symbol) - def search_symbol(self, text: str, exchange: str = ''): - url = self.__search_url.format(text, exchange) + def search_symbol(self, symbol: str, exchange: str = ''): + url = self.__search_url.format(symbol, exchange) symbols_list = [] try: - resp = requests.get(url) + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Referer': 'https://www.tradingview.com/', + 'Origin': 'https://www.tradingview.com' + } + resp = requests.get(url, headers=headers) + + if resp.status_code != 200: + logger.error(f"Search symbol failed with status: {resp.status_code}, content: {resp.text[:100]}") symbols_list = json.loads(resp.text.replace( '', '').replace('', '')) except Exception as e: - logger.error(e) + logger.error(f"Exception in search_symbol: {e}") return symbols_list + def find_exchange(self, symbol: str): + """ + Finds the exchange for a given symbol. + Returns the exchange of the first match found, or None if no match is found. + """ + matches = self.search_symbol(symbol) + if matches: + return matches[0]['exchange'] + return None + + if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) From 95d9432e7ea180935a2001754fcf9367a8fbd69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96zkan=20Gezmi=C5=9F?= Date: Thu, 5 Feb 2026 16:33:31 +0300 Subject: [PATCH 3/3] feat: add get_quotes batch realtime API and harden ws lifecycle - Add TvDatafeed.get_quotes(symbols, timeout) using TradingView Quote Session (qsd parsing) - Reuse one WebSocket connection for many symbols; handle timeouts and invalid symbols - Ensure websocket cleanup (try/finally) and close connections after get_hist - Add benchmark_realtime.py and document get_quotes + benchmark in README --- .gitignore | 4 +- README.md | 118 +++--- benchmark_realtime.py | 73 ++++ tvDatafeed/main.py | 840 ++++++++++++++++++++++++------------------ 4 files changed, 632 insertions(+), 403 deletions(-) create mode 100644 benchmark_realtime.py diff --git a/.gitignore b/.gitignore index d269ca0..01e7bcb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,6 @@ ENV/ env.bak/ venv.bak/ tvDatafeed/symbols.pkl -tvDatafeed/beta.py \ No newline at end of file +tvDatafeed/beta.py +.opencode +implementation_plan.md diff --git a/README.md b/README.md index 1f15a09..aa0aeec 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,24 @@ -# **NOTE** +# TvDatafeed -> **This is an improved fork of TvDatafeed.** -> -> - Windows compatibility: Dates before 1970 now work correctly when fetching historical data. -> - Nologin method: Can fetch more than 5000 bars for some symbols (typically daily timeframe, e.g., XAUUSD:OANDA). -> - See below for more details and usage notes. +This is an improved fork of the original [TvDatafeed](https://github.com/rongardF/tvdatafeed.git) project. ---- - -# **NOTE** - -This is a fork of the original [TvDatafeed](https://github.com/rongardF/tvdatafeed.git) project by StreamAlpha. This fork has live data retrieving feature implemented. -More information about this will be found in the TvDatafeedLive section down below in the README. - -# **TvDatafeed** +Key changes in this fork: +- Windows compatibility: dates before 1970 work correctly when fetching historical data. +- Improved `nologin`: can fetch more than 5000 bars for some symbols (typically daily timeframe, e.g. `XAUUSD:OANDA`). +- Batch real-time quotes: `TvDatafeed.get_quotes()` fetches many symbols over a single Quote Session WebSocket. +- Bar-based live feed: `TvDatafeedLive` can monitor symbols and invoke callbacks when a new bar is produced. A simple TradingView historical Data Downloader. Tvdatafeed allows downloading upto 5000 bars on any of the supported timeframe. -If you found the content useful and want to support my work, you can buy me a coffee! -[![](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/StreamAlpha) - ## Installation -This module can be installed from github repo +This module can be installed from a Git repo. ```sh -pip install --upgrade --no-cache-dir git+https://github.com/rongardF/tvdatafeed.git +pip install --upgrade --no-cache-dir git+https://github.com/gezmisozkan/tvdatafeed.git ``` -For usage instructions, watch these videos- - -v1.2 tutorial with installation and backtrader usage - -[![Watch the video](https://img.youtube.com/vi/f76dOZW2gwI/hqdefault.jpg)](https://youtu.be/f76dOZW2gwI) - -Full tutorial - -[![Watch the video](https://img.youtube.com/vi/qDrXmb2ZRjo/hqdefault.jpg)](https://youtu.be/qDrXmb2ZRjo) - ---- - -## About release 2.0.0 - -Version 2.0.0 is a major release and is not backward compatible. make sure you update your code accordingly. Thanks to [stefanomorni](https://github.com/stefanomorni) for contributing and removing selenium dependancy. +If you are using a fork, replace the URL with your fork URL. ## Usage @@ -101,6 +77,44 @@ extended_price_data = tv.get_hist(symbol="EICHERMOT",exchange="NSE",interval=Int --- +## Real-time Quotes + +For high-throughput real-time price checks across many assets, prefer `get_quotes()` over calling `get_hist()` in a loop. + +`get_quotes()` uses the TradingView Quote Session protocol and a single WebSocket connection for the full batch. + +```python +quotes = tv.get_quotes([ + 'BINANCE:BTCUSDT', + 'NASDAQ:AAPL', + 'FX:USDTRY', +]) + +print(quotes['NASDAQ:AAPL']) +``` + +Return format: + +```python +{ + 'NASDAQ:AAPL': { + 'lp': 276.49, + 'lp_time': 1770253196, + 'ch': 7.01, + 'chp': 2.6, + 'status': 'ok' + }, + 'INVALID:SYMBOL': { + 'status': 'error', + 'error': '...' + } +} +``` + +Fields requested: `lp`, `lp_time`, `ch`, `chp`, `status`. + +--- + ## Search Symbol To find the exact symbols for an instrument you can use `tv.search_symbol` method. @@ -125,6 +139,8 @@ Indicators data is not downloaded from tradingview. For that you can use [TA-Lib ### Description +Note: for simple real-time price retrieval (many symbols), use `TvDatafeed.get_quotes()`. + **TvDatafeedLive** is a sub-class of **TvDatafeed** to extend the functionality and provide live data feed feature. The live data feed feature means that the user can specify symbol, exchange and interval set (also called as seis) for which they want the new data bar to be retrieved from TradingView whenever it is produced. The user can then provide any number of callback functions for that seis which will be called with the newly retrieved data bar. Callback functions and retrieving data from TradingView is @@ -239,6 +255,22 @@ data=seis.get_hist(n_bars=10, timeout=-1) --- +## Performance & Benchmarks + +`benchmark_realtime.py` compares: +- Method A: loop `get_hist(..., n_bars=1)` for spot prices (slow; new WS per symbol) +- Method B: single `get_quotes([...])` call (fast; single WS) + +Run: + +```sh +python benchmark_realtime.py +``` + +Typical results show a large speedup (often 10x+), depending on network conditions and symbol mix. + +--- + ## Supported Time Intervals Following timeframes intervals are supported- @@ -273,18 +305,6 @@ Following timeframes intervals are supported- ## Read this before creating an issue -Before creating an issue in this library, please follow the following steps. - -1. Search the problem you are facing is already asked by someone else. There might be some issues already there, either solved/unsolved related to your problem. Go to [issues](https://github.com/StreamAlpha/tvdatafeed/issues) page, use `is:issue` as filter and search your problem. ![image](https://user-images.githubusercontent.com/59556194/128167319-2654cfa1-f718-4a52-82f8-b0c0d26bf4ef.png) -2. If you feel your problem is not asked by anyone or no issues are related to your problem, then create a new issue. -3. Describe your problem in detail while creating the issue. If you don't have time to detail/describe the problem you are facing, assume that I also won't be having time to respond to your problem. -4. Post a sample code of the problem you are facing. If I copy paste the code directly from issue, I should be able to reproduce the problem you are facing. -5. Before posting the sample code, test your sample code yourself once. Only sample code should be tested, no other addition should be there while you are testing. -6. Have some print() function calls to display the values of some variables related to your problem. -7. Post the results of print() functions also in the issue. -8. Use the insert code feature of github to inset code and print outputs, so that the code is displyed neat. ! -9. If you have multiple lines of code, use tripple grave accent ( ``` ) to insert multiple lines of code. - - [Example:](https://docs.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks) - - ![1659809630082](image/README/1659809630082.png) +- Search existing issues first. +- Include a minimal reproducible snippet (symbol/exchange/interval, login/nologin) and the full traceback/output. +- Confirm the snippet reproduces on a clean environment (fresh venv). diff --git a/benchmark_realtime.py b/benchmark_realtime.py new file mode 100644 index 0000000..2192120 --- /dev/null +++ b/benchmark_realtime.py @@ -0,0 +1,73 @@ +import time +import logging +from tvDatafeed import TvDatafeed, Interval + +# Configure logging to suppress debug output during benchmark +logging.basicConfig(level=logging.INFO) + +def benchmark(): + tv = TvDatafeed() + + # 20 diverse symbols (Indices, Stocks, Crypto, Forex) + # Using specific exchanges to ensure they work + symbols = [ + "NSE:RELIANCE", "NSE:TCS", "NSE:INFY", "NSE:HDFCBANK", "NSE:ICICIBANK", + "NASDAQ:AAPL", "NASDAQ:MSFT", "NASDAQ:GOOGL", "NASDAQ:AMZN", "NASDAQ:TSLA", + "NYSE:BABA", "NYSE:NIO", "NYSE:F", "NYSE:GE", "NYSE:T", + "BINANCE:BTCUSDT", "BINANCE:ETHUSDT", "BINANCE:BNBUSDT", "BINANCE:ADAUSDT", "BINANCE:XRPUSDT" + ] + + print(f"Benchmarking with {len(symbols)} symbols...") + + # Method A: Loop get_hist + print("\nMethod A: Loop get_hist (1 bar)...") + start_time_a = time.time() + for symbol in symbols: + try: + # parsing exchange and symbol + if ":" in symbol: + exchange, sym = symbol.split(":") + else: + exchange = "NSE" # Default + sym = symbol + + # get_hist takes symbol and exchange + # We request 1 bar to be as fast as possible + tv.get_hist(symbol=sym, exchange=exchange, n_bars=1, interval=Interval.in_daily) + # print(f"Fetched {symbol}") + except Exception as e: + print(f"Failed {symbol}: {e}") + end_time_a = time.time() + duration_a = end_time_a - start_time_a + print(f"Method A took {duration_a:.2f} seconds") + + # Method B: Batch get_quotes + print("\nMethod B: Batch get_quotes...") + start_time_b = time.time() + try: + quotes = tv.get_quotes(symbols, timeout=10.0) # slightly larger timeout for benchmark safety + print(f"Fetched {len(quotes)} quotes") + # Optional: print one to verify structure + if quotes: + first_key = list(quotes.keys())[0] + print(f"Sample quote for {first_key}: {quotes[first_key]}") + except AttributeError: + print("get_quotes not implemented yet") + duration_b = 9999 + except Exception as e: + print(f"Method B failed: {e}") + duration_b = 9999 + else: + end_time_b = time.time() + duration_b = end_time_b - start_time_b + print(f"Method B took {duration_b:.2f} seconds") + + # Comparison + if duration_b < duration_a: + speedup = duration_a / duration_b + print(f"\nSpeedup: {speedup:.2f}x") + else: + print("\nNo speedup or failure.") + +if __name__ == "__main__": + benchmark() diff --git a/tvDatafeed/main.py b/tvDatafeed/main.py index d6656be..a1a1b23 100644 --- a/tvDatafeed/main.py +++ b/tvDatafeed/main.py @@ -1,353 +1,487 @@ -import datetime -import enum -import json -import logging -import random -import re -import string -import pandas as pd -from websocket import create_connection -import requests -import json - -logger = logging.getLogger(__name__) - - -class Interval(enum.Enum): - in_1_minute = "1" - in_3_minute = "3" - in_5_minute = "5" - in_15_minute = "15" - in_30_minute = "30" - in_45_minute = "45" - in_1_hour = "1H" - in_2_hour = "2H" - in_3_hour = "3H" - in_4_hour = "4H" - in_daily = "1D" - in_weekly = "1W" - in_monthly = "1M" - - -class TvDatafeed: - __sign_in_url = 'https://www.tradingview.com/accounts/signin/' - __search_url = 'https://symbol-search.tradingview.com/symbol_search/?text={}&hl=1&exchange={}&lang=en&type=&domain=production' - __ws_headers = json.dumps({"Origin": "https://data.tradingview.com"}) - __signin_headers = {'Referer': 'https://www.tradingview.com'} - __ws_timeout = 5 - - def __init__( - self, - username: str = None, - password: str = None, - ) -> None: - """Create TvDatafeed object - - Args: - username (str, optional): tradingview username. Defaults to None. - password (str, optional): tradingview password. Defaults to None. - """ - - self.ws_debug = False - - self.token = self.__auth(username, password) - - if self.token is None: - self.token = "unauthorized_user_token" - logger.warning( - "you are using nologin method, data you access may be limited" - ) - - self.ws = None - self.session = self.__generate_session() - self.chart_session = self.__generate_chart_session() - - def __auth(self, username, password): - - if (username is None or password is None): - token = None - - else: - data = {"username": username, - "password": password, - "remember": "on"} - try: - response = requests.post( - url=self.__sign_in_url, data=data, headers=self.__signin_headers) - token = response.json()['user']['auth_token'] - except Exception as e: - logger.error('error while signin') - token = None - - return token - - def __create_connection(self): - logging.debug("creating websocket connection") - self.ws = create_connection( - "wss://data.tradingview.com/socket.io/websocket", headers=self.__ws_headers, timeout=self.__ws_timeout - ) - - @staticmethod - def __filter_raw_message(text): - try: - found = re.search('"m":"(.+?)",', text).group(1) - found2 = re.search('"p":(.+?"}"])}', text).group(1) - - return found, found2 - except AttributeError: - logger.error("error in filter_raw_message") - - @staticmethod - def __generate_session(): - stringLength = 12 - letters = string.ascii_lowercase - random_string = "".join(random.choice(letters) - for i in range(stringLength)) - return "qs_" + random_string - - @staticmethod - def __generate_chart_session(): - stringLength = 12 - letters = string.ascii_lowercase - random_string = "".join(random.choice(letters) - for i in range(stringLength)) - return "cs_" + random_string - - @staticmethod - def __prepend_header(st): - return "~m~" + str(len(st)) + "~m~" + st - - @staticmethod - def __construct_message(func, param_list): - return json.dumps({"m": func, "p": param_list}, separators=(",", ":")) - - def __create_message(self, func, paramList): - return self.__prepend_header(self.__construct_message(func, paramList)) - - def __send_message(self, func, args): - m = self.__create_message(func, args) - if self.ws_debug: - print(m) - self.ws.send(m) - - @staticmethod - def __create_df(raw_data, symbol): - try: - out = re.search(r'"s":\[(.+?)\}\]', raw_data).group(1) - x = out.split(',{"') - data = list() - volume_data = True - - epoch = datetime.datetime(1970, 1, 1) - - for xi in x: - xi = re.split(r"\[|:|,|\]", xi) - - try: - ts_raw = float(xi[4]) - except ValueError: - # malformed row, skip - continue - - try: - # works for negative timestamps too (pre-1970) - ts = epoch + datetime.timedelta(0, ts_raw) - except OverflowError: - # insane timestamp, skip - continue - - row = [ts] - - for i in range(5, 10): - - # skip converting volume data if does not exists - if not volume_data and i == 9: - row.append(0.0) - continue - try: - row.append(float(xi[i])) - - except ValueError: - volume_data = False - row.append(0.0) - logger.debug('no volume data') - - data.append(row) - - data = pd.DataFrame( - data, columns=["datetime", "open", - "high", "low", "close", "volume"] - ).set_index("datetime") - data.insert(0, "symbol", value=symbol) - return data - except AttributeError: - logger.error("no data, please check the exchange and symbol") - - @staticmethod - def __format_symbol(symbol, exchange, contract: int = None): - - if ":" in symbol: - pass - elif contract is None: - symbol = f"{exchange}:{symbol}" - - elif isinstance(contract, int): - symbol = f"{exchange}:{symbol}{contract}!" - - else: - raise ValueError("not a valid contract") - - return symbol - - def get_hist( - self, - symbol: str, - exchange: str = "NSE", - interval: Interval = Interval.in_daily, - n_bars: int = 10, - fut_contract: int = None, - extended_session: bool = False, - ) -> pd.DataFrame: - """get historical data - - Args: - symbol (str): symbol name - exchange (str, optional): exchange, not required if symbol is in format EXCHANGE:SYMBOL. Defaults to None. - interval (str, optional): chart interval. Defaults to 'D'. - n_bars (int, optional): no of bars to download, max 5000. Defaults to 10. - fut_contract (int, optional): None for cash, 1 for continuous current contract in front, 2 for continuous next contract in front . Defaults to None. - extended_session (bool, optional): regular session if False, extended session if True, Defaults to False. - - Returns: - pd.Dataframe: dataframe with sohlcv as columns - """ - symbol = self.__format_symbol( - symbol=symbol, exchange=exchange, contract=fut_contract - ) - - interval = interval.value - - self.__create_connection() - - self.__send_message("set_auth_token", [self.token]) - self.__send_message("chart_create_session", [self.chart_session, ""]) - self.__send_message("quote_create_session", [self.session]) - self.__send_message( - "quote_set_fields", - [ - self.session, - "ch", - "chp", - "current_session", - "description", - "local_description", - "language", - "exchange", - "fractional", - "is_tradable", - "lp", - "lp_time", - "minmov", - "minmove2", - "original_name", - "pricescale", - "pro_name", - "short_name", - "type", - "update_mode", - "volume", - "currency_code", - "rchp", - "rtc", - ], - ) - - self.__send_message( - "quote_add_symbols", [self.session, symbol, - {"flags": ["force_permission"]}] - ) - self.__send_message("quote_fast_symbols", [self.session, symbol]) - - self.__send_message( - "resolve_symbol", - [ - self.chart_session, - "symbol_1", - '={"symbol":"' - + symbol - + '","adjustment":"splits","session":' - + ('"regular"' if not extended_session else '"extended"') - + "}", - ], - ) - self.__send_message( - "create_series", - [self.chart_session, "s1", "s1", "symbol_1", interval, n_bars], - ) - self.__send_message("switch_timezone", [ - self.chart_session, "exchange"]) - - raw_data = "" - - logger.debug(f"getting data for {symbol}...") - while True: - try: - result = self.ws.recv() - raw_data = raw_data + result + "\n" - except Exception as e: - logger.error(e) - break - - if "series_completed" in result: - break - - return self.__create_df(raw_data, symbol) - - def search_symbol(self, symbol: str, exchange: str = ''): - url = self.__search_url.format(symbol, exchange) - - symbols_list = [] - try: - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - 'Referer': 'https://www.tradingview.com/', - 'Origin': 'https://www.tradingview.com' - } - resp = requests.get(url, headers=headers) - - if resp.status_code != 200: - logger.error(f"Search symbol failed with status: {resp.status_code}, content: {resp.text[:100]}") - - symbols_list = json.loads(resp.text.replace( - '', '').replace('', '')) - except Exception as e: - logger.error(f"Exception in search_symbol: {e}") - - return symbols_list - - def find_exchange(self, symbol: str): - """ - Finds the exchange for a given symbol. - Returns the exchange of the first match found, or None if no match is found. - """ - matches = self.search_symbol(symbol) - if matches: - return matches[0]['exchange'] - return None - - - -if __name__ == "__main__": - logging.basicConfig(level=logging.DEBUG) - tv = TvDatafeed() - print(tv.get_hist("CRUDEOIL", "MCX", fut_contract=1)) - print(tv.get_hist("NIFTY", "NSE", fut_contract=1)) - print( - tv.get_hist( - "EICHERMOT", - "NSE", - interval=Interval.in_1_hour, - n_bars=500, - extended_session=False, - ) - ) +import datetime +import time +import enum + +import json +import logging +import random +import re +import string +import pandas as pd +from websocket import create_connection +import requests + +logger = logging.getLogger(__name__) + + +class Interval(enum.Enum): + in_1_minute = "1" + in_3_minute = "3" + in_5_minute = "5" + in_15_minute = "15" + in_30_minute = "30" + in_45_minute = "45" + in_1_hour = "1H" + in_2_hour = "2H" + in_3_hour = "3H" + in_4_hour = "4H" + in_daily = "1D" + in_weekly = "1W" + in_monthly = "1M" + + +class TvDatafeed: + __sign_in_url = 'https://www.tradingview.com/accounts/signin/' + __search_url = 'https://symbol-search.tradingview.com/symbol_search/?text={}&hl=1&exchange={}&lang=en&type=&domain=production' + __ws_headers = json.dumps({"Origin": "https://data.tradingview.com"}) + __signin_headers = {'Referer': 'https://www.tradingview.com'} + __ws_timeout = 5 + + def __init__( + self, + username: str = None, + password: str = None, + ) -> None: + """Create TvDatafeed object + + Args: + username (str, optional): tradingview username. Defaults to None. + password (str, optional): tradingview password. Defaults to None. + """ + + self.ws_debug = False + + self.token = self.__auth(username, password) + + if self.token is None: + self.token = "unauthorized_user_token" + logger.warning( + "you are using nologin method, data you access may be limited" + ) + + self.ws = None + self.session = self.__generate_session() + self.chart_session = self.__generate_chart_session() + + def __auth(self, username, password): + + if (username is None or password is None): + token = None + + else: + data = {"username": username, + "password": password, + "remember": "on"} + try: + response = requests.post( + url=self.__sign_in_url, data=data, headers=self.__signin_headers) + token = response.json()['user']['auth_token'] + except Exception as e: + logger.error('error while signin') + token = None + + return token + + def __create_connection(self): + logging.debug("creating websocket connection") + self.ws = create_connection( + "wss://data.tradingview.com/socket.io/websocket", headers=self.__ws_headers, timeout=self.__ws_timeout + ) + + @staticmethod + def __filter_raw_message(text): + try: + found = re.search('"m":"(.+?)",', text).group(1) + found2 = re.search('"p":(.+?"}"])}', text).group(1) + + return found, found2 + except AttributeError: + logger.error("error in filter_raw_message") + + @staticmethod + def __generate_session(): + stringLength = 12 + letters = string.ascii_lowercase + random_string = "".join(random.choice(letters) + for i in range(stringLength)) + return "qs_" + random_string + + @staticmethod + def __generate_chart_session(): + stringLength = 12 + letters = string.ascii_lowercase + random_string = "".join(random.choice(letters) + for i in range(stringLength)) + return "cs_" + random_string + + @staticmethod + def __prepend_header(st): + return "~m~" + str(len(st)) + "~m~" + st + + @staticmethod + def __construct_message(func, param_list): + return json.dumps({"m": func, "p": param_list}, separators=(",", ":")) + + def __create_message(self, func, paramList): + return self.__prepend_header(self.__construct_message(func, paramList)) + + def __send_message(self, func, args): + m = self.__create_message(func, args) + if self.ws_debug: + print(m) + self.ws.send(m) + + @staticmethod + def __create_df(raw_data, symbol): + try: + out = re.search(r'"s":\[(.+?)\}\]', raw_data).group(1) + x = out.split(',{"') + data = list() + volume_data = True + + epoch = datetime.datetime(1970, 1, 1) + + for xi in x: + xi = re.split(r"\[|:|,|\]", xi) + + try: + ts_raw = float(xi[4]) + except ValueError: + # malformed row, skip + continue + + try: + # works for negative timestamps too (pre-1970) + ts = epoch + datetime.timedelta(0, ts_raw) + except OverflowError: + # insane timestamp, skip + continue + + row = [ts] + + for i in range(5, 10): + + # skip converting volume data if does not exists + if not volume_data and i == 9: + row.append(0.0) + continue + try: + row.append(float(xi[i])) + + except ValueError: + volume_data = False + row.append(0.0) + logger.debug('no volume data') + + data.append(row) + + data = pd.DataFrame( + data, columns=["datetime", "open", + "high", "low", "close", "volume"] + ).set_index("datetime") + data.insert(0, "symbol", value=symbol) + return data + except AttributeError: + logger.error("no data, please check the exchange and symbol") + + @staticmethod + def __format_symbol(symbol, exchange, contract: int = None): + + if ":" in symbol: + pass + elif contract is None: + symbol = f"{exchange}:{symbol}" + + elif isinstance(contract, int): + symbol = f"{exchange}:{symbol}{contract}!" + + else: + raise ValueError("not a valid contract") + + return symbol + + def get_hist( + self, + symbol: str, + exchange: str = "NSE", + interval: Interval = Interval.in_daily, + n_bars: int = 10, + fut_contract: int = None, + extended_session: bool = False, + ) -> pd.DataFrame: + """get historical data + + Args: + symbol (str): symbol name + exchange (str, optional): exchange, not required if symbol is in format EXCHANGE:SYMBOL. Defaults to None. + interval (str, optional): chart interval. Defaults to 'D'. + n_bars (int, optional): no of bars to download, max 5000. Defaults to 10. + fut_contract (int, optional): None for cash, 1 for continuous current contract in front, 2 for continuous next contract in front . Defaults to None. + extended_session (bool, optional): regular session if False, extended session if True, Defaults to False. + + Returns: + pd.Dataframe: dataframe with sohlcv as columns + """ + symbol = self.__format_symbol( + symbol=symbol, exchange=exchange, contract=fut_contract + ) + + interval = interval.value + + self.__create_connection() + + try: + self.__send_message("set_auth_token", [self.token]) + self.__send_message("chart_create_session", [self.chart_session, ""]) + self.__send_message("quote_create_session", [self.session]) + self.__send_message( + "quote_set_fields", + [ + self.session, + "ch", + "chp", + "current_session", + "description", + "local_description", + "language", + "exchange", + "fractional", + "is_tradable", + "lp", + "lp_time", + "minmov", + "minmove2", + "original_name", + "pricescale", + "pro_name", + "short_name", + "type", + "update_mode", + "volume", + "currency_code", + "rchp", + "rtc", + ], + ) + + self.__send_message( + "quote_add_symbols", [self.session, symbol, + {"flags": ["force_permission"]}] + ) + self.__send_message("quote_fast_symbols", [self.session, symbol]) + + self.__send_message( + "resolve_symbol", + [ + self.chart_session, + "symbol_1", + '={"symbol":"' + + symbol + + '","adjustment":"splits","session":' + + ('"regular"' if not extended_session else '"extended"') + + "}", + ], + ) + self.__send_message( + "create_series", + [self.chart_session, "s1", "s1", "symbol_1", interval, n_bars], + ) + self.__send_message("switch_timezone", [ + self.chart_session, "exchange"]) + + raw_data = "" + + logger.debug(f"getting data for {symbol}...") + while True: + try: + result = self.ws.recv() + if isinstance(result, bytes): + result = result.decode("utf-8") + raw_data = raw_data + result + "\n" + except Exception as e: + logger.error(e) + break + + if "series_completed" in result: + break + + return self.__create_df(raw_data, symbol) + finally: + if self.ws: + self.ws.close() + + def search_symbol(self, symbol: str, exchange: str = ''): + + url = self.__search_url.format(symbol, exchange) + + symbols_list = [] + try: + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Referer': 'https://www.tradingview.com/', + 'Origin': 'https://www.tradingview.com' + } + resp = requests.get(url, headers=headers) + + if resp.status_code != 200: + logger.error(f"Search symbol failed with status: {resp.status_code}, content: {resp.text[:100]}") + + symbols_list = json.loads(resp.text.replace( + '', '').replace('', '')) + except Exception as e: + logger.error(f"Exception in search_symbol: {e}") + + return symbols_list + + def find_exchange(self, symbol: str): + """ + Finds the exchange for a given symbol. + Returns the exchange of the first match found, or None if no match is found. + """ + matches = self.search_symbol(symbol) + if matches: + return matches[0]['exchange'] + return None + + + def get_quotes(self, symbols: list[str], timeout: float = 5.0): + """Get real-time quotes for a list of symbols. + + Args: + symbols (list[str]): List of symbols (e.g., ["NSE:RELIANCE", "NASDAQ:AAPL"]) + timeout (float, optional): Timeout in seconds. Defaults to 5.0. + + Returns: + dict: Dictionary of quotes {symbol: {field: value, ...}} + """ + unique_symbols = set(symbols) + self.__create_connection() + + try: + self.__send_message("set_auth_token", [self.token]) + + # Setup session + self.__send_message("quote_create_session", [self.session]) + self.__send_message( + "quote_set_fields", + [ + self.session, + "ch", + "chp", + "lp", + "lp_time", + "status", + ], + ) + + # Add symbols + for symbol in unique_symbols: + self.__send_message( + "quote_add_symbols", [self.session, symbol] + ) + + results = {} + start_time = time.time() + buffer = "" + + while time.time() - start_time < timeout: + if len(results) >= len(unique_symbols): + break + + remaining = timeout - (time.time() - start_time) + if remaining <= 0: + break + + try: + self.ws.settimeout(remaining) + data = self.ws.recv() + if isinstance(data, bytes): + data = data.decode("utf-8") + buffer += data + except Exception: + break + + while "~m~" in buffer: + match = re.search(r"~m~(\d+)~m~", buffer) + if not match: + break + + header_len = len(match.group(0)) + msg_len = int(match.group(1)) + + if len(buffer) < match.start() + header_len + msg_len: + break + + msg_str = buffer[match.start() + header_len : match.start() + header_len + msg_len] + buffer = buffer[match.start() + header_len + msg_len:] + + if not msg_str: + continue + + try: + if re.match(r"^\d+$", msg_str): + continue + + msg = json.loads(msg_str) + + if not isinstance(msg, dict): + continue + + func = msg.get("m") + params = msg.get("p") + + if func == "qsd" and params and len(params) > 1: + if params[0] == self.session: + data_item = params[1] + symbol_name = data_item.get("n") + status = data_item.get("s") + values = data_item.get("v") + + if symbol_name: + if symbol_name not in results: + results[symbol_name] = {} + + if status: + results[symbol_name]["status"] = status + + if values: + results[symbol_name].update(values) + + elif func == "quote_completed" and params and len(params) > 1: + # Sometimes completion message comes differently or for other purposes + # But specifically looking for error handling here if possible + pass + + elif func == "quote_error" and params and len(params) > 1: + if params[0] == self.session: + symbol_name = params[1] + error_desc = params[2] if len(params) > 2 else "Unknown error" + # We treat this as a result so we stop waiting for it + if symbol_name not in results: + results[symbol_name] = {} + results[symbol_name]["status"] = "error" + results[symbol_name]["error"] = error_desc + + except Exception as e: + logger.debug(f"Error parsing message: {e}") + finally: + if self.ws: + self.ws.close() + + return results + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + tv = TvDatafeed() + print(tv.get_hist("CRUDEOIL", "MCX", fut_contract=1)) + print(tv.get_hist("NIFTY", "NSE", fut_contract=1)) + print( + tv.get_hist( + "EICHERMOT", + "NSE", + interval=Interval.in_1_hour, + n_bars=500, + extended_session=False, + ) + )