feat: add market_schedule module (#112)

* feat: add holiday_hours module

* feat: integrate holiday schedule

* feat: update gitignore to ignore .DS_Store

* refactor: imports

* fix: pre-commit

* feat: wip market schedule using winnow parser

* feat: wip add holiday day schedule parser

* fix: format

* feat: wip implement market_schedule parser

* fix: format

* feat: add market schedule can publish at test

* feat: use new market hours in pyth agent

* fix: format

* refactor: rollback module restructure

* rename market hours to legacy schedule

* chore: add comment

* feat: add support for 24:00

* fix: avoid parsing twice for verification

* refactor: use match instead of if

* refactor: use seq for time range parser

* refactor: improve parser

* refactor: improve parser

* refactor: implement from trait

* feat: add proptest

* fix: day kind regex and add comments

* refactor: improve comment

* chore: increase pyth agent minor version

Author: keyvankhademi

.gitignore

@@ -10,3 +10,6 @@ result
 **/*.rs.bk
 __pycache__
 keystore
+
+# Mac OS
+.DS_Store

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "pyth-agent"
-version = "2.5.2"
+version = "2.6.0"
 edition = "2021"
 
 [[bin]]
@@ -52,6 +52,8 @@ prometheus-client = "0.22.2"
 lazy_static = "1.4.0"
 toml_edit = "0.22.9"
 slog-bunyan = "2.5.0"
+winnow = "0.6.5"
+proptest = "1.4.0"
 
 [dev-dependencies]
 tokio-util = { version = "0.7.10", features = ["full"] }

Cargo.toml

@@ -1,4 +1,5 @@
 import asyncio
+from datetime import datetime
 import json
 import os
 import requests
@@ -63,9 +64,23 @@
         "quote_currency": "USD",
         "generic_symbol": "BTCUSD",
         "description": "BTC/USD",
+        "schedule": f"America/New_York;O,O,O,O,O,O,O;{datetime.now().strftime('%m%d')}/O"
     },
     "metadata": {"jump_id": "78876709", "jump_symbol": "BTCUSD", "price_exp": -8, "min_publishers": 1},
 }
+SOL_USD = {
+    "account": "",
+    "attr_dict": {
+        "symbol": "Crypto.SOL/USD",
+        "asset_type": "Crypto",
+        "base": "SOL",
+        "quote_currency": "USD",
+        "generic_symbol": "SOLUSD",
+        "description": "SOL/USD",
+        "schedule": f"America/New_York;O,O,O,O,O,O,O;{datetime.now().strftime('%m%d')}/C"
+    },
+    "metadata": {"jump_id": "78876711", "jump_symbol": "SOLUSD", "price_exp": -8, "min_publishers": 1},
+}
 AAPL_USD = {
     "account": "",
     "attr_dict": {
@@ -95,7 +110,7 @@
     },
     "metadata": {"jump_id": "78876710", "jump_symbol": "ETHUSD", "price_exp": -8, "min_publishers": 1},
 }
-ALL_PRODUCTS=[BTC_USD, AAPL_USD, ETH_USD]
+ALL_PRODUCTS=[BTC_USD, AAPL_USD, ETH_USD, SOL_USD]
 
 asyncio.set_event_loop(asyncio.new_event_loop())
 
@@ -277,6 +292,7 @@ def refdata_permissions(self, refdata_path):
                     "AAPL": {"price": ["some_publisher_a"]},
                     "BTCUSD": {"price": ["some_publisher_b", "some_publisher_a"]}, # Reversed order helps ensure permission discovery works correctly for publisher A
                     "ETHUSD": {"price": ["some_publisher_b"]},
+                    "SOLUSD": {"price": ["some_publisher_a"]},
                     }))
             f.flush()
             yield f.name
@@ -769,3 +785,38 @@ async def test_agent_respects_market_hours(self, client: PythAgentClient):
         assert final_price_account["price"] == 0
         assert final_price_account["conf"] == 0
         assert final_price_account["status"] == "unknown"
+
+    @pytest.mark.asyncio
+    async def test_agent_respects_holiday_hours(self, client: PythAgentClient):
+        '''
+        Similar to test_agent_respects_market_hours, but using SOL_USD and
+        asserting that nothing is published due to the symbol's all-closed holiday.
+        '''
+
+        # Fetch all products
+        products = {product["attr_dict"]["symbol"]: product for product in await client.get_all_products()}
+
+        # Find the product account ID corresponding to the AAPL/USD symbol
+        product = products[SOL_USD["attr_dict"]["symbol"]]
+        product_account = product["account"]
+
+        # Get the price account with which to send updates
+        price_account = product["price_accounts"][0]["account"]
+
+        # Send an "update_price" request
+        await client.update_price(price_account, 42, 2, "trading")
+        time.sleep(2)
+
+        # Send another update_price request to "trigger" aggregation
+        # (aggregation would happen if market hours were to fail, but
+        # we want to catch that happening if there's a problem)
+        await client.update_price(price_account, 81, 1, "trading")
+        time.sleep(2)
+
+        # Confirm that the price account has not been updated
+        final_product_state = await client.get_product(product_account)
+
+        final_price_account = final_product_state["price_accounts"][0]
+        assert final_price_account["price"] == 0
+        assert final_price_account["conf"] == 0
+        assert final_price_account["status"] == "unknown"

integration-tests/tests/test_integration.py

@@ -0,0 +1,8 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 173b9a862e3ad1149b0fdef292a11164ecab5b67b395857178f63294c3c9c0b7 # shrinks to s = "0000-0060"
+cc 6cf32e18287cb6de4b40f4326d1e9fd3be409086af3ccf75eac6f980c1f67052 # shrinks to s = TimeRange(00:00:00, 00:00:01)

proptest-regressions/agent/market_schedule.txt

@@ -63,7 +63,8 @@ Note that there is an Oracle and Exporter for each network, but only one Local S
 ################################################################################################################################## */
 
 pub mod dashboard;
-pub mod market_hours;
+pub mod legacy_schedule;
+pub mod market_schedule;
 pub mod metrics;
 pub mod pythd;
 pub mod remote_keypair_loader;

src/agent.rs

@@ -28,8 +28,10 @@ lazy_static! {
 }
 
 /// Weekly market hours schedule
+/// TODO: Remove after all publishers have upgraded to support the new schedule format
 #[derive(Clone, Default, Debug, Eq, PartialEq)]
-pub struct WeeklySchedule {
+#[deprecated(note = "This struct is deprecated, use MarketSchedule instead.")]
+pub struct LegacySchedule {
     pub timezone: Tz,
     pub mon:      MHKind,
     pub tue:      MHKind,
@@ -40,7 +42,7 @@ pub struct WeeklySchedule {
     pub sun:      MHKind,
 }
 
-impl WeeklySchedule {
+impl LegacySchedule {
     pub fn all_closed() -> Self {
         Self {
             timezone: Default::default(),
@@ -76,7 +78,7 @@ impl WeeklySchedule {
     }
 }
 
-impl FromStr for WeeklySchedule {
+impl FromStr for LegacySchedule {
     type Err = anyhow::Error;
     fn from_str(s: &str) -> Result<Self> {
         let mut split_by_commas = s.split(",");
@@ -235,9 +237,9 @@ mod tests {
         // Mon-Fri 9-5, inconsistent leading space on Tuesday, leading 0 on Friday (expected to be fine)
         let s = "Europe/Warsaw,9:00-17:00, 9:00-17:00,9:00-17:00,9:00-17:00,09:00-17:00,C,C";
 
-        let parsed: WeeklySchedule = s.parse()?;
+        let parsed: LegacySchedule = s.parse()?;
 
-        let expected = WeeklySchedule {
+        let expected = LegacySchedule {
             timezone: Tz::Europe__Warsaw,
             mon:      MHKind::TimeRange(
                 NaiveTime::from_hms_opt(9, 0, 0).unwrap(),
@@ -273,7 +275,7 @@ mod tests {
         // Valid but missing a timezone
         let s = "O,C,O,C,O,C,O";
 
-        let parsing_result: Result<WeeklySchedule> = s.parse();
+        let parsing_result: Result<LegacySchedule> = s.parse();
 
         dbg!(&parsing_result);
         assert!(parsing_result.is_err());
@@ -284,7 +286,7 @@ mod tests {
         // One day short
         let s = "Asia/Hong_Kong,C,O,C,O,C,O";
 
-        let parsing_result: Result<WeeklySchedule> = s.parse();
+        let parsing_result: Result<LegacySchedule> = s.parse();
 
         dbg!(&parsing_result);
         assert!(parsing_result.is_err());
@@ -294,7 +296,7 @@ mod tests {
     fn test_parsing_gibberish_timezone_is_error() {
         // Pretty sure that one's extinct
         let s = "Pangea/New_Dino_City,O,O,O,O,O,O,O";
-        let parsing_result: Result<WeeklySchedule> = s.parse();
+        let parsing_result: Result<LegacySchedule> = s.parse();
 
         dbg!(&parsing_result);
         assert!(parsing_result.is_err());
@@ -303,7 +305,7 @@ mod tests {
     #[test]
     fn test_parsing_gibberish_day_schedule_is_error() {
         let s = "Europe/Amsterdam,mondays are alright I guess,O,O,O,O,O,O";
-        let parsing_result: Result<WeeklySchedule> = s.parse();
+        let parsing_result: Result<LegacySchedule> = s.parse();
 
         dbg!(&parsing_result);
         assert!(parsing_result.is_err());
@@ -313,7 +315,7 @@ mod tests {
     fn test_parsing_too_many_days_is_error() {
         // One day too many
         let s = "Europe/Lisbon,O,O,O,O,O,O,O,O,C";
-        let parsing_result: Result<WeeklySchedule> = s.parse();
+        let parsing_result: Result<LegacySchedule> = s.parse();
 
         dbg!(&parsing_result);
         assert!(parsing_result.is_err());
@@ -322,7 +324,7 @@ mod tests {
     #[test]
     fn test_market_hours_happy_path() -> Result<()> {
         // Prepare a schedule of narrow ranges
-        let wsched: WeeklySchedule = "America/New_York,00:00-1:00,1:00-2:00,2:00-3:00,3:00-4:00,4:00-5:00,5:00-6:00,6:00-7:00".parse()?;
+        let wsched: LegacySchedule = "America/New_York,00:00-1:00,1:00-2:00,2:00-3:00,3:00-4:00,4:00-5:00,5:00-6:00,6:00-7:00".parse()?;
 
         // Prepare UTC datetimes that fall before, within and after market hours
         let format = "%Y-%m-%d %H:%M";
@@ -379,7 +381,7 @@ mod tests {
     #[test]
     fn test_market_hours_midnight_00_24() -> Result<()> {
         // Prepare a schedule of midnight-neighboring ranges
-        let wsched: WeeklySchedule =
+        let wsched: LegacySchedule =
             "Europe/Amsterdam,23:00-24:00,00:00-01:00,O,C,C,C,C".parse()?;
 
         let format = "%Y-%m-%d %H:%M";
@@ -433,8 +435,8 @@ mod tests {
         //   CDT/CET 6h offset in use for 2 weeks, CDT/CEST 7h offset after)
         // * Autumn 2023: Oct29(EU)-Nov5(US) (clocks go back 1h,
         //   CDT/CET 6h offset in use 1 week, CST/CET 7h offset after)
-        let wsched_eu: WeeklySchedule = "Europe/Amsterdam,9:00-17:00,O,O,O,O,O,O".parse()?;
-        let wsched_us: WeeklySchedule = "America/Chicago,2:00-10:00,O,O,O,O,O,O".parse()?;
+        let wsched_eu: LegacySchedule = "Europe/Amsterdam,9:00-17:00,O,O,O,O,O,O".parse()?;
+        let wsched_us: LegacySchedule = "America/Chicago,2:00-10:00,O,O,O,O,O,O".parse()?;
 
         let format = "%Y-%m-%d %H:%M";