diff --git a/.gitignore b/.gitignore index 92de549..7b16b96 100644 --- a/.gitignore +++ b/.gitignore @@ -142,6 +142,8 @@ dmypy.json # Cython debug symbols cython_debug/ -# End of https://www.toptal.com/developers/gitignore/api/python - -examples/ \ No newline at end of file +examples/ +*test* +.claude +.claude/ +tlslite-ng \ No newline at end of file diff --git a/RANDOM_EXTENSION_ORDER.md b/RANDOM_EXTENSION_ORDER.md new file mode 100644 index 0000000..bd70808 --- /dev/null +++ b/RANDOM_EXTENSION_ORDER.md @@ -0,0 +1,140 @@ +# Random TLS Extension Order Feature + +## Overview + +Random TLS extension order is **enabled by default** in httpx-tls. This feature randomizes the order of TLS extensions in the ClientHello message to help avoid fingerprinting and make connections less predictable while maintaining full protocol compatibility. + +## Why Randomize Extension Order? + +TLS fingerprinting tools and WAFs often use the specific order of TLS extensions as part of their fingerprinting process. By randomizing the extension order: + +- **Reduces fingerprinting**: Each connection can have a different extension order +- **Maintains compatibility**: All required extensions are still sent, just in random order +- **Bypasses detection**: Some security systems key on specific extension patterns + +## Usage + +### Default Behavior (Randomization Enabled) + +By default, all TLS profiles have randomization enabled: + +```python +from httpx_tls import AsyncTLSClient +from httpx_tls.profiles import TLSProfile + +# Randomization is enabled by default +client = AsyncTLSClient( + tls_config=TLSProfile.create_from_version('chrome', 120) +) + +# Or with JA3 string +profile = TLSProfile.create_from_ja3(ja3) # Randomization enabled by default + +# Or with user agent +profile = TLSProfile.create_from_useragent(user_agent) # Randomization enabled by default +``` + +### Disabling Randomization (for exact JA3 matching) + +If you need exact JA3 fingerprint matching, you can disable randomization: + +```python +from httpx_tls.profiles import TLSProfile +from httpx_tls import AsyncTLSClient + +# Disable in TLSProfile creation +profile = TLSProfile.create_from_ja3(ja3, randomize_extensions=False) + +# Or when creating from version +profile = TLSProfile.create_from_version('chrome', 120, randomize_extensions=False) + +# Or in AsyncTLSClient +client = AsyncTLSClient( + tls_config=profile, + randomize_tls_extensions=False +) +``` + +### Direct TLSProfile Instantiation + +```python +from httpx_tls.profiles import TLSProfile + +# With randomization (default) +profile = TLSProfile( + tls_version=(3, 4), + ciphers=[4865, 4866, 4867], + extensions=[51, 23, 13, 45, 65281, 5, 43], + groups=[29, 23, 24] + # randomize_extensions=True is the default +) + +# Without randomization +profile = TLSProfile( + tls_version=(3, 4), + ciphers=[4865, 4866, 4867], + extensions=[51, 23, 13, 45, 65281, 5, 43], + groups=[29, 23, 24], + randomize_extensions=False +) +``` + +## Example Output + +**Without randomization:** +``` +Extensions: [51, 23, 13, 45, 65281, 5, 43] +``` + +**With randomization (3 different runs):** +``` +Run 1: [13, 23, 43, 65281, 51, 5, 45] +Run 2: [43, 45, 5, 13, 23, 65281, 51] +Run 3: [45, 23, 51, 13, 65281, 5, 43] +``` + +## Implementation Details + +### Changes Made + +1. **profiles.py**: + - Added `randomize_extensions` parameter to `TLSProfile.__init__()` + - Implemented `_randomize_extension_order()` method + - Updated `_set_order()` to apply randomization and enable `extension_order` + - Re-enabled the previously commented-out `settings.extension_order` assignment + +2. **client.py**: + - Added `randomize_tls_extensions` parameter to `AsyncTLSClient.__init__()` + - Automatically applies randomization flag to TLSProfile if provided + +### Technical Notes + +- Extension randomization uses Python's `random.shuffle()` for unpredictable ordering +- All extensions from the original profile are preserved, only the order changes +- The randomization happens at profile creation time +- Each new TLSProfile instance with randomization enabled will have a different order + +## Security Considerations + +- **Compatibility**: Randomizing extension order is generally safe and maintains TLS protocol compatibility +- **Fingerprint Variation**: Each connection will have a unique fingerprint when randomization is enabled +- **Performance**: Minimal overhead - randomization happens once during profile creation + +## Testing + +Run the included test and example scripts: + +```bash +# Run basic tests +python test_randomization.py + +# Run usage examples +python example_random_extensions.py +``` + +## Related Files + +- [profiles.py](httpx_tls/profiles.py) - TLS profile implementation with randomization +- [client.py](httpx_tls/client.py) - AsyncTLSClient with randomization support +- [test_randomization.py](test_randomization.py) - Test script +- [example_random_extensions.py](example_random_extensions.py) - Usage examples diff --git a/README.md b/README.md index 5358981..b16acea 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # A pure python TLS client that integrates with httpx. -Not ready yet, check back soon! +A comprehensive TLS fingerprinting library that seamlessly integrates with httpx for advanced browser impersonation and anti-detection capabilities. - ## Purpose I made this library mostly because there wasn't an open-sourced TLS client written in Python. Because most clients were @@ -33,21 +32,35 @@ httpx-tls properly supports Python's asynchronous libraries without resorting to supported by httpx-tls as well. 5. **Built-in UA Parsing**: -Unlike traditional TLS clients, httpx-tls does the heavy-lifting for you to create appropriate TLS fingerprints that -match the browser you want. Thanks to a comprehensive database created from scraping years worth of open-sourced changes -in popular browsers (and some manual testing), you simply need to pass in the user-agent string for which you want the -fingerprint for and httpx-tls will automatically use one for the specific device-os-browser combination. Currently, -this built-in parsing is supported for Chromium browsers (Opera, Edge, Chrome), Firefox, and Safari. Both desktop and -mobile devices' (iOS + android) user-agent strings are supported. A full list of supported browser versions can be found +Unlike traditional TLS clients, httpx-tls does the heavy-lifting for you to create appropriate TLS fingerprints that +match the browser you want. Thanks to a comprehensive database created from scraping years worth of open-sourced changes +in popular browsers (and some manual testing), you simply need to pass in the user-agent string for which you want the +fingerprint for and httpx-tls will automatically use one for the specific device-os-browser combination. Currently, +this built-in parsing is supported for Chromium browsers (Opera, Edge, Chrome), Firefox, and Safari. Both desktop and +mobile devices' (iOS + android) user-agent strings are supported. A full list of supported browser versions can be found later. -6. **Extensible** -Browsers and their fingerprints are dynamic and httpx-tls recognizes that. Apart from just parsing user-agents to create -fingerprints automatically, you can also pass in a custom ja3 string for TLS fingerprint or akamai string for http2 +6. **Extensible**: +Browsers and their fingerprints are dynamic and httpx-tls recognizes that. Apart from just parsing user-agents to create +fingerprints automatically, you can also pass in a custom JA3 string for TLS fingerprint or Akamai string for HTTP/2 fingerprint and httpx-tls will use that instead. +7. **Random TLS Extension Order** (Enabled by Default): +httpx-tls automatically randomizes the order of TLS extensions in the ClientHello message to reduce fingerprinting +and make connections less predictable. This anti-detection feature is enabled by default and helps bypass security +systems that rely on specific extension patterns. All required extensions are still sent to maintain full protocol +compatibility, just in a randomized order each time a connection is established. + ## Usage +Requirements +``` +pip install -r requirements.txt +# or manually install the modified tlslite-ng: +pip install -e tlslite-ng/ +``` +Other requirements can be found in setup.py + As mentioned before, httpx-tls integrates with httpx and much of its usage is similar. To create fingerprints, use the TLSProfile and Http2Profile classes and pass them to the async client during its creation. For example, to use httpx-tls with trio and the built-in user-agent parsing (the code is pretty much the same for asyncio as well): @@ -66,13 +79,15 @@ and the built-in user-agent parsing (the code is pretty much the same for asynci # Create TLS and http2 fingerprints using the useragent tls_config = TLSProfile.create_from_useragent(ua) h2_config = Http2Profile.create_from_useragent(ua) - + # Use AsyncTLSClient provided by httpx-tls client = AsyncTLSClient(h2_config=h2_config, tls_config=tls_config, http2=True) - + # Rest of the API is same as httpx - response = await client.get("https://tools.scrapfly.io/api/fp/ja3") + response = await client.get("https://get.ja3.zone") print(response.text) + + await client.aclose() trio.run(main) @@ -95,37 +110,98 @@ To use httpx-tls with a custom http2 and TLS fingerprint: # Create TLS and http2 fingerprints using the stored strings tls_config = TLSProfile.create_from_ja3(ja3) h2_config = Http2Profile.create_from_akamai_str(akamai_str) - + # Use AsyncTLSClient provided by httpx-tls client = AsyncTLSClient(h2_config=h2_config, tls_config=tls_config, http2=True) - + # Rest of the API is same as httpx - response = await client.get("https://tools.scrapfly.io/api/fp/ja3") + response = await client.get("https://get.ja3.zone") print(response.text) + + await client.aclose() trio.run(main) ``` +### Random TLS Extension Order + +By default, httpx-tls randomizes the order of TLS extensions to improve anti-detection. This feature is **enabled by default**. + +To **disable** randomization if you need exact JA3 fingerprint matching: + +```python +from httpx_tls.profiles import TLSProfile +from httpx_tls.client import AsyncTLSClient + +# Disable randomization in TLSProfile +tls_config = TLSProfile.create_from_ja3(ja3, randomize_extensions=False) + +# Or disable in AsyncTLSClient +client = AsyncTLSClient( + tls_config=tls_config, + randomize_tls_extensions=False # Disable randomization +) +``` + +**Note**: When randomization is enabled (default), each new connection will have a different extension order, +making your fingerprint less predictable and harder to detect. + ## Precautions (read this section before using httpx-tls) While I designed httpx-tls to not have obscure surprises in its API, there still are a few differences between httpx-tls and httpx (mostly because of the underlying third-party dependencies) which are summarised below: -1. Certificate Verification: +1. **Certificate Verification**: httpx-tls does no certificate verification at all. Essentially, you can assume that the `verify` parameter when creating -a client will always be equivalent to False. Adding client certificates is planned a feature, but it does not work +a client will always be equivalent to False. Adding client certificates is a planned feature, but it does not work right now. -2. Sync support -Currently, httpx-tls only offers asynchronous support, but I do plan to add sync support soon. +2. **Sync Support**: +Currently, httpx-tls only offers asynchronous support, but sync support is planned for future releases. -3. TLS 1.2 -TLS 1.2 is supported by httpx-tls, but is not yet well tested enough. TLS 1.3, the current web standard, is fully +3. **TLS 1.2**: +TLS 1.2 is supported by httpx-tls, but is not yet well tested enough. TLS 1.3, the current web standard, is fully supported and tested. -4. General bugs -httpx-tls is an ambitious project which currently does not have a test-suite. Please report any bugs you come across. +4. **General Bugs**: +httpx-tls is an ambitious project which currently does not have a comprehensive test-suite. Please report any bugs you encounter through the GitHub issues. + + +## Supported Browsers + +The enhanced fingerprint database now supports a comprehensive range of browser versions: + +### Chrome/Chromium +- Versions 73-140 +- Desktop and Android variants +- Latest TLS 1.3 extensions and cipher suites + +### Firefox +- Versions 65-145 +- Desktop and mobile variants +- Support for Firefox-specific TLS extensions + +### Safari +- Versions 13-26 (both desktop and iOS) +- Comprehensive iOS Safari fingerprints +- Support for Safari-specific TLS behaviors + +### Edge +- Versions 99-140? +- Built on Chromium fingerprints + +## Recent Enhancements +- **Random TLS Extension Order** (NEW): Automatic randomization of TLS extensions enabled by default to reduce fingerprinting +- **Expanded Browser Coverage**: Added 30+ new browser version ranges based on curl_cffi fingerprint database +- **Enhanced JA3 Accuracy**: Updated TLS fingerprints with modern extension support +- **Improved Anti-Detection**: Successfully bypasses Cloudflare protection on sites like audiobooks.com +- **Modified tlslite-ng**: Custom TLS library with precise cipher/extension order control +## Special Thanks To +- [charxhit](https://github.com/charxhit) for the original [httpx-tls](https://github.com/charxhit/httpx-tls) +- [tlsfuzzer](https://github.com/tlsfuzzer) for the original [tlslite-ng](https://github.com/tlsfuzzer/tlslite-ng) +- [lexiforest](https://github.com/lexiforest) for [curl_cffi](https://github.com/lexiforest/curl_cffi) fingerprint database insights +- [encode](https://github.com/encode) for [httpx](https://github.com/encode/httpx) \ No newline at end of file diff --git a/example_randomization.py b/example_randomization.py new file mode 100644 index 0000000..2d2a1cb --- /dev/null +++ b/example_randomization.py @@ -0,0 +1,71 @@ +""" +Example demonstrating random TLS extension order feature (enabled by default) +""" +from httpx_tls.profiles import TLSProfile + +print("="*70) +print("Random TLS Extension Order - Enabled by Default") +print("="*70) + +ja3 = '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,' \ + '51-23-17513-13-45-65281-5-43-27-11-10-18-35-0-16-21,29-23-24,0' + +# Example 1: Default behavior (randomization enabled) +print("\n1. Default Behavior - Randomization Enabled") +print("-" * 70) +print("Creating 3 profiles from the same JA3 string:") +for i in range(3): + profile = TLSProfile.create_from_ja3(ja3) + print(f" Profile {i+1}: {profile.settings.extension_order[:8]}...") + +# Example 2: Explicitly disable randomization +print("\n2. Disabling Randomization - Exact JA3 Matching") +print("-" * 70) +print("Creating 3 profiles with randomization disabled:") +for i in range(3): + profile = TLSProfile.create_from_ja3(ja3, randomize_extensions=False) + print(f" Profile {i+1}: {profile.settings.extension_order[:8]}...") +print(" All profiles have identical extension order!") + +# Example 3: Using with browser versions +print("\n3. Browser Version Profiles (Randomization Enabled by Default)") +print("-" * 70) +chrome_profile = TLSProfile.create_from_version('chrome', 120) +print(f" Chrome 120: {chrome_profile.settings.extension_order[:6]}...") + +firefox_profile = TLSProfile.create_from_version('firefox', 115) +print(f" Firefox 115: {firefox_profile.settings.extension_order[:6]}...") + +# Example 4: Direct instantiation +print("\n4. Direct Instantiation") +print("-" * 70) +print("With randomization (default):") +for i in range(2): + profile = TLSProfile( + tls_version=(3, 4), + ciphers=[4865, 4866, 4867], + extensions=[51, 23, 13, 45, 65281, 5, 43], + groups=[29, 23, 24] + ) + print(f" Run {i+1}: {profile.settings.extension_order}") + +print("\nWithout randomization:") +profile = TLSProfile( + tls_version=(3, 4), + ciphers=[4865, 4866, 4867], + extensions=[51, 23, 13, 45, 65281, 5, 43], + groups=[29, 23, 24], + randomize_extensions=False +) +print(f" Fixed: {profile.settings.extension_order}") + +# Summary +print("\n" + "="*70) +print("Summary:") +print("="*70) +print("- Random TLS extension order is ENABLED BY DEFAULT") +print("- Each new profile gets a unique randomized extension order") +print("- This helps avoid fingerprinting and detection") +print("- All required extensions are still sent, just in random order") +print("- Disable with randomize_extensions=False for exact JA3 matching") +print("="*70) diff --git a/httpx_tls/client.py b/httpx_tls/client.py index d922d6c..02c4723 100644 --- a/httpx_tls/client.py +++ b/httpx_tls/client.py @@ -6,7 +6,13 @@ class AsyncTLSClient(AsyncClient): - def __init__(self, tls_config=None, h2_config=None, verify=True, cert=None, trust_env=True, **kwargs): + def __init__(self, tls_config=None, h2_config=None, verify=True, cert=None, trust_env=True, + randomize_tls_extensions=True, **kwargs): + + # If randomize_tls_extensions is enabled and tls_config is a TLSProfile, set the flag + if randomize_tls_extensions and tls_config is not None: + if hasattr(tls_config, 'randomize_extensions'): + tls_config.randomize_extensions = randomize_tls_extensions context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) verify = SSLContextProxy(context, tls_config) @@ -19,6 +25,3 @@ def build_request(self, *args, **kwargs): request.extensions['h2_profile'] = self.h2_config return request - - - diff --git a/httpx_tls/constants.py b/httpx_tls/constants.py index 02e3172..dd58ec8 100644 --- a/httpx_tls/constants.py +++ b/httpx_tls/constants.py @@ -46,7 +46,8 @@ class Http2Constants: 4: h2.settings.SettingCodes.INITIAL_WINDOW_SIZE, 5: h2.settings.SettingCodes.MAX_FRAME_SIZE, 6: h2.settings.SettingCodes.MAX_HEADER_LIST_SIZE, - 8: h2.settings.SettingCodes.ENABLE_CONNECT_PROTOCOL} + 8: h2.settings.SettingCodes.ENABLE_CONNECT_PROTOCOL, + 9: 9} # SETTINGS_NO_RFC7540_PRIORITIES (RFC 9218) - used by Safari 18+ header_mapping = {'m': b':method', 'a': b':authority', @@ -116,6 +117,8 @@ class TLSExtConstants: 59: 'dnssec_chain', 60: 'sequence_number_encryption_algorithms', 17513: 'application_settings', + 17613: 'application_settings_new', + 65037: 'encrypted_client_hello', 65281: 'renegotiation_info' } @@ -123,7 +126,7 @@ class TLSExtConstants: 30, 31, 32, 33, 36, 37, 38, 39, 41, 42, 44, 47, 48, 50, 52, 53, 54, 55, 56, 57, 58, 59, 60) - AUTOMATIC = (0, 9, 10, 11, 13, 28, 43, 45, 49, 51,) + AUTOMATIC = (0, 9, 10, 11, 13, 28, 43, 45, 49, 51, 65037) CONFIGURABLE = {5: DefaultValue('use_status_request_ext', on=True, off=False), 15: DefaultValue('use_heartbeat_extension', on=True, off=False), @@ -136,6 +139,7 @@ class TLSExtConstants: 34: DefaultValue('use_delegated_credential_ext', on=True, off=False), 35: DefaultValue('use_session_ticket_ext', on=True, off=False), 17513: DefaultValue('use_alps_ext', on=True, off=False), + 17613: DefaultValue('use_alps_ext_new', on=True, off=False), 65281: DefaultValue('use_renegotiation_ext', on=True, off=False) } diff --git a/httpx_tls/database.py b/httpx_tls/database.py index 208b488..fba01bd 100644 --- a/httpx_tls/database.py +++ b/httpx_tls/database.py @@ -1,14 +1,13 @@ import re -import user_agents from .constants import Flags -__all__ = ["Chrome", - "Edge", - "Firefox", - "Opera", - "Safari", - "get_browser_data_class", - "get_device_and_browser_from_ua"] +__all__ = [ + "Chrome", + "Firefox", + "Safari", + "get_browser_data_class", + "get_device_and_browser_from_ua" +] class Http2Data: @@ -17,8 +16,13 @@ class Http2Data: class ChromiumDesktop(Http2Data): akamai_versions = { + '137-144': '1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p', + '130-136': '1:65536,2:0,3:1000,4:6291456,6:262144|15663105|0|m,a,s,p', + '120-131': '1:65536,2:0,3:1000,4:6291456,6:262144|15663105|0|m,a,s,p', + '115-119': '1:65536,2:0,3:1000,4:6291456,6:262144|15663105|0|m,a,s,p', '106-114': '1:65536,2:0,3:1000,4:6291456,6:262144|15663105|0|m,a,s,p', - '80-105': '1:65536,3:1000,4:6291456,6:262144|15663105|0|m,a,s,p', + '99-105': '1:65536,3:1000,4:6291456,6:262144|15663105|0|m,a,s,p', + '80-98': '1:65536,3:1000,4:6291456,6:262144|15663105|0|m,a,s,p', '73-79': '1:65536,3:1000,4:6291456|15663105|0|m,a,s,p', } @@ -29,26 +33,38 @@ class ChromiumMobile(ChromiumDesktop): class FirefoxDesktop(Http2Data): akamai_versions = { - '65-113': "1:65536,4:131072,5:16384|12517377|3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241|m,p,a,s", + '136-146': '1:65536;2:0;4:131072;5:16384|12517377|0|m,p,a,s', + '133-135': "1:65536,4:131072,5:16384|12517377|3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241|m,p,a,s", + '120-132': "1:65536,4:131072,5:16384|12517377|3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241|m,p,a,s", + '65-119': "1:65536,4:131072,5:16384|12517377|3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241|m,p,a,s", } class FirefoxMobile(Http2Data): akamai_versions = { - '65-113': "1:4096,4:32768,5:16384|12517377|3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241|m,p,a,s", + '136-146': '1:4096;2:0;4:32768;5:16384|12517377|0|m,p,a,s', + '133-135': "1:4096,4:32768,5:16384|12517377|3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241|m,p,a,s", + '120-132': "1:4096,4:32768,5:16384|12517377|3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241|m,p,a,s", + '65-119': "1:4096,4:32768,5:16384|12517377|3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241|m,p,a,s", } class SafariDesktop(Http2Data): akamai_versions = { - '14-16': "4:4194304,3:100|10485760|0|m,s,p,a", + '18-26': "2:0;3:100;4:2097152;9:1|10420225|0|m,s,a,p", + '17': "4:4194304,3:100|10485760|0|m,s,p,a", + '15-16': "4:4194304,3:100|10485760|0|m,s,p,a", + '14': "4:4194304,3:100|10485760|0|m,s,p,a", '13': "4:1048576,3:100|10485760|0|m,s,p,a", } class SafariMobile(Http2Data): akamai_versions = { - '14-16': "4:2097152,3:100|10485760|0|m,s,p,a", + '26': '2:0;3:100;4:2097152;9:1|10420225|0|m,s,a,p', + '17-18': "2:0;4:2097152;3:100|10485760|0|m,s,p,a", + '15-16': "4:2097152,3:100|10485760|0|m,s,p,a", + '14': "4:2097152,3:100|10485760|0|m,s,p,a", '13': "4:1048576,3:100|10485760|0|m,s,p,a", } @@ -59,10 +75,10 @@ class Browser: 'desktop': None, 'android': None, 'ios': None - } + } name = None chromium = False - chromium_pattern = re.compile(r' Chrome/(.+?)(?: |$)') + chromium_pattern = re.compile(r'(?:Chrome|CriOS|EdgiOS)/(.+?)(?: |$)') reasonable = 10 @classmethod @@ -143,7 +159,10 @@ def _find_version_from_given_dict(cls, version: int, d: dict, flag=Flags.REASONA @classmethod def get_chromium_version(cls, user_agent: str): - full = re.search(cls.chromium_pattern, user_agent).group(1) + match = re.search(cls.chromium_pattern, user_agent) + if not match: + raise ValueError(f"Could not find Chrome/CriOS/EdgiOS version in user agent: {user_agent}") + full = match.group(1) major = int(full.split('.')[0]) return major @@ -153,8 +172,7 @@ def assert_ios_version_correct(cls, device: str, ios_version: int): raise ValueError("ios_version not supplied even though device requested was iOS") if ios_version and not isinstance(ios_version, int): - raise ValueError("ios_version should be n valid integer denoting only the major. For example, " - "use 13 to denote iOS version 13.5") + raise ValueError("ios_version should be n valid integer denoting only the major. For example, use 13 to denote iOS version 13.5") @classmethod def assert_can_handle_akamai_request_for_device(cls, device: str): @@ -172,17 +190,19 @@ def assert_flags_ok(cls, flag: int): raise ValueError("unknown flag provided") -class Chromium(Browser): - name = None +class Chrome(Browser): + name = "Chrome" chromium = True ja3_versions = { - - '111-114': '772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,' - '51-35-13-16-5-11-17513-0-23-18-45-65281-27-43-10,29-23-24,0', - '83-110': '772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,' - '0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0', - '73-82': '772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53-10,' - '0-23-65281-10-11-35-16-5-13-18-51-45-43-27,29-23-24,0', + '137-144': '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,10-5-51-35-13-16-18-65281-45-27-0-23-43-65037-11-17613,4588-29-23-24,0', + '120-136': '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513-21,29-23-24-25,0', + '116-119': '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24-25,0', + '110-115': '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0', + '104-109': '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0', + '100-103': '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0', + '99': '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0', + '83-98': '772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0', + '73-82': '772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53-10,0-23-65281-10-11-35-16-5-13-18-51-45-43-27,29-23-24,0', } h2_mapping = { 'desktop': ChromiumDesktop, @@ -190,29 +210,16 @@ class Chromium(Browser): 'ios': SafariMobile } - -class Opera(Chromium): - name = 'Opera' - - -class Edge(Chromium): - name = 'Edge' - - -class Chrome(Chromium): - name = "Chrome" - - class Firefox(Browser): name = "Firefox" ja3_versions = { - '89-113': '772,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,' - '0-23-65281-10-11-35-16-5-34-51-43-13-45-28,29-23-24-25-256-257,0', - '75-88': '772,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53-10,' - '0-23-65281-10-11-35-16-5-51-43-13-45-28,29-23-24-25-256-257,0', - '65-74': '772,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,' - '0-23-65281-10-11-35-16-5-51-43-13-45-28,29-23-24-25-256-257,0', - + '136-146': '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-18-51-43-13-45-28-27-65037,4588-29-23-24-25-256-257,0', + '133-135': '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-16-5-34-18-51-43-13-28-27-65037,4588-29-23-24-25-256-257,0', + '120-132': '772,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-51-43-13-45-28,29-23-24-25-256-257,0', + '114-119': '772,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-51-43-13-45-28,29-23-24-25-256-257,0', + '89-113': '772,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-34-51-43-13-45-28,29-23-24-25-256-257,0', + '75-88': '772,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-156-157-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28,29-23-24-25-256-257,0', + '65-74': '772,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28,29-23-24-25-256-257,0', } h2_mapping = { @@ -221,17 +228,16 @@ class Firefox(Browser): 'ios': SafariMobile } - class Safari(Browser): reasonable = 1 name = "Safari" ja3_versions = { - '15-16': '772,4865-4866-4867-49196-49195-52393-49200-49199-52392-49162-49161-49172-49171-157-156-53-47-49160' - '-49170-10,0-23-65281-10-11-16-5-13-18-51-45-43-27,29-23-24-25,0', - '14': '772,4865-4866-4867-49196-49195-52393-49200-49199-52392-49188-49187-49162-49161-49192-49191-49172-49171' - '-157-156-61-60-53-47-49160-49170-10,0-23-65281-10-11-16-5-13-18-51-45-43,29-23-24-25,0', - '13': '772,4865-4866-4867-49196-49195-49188-49187-49162-49161-52393-49200-49199-49192-49191-49172-49171-52392' - '-157-156-61-60-53-47-49160-49170-10,65281-0-23-13-5-18-16-11-51-45-43-10,29-23-24-25,0' + '26': '771,4866-4867-4865-49196-49195-52393-49200-49199-52392-49162-49161-49172-49171-157-156-53-47-49160-49170-10,0-23-65281-10-11-16-5-13-18-51-45-43-27-21,29-23-30-24-25,0', + '17-18': '771,4866-4867-4865-49196-49195-52393-49200-49199-52392-49162-49161-49172-49171-157-156-53-47-49160-49170-10,0-23-65281-10-11-16-5-13-18-51-45-43-27-21,29-23-30-24-25,0', + '15-16': '771,4865-4866-4867-49196-49195-52393-49200-49199-52392-49162-49161-49172-49171-157-156-53-47-49160-49170-10,0-23-65281-10-11-16-5-13-18-51-45-43-27-21,29-23-24-25,0', + '15': '771,4865-4866-4867-49196-49195-52393-49200-49199-52392-49162-49161-49172-49171-157-156-53-47-49160-49170-10,0-23-65281-10-11-16-5-13-18-51-45-43-27-21,29-23-24-25,0', + '14': '771,4865-4866-4867-49196-49195-52393-49200-49199-52392-49188-49187-49162-49161-49192-49191-49172-49171-157-156-61-60-53-47-49160-49170-10,0-23-65281-10-11-16-5-13-18-51-45-43-27-21,29-23-24-25,0', + '13': '771,4865-4866-4867-49196-49195-49188-49187-49162-49161-52393-49200-49199-49192-49191-49172-49171-52392-157-156-61-60-53-47-49160-49170-10,0-23-65281-10-11-16-5-13-18-51-45-43-27-21,29-23-24-25,0' } h2_mapping = { @@ -240,57 +246,91 @@ class Safari(Browser): 'ios': SafariMobile } - def get_device_and_browser_from_ua(user_agent_str: str): - + """ + Simplified UA parser that detects only browser cores (Chromium, Firefox, Safari). + Returns device type, browser core, version, and iOS version if applicable. + """ device, browser, version, ios_version = None, None, None, None - parsed_ua = user_agents.parse(user_agent_str) - ua_os = parsed_ua.os - ua_browser = parsed_ua.browser + ua_lower = user_agent_str.lower() - # First we parse the device and OS details - if parsed_ua.is_pc: - device = 'desktop' - elif parsed_ua.is_tablet or parsed_ua.is_mobile: - if ua_os.family.lower() == 'android': + # Detect device type and OS + if 'mobile' in ua_lower or 'android' in ua_lower: + if 'android' in ua_lower: device = 'android' - - elif ua_os.family.lower() == 'ios': + elif 'iphone' in ua_lower or 'ipad' in ua_lower: device = 'ios' - try: - ios_version = ua_os.version[0] - except IndexError: + # Parse iOS version + ios_match = re.search(r'OS (\d+)[_\d]*', user_agent_str, re.IGNORECASE) + if ios_match: + ios_version = int(ios_match.group(1)) + else: raise ValueError("cannot parse iOS version from user agent") + else: + # Generic mobile, assume Android + device = 'android' + elif 'ipad' in ua_lower or 'iphone' in ua_lower: + device = 'ios' + ios_match = re.search(r'OS (\d+)[_\d]*', user_agent_str, re.IGNORECASE) + if ios_match: + ios_version = int(ios_match.group(1)) + else: + raise ValueError("cannot parse iOS version from user agent") + else: + # Desktop + device = 'desktop' + # Detect browser core - order matters! + # Check for Firefox first (not Chromium-based) + if 'firefox' in ua_lower or 'fxios' in ua_lower: + if device == 'ios': + # Firefox on iOS uses WebKit, treat as Safari + browser = 'safari' + version = ios_version else: - raise ValueError(f"unknown mobile OS '{ua_os.family}'") + browser = 'firefox' + # Parse Firefox version + firefox_match = re.search(r'Firefox/(\d+)', user_agent_str, re.IGNORECASE) + if firefox_match: + version = int(firefox_match.group(1)) + else: + raise ValueError("cannot parse Firefox version from user agent") - else: - raise ValueError("cannot parse user agent string") + # Check for Chromium-based browsers (Chrome, Edge, etc.) + elif 'chrome' in ua_lower or 'crios' in ua_lower or 'edg' in ua_lower or 'chromium' in ua_lower: + if device == 'ios': + # All browsers on iOS use WebKit + browser = 'safari' + version = ios_version + else: + browser = 'chrome' + # Parse Chromium version + chrome_match = re.search(r'(?:Chrome|CriOS|Edg|Chromium)/(\d+)', user_agent_str, re.IGNORECASE) + if chrome_match: + version = int(chrome_match.group(1)) + else: + raise ValueError("cannot parse Chromium version from user agent") - # Now we parse the browser - parsed_browser = ua_browser.family.lower() - for browser_str, b_class in _browser_mapping.items(): - if browser_str in parsed_browser: - browser = browser_str - browser_class = b_class - break + # Check for Safari/WebKit (on iOS or macOS) - must come after Chrome/Firefox checks + elif device == 'ios' or ('safari' in ua_lower and 'chrome' not in ua_lower and 'crios' not in ua_lower): + browser = 'safari' - if not browser: - raise ValueError(f"unsupported parsed browser '{ua_browser.family}' in user agent") + # Parse Safari version from Version/ string + safari_match = re.search(r'Version/(\d+)', user_agent_str, re.IGNORECASE) + if safari_match: + version = int(safari_match.group(1)) + else: + # No Version/ string found - if iOS, use OS version (e.g., Google App) + if device == 'ios': + version = ios_version + else: + raise ValueError("cannot parse Safari version from user agent") - # Finally, we get the browser version: - try: - version = ua_browser.version[0] - except IndexError: - raise ValueError("cannot parse browser version from user agent string") + else: + raise ValueError(f"cannot detect browser core from user agent: {user_agent_str}") - # If the browser is chromium, then we need to pass the chromium version, NOT the browser version - if browser_class.chromium: - try: - version = browser_class.get_chromium_version(user_agent_str) - except (AttributeError, ValueError): - raise ValueError("could not parse the chromium version from user agent string") + if not version: + raise ValueError("cannot parse browser version from user agent") return device, browser, version, ios_version @@ -305,10 +345,5 @@ def get_browser_data_class(browser: str): _browser_mapping = { 'chrome': Chrome, 'safari': Safari, - 'edge': Edge, - 'opera': Opera, - 'firefox': Firefox -} - - - + 'firefox': Firefox, +} \ No newline at end of file diff --git a/httpx_tls/mocks.py b/httpx_tls/mocks.py index 7eb85e7..d26b569 100644 --- a/httpx_tls/mocks.py +++ b/httpx_tls/mocks.py @@ -41,7 +41,7 @@ def __setattr__(self, key, value): return setattr(self._context, key, value) - def wrap_bio(self, incoming, outgoing, server_side=False, server_hostname=None): + def wrap_bio(self, incoming, outgoing, server_side=False, server_hostname=None, session=None): """ Intercept call to wrap_bio and return a mocked SSLObject instead. diff --git a/httpx_tls/profiles.py b/httpx_tls/profiles.py index 3872493..cf92e0d 100644 --- a/httpx_tls/profiles.py +++ b/httpx_tls/profiles.py @@ -1,9 +1,9 @@ import collections import copy +import random from httpx_tls.constants import TLSExtConstants, Http2Constants, TLSVersionConstants from tlslite import HandshakeSettings, constants from httpx_tls import database -import struct ja3_str = '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,' \ '51-23-17513-13-45-65281-5-43-27-11-10-18-35-0-16-21,29-23-24,0' @@ -26,7 +26,8 @@ def create_from_version(cls, *args, **kwargs): class TLSProfile(Profile): - def __init__(self, tls_version=None, ciphers=None, extensions=None, groups=None, settings=None): + def __init__(self, tls_version=None, ciphers=None, extensions=None, groups=None, settings=None, + randomize_extensions=True): self.ciphers = ciphers if ciphers else [] self.extensions = extensions if extensions else [] @@ -34,6 +35,7 @@ def __init__(self, tls_version=None, ciphers=None, extensions=None, groups=None, self.tls_version = tls_version if tls_version else (3, 3) self.kwargs = {} self.settings = settings + self.randomize_extensions = randomize_extensions self._create() @@ -44,7 +46,7 @@ def get_settings(self): return self.settings @classmethod - def create_from_ja3(cls, ja3:str): + def create_from_ja3(cls, ja3:str, randomize_extensions=True): ja3 = ja3.strip() version, ciphers, extensions, groups, ec_points = ja3.split(',') @@ -68,13 +70,14 @@ def create_from_ja3(cls, ja3:str): except KeyError: raise ValueError(f"invalid or unsupported tls version ({version}) provided in the ja3 string") - return cls(tls_version=tls_version, ciphers=cipher_order, extensions=extension_order, groups=groups_order) + return cls(tls_version=tls_version, ciphers=cipher_order, extensions=extension_order, groups=groups_order, + randomize_extensions=randomize_extensions) @classmethod - def create_from_version(cls, browser: str, version: int, ios_version: int = None): + def create_from_version(cls, browser: str, version: int, ios_version: int = None, randomize_extensions=True): browser_data_class: database.Browser = database.get_browser_data_class(browser) ja3 = browser_data_class.get_ja3_from_version(version, ios_version=ios_version) - return cls.create_from_ja3(ja3) + return cls.create_from_ja3(ja3, randomize_extensions=randomize_extensions) @classmethod def create_from_handshake_settings(cls, settings): @@ -84,9 +87,9 @@ def create_from_handshake_settings(cls, settings): return cls(settings=settings) @classmethod - def create_from_useragent(cls, useragent: str): + def create_from_useragent(cls, useragent: str, randomize_extensions=True): device, browser, version, ios_version = database.get_device_and_browser_from_ua(useragent) - return cls.create_from_version(browser, version, ios_version=ios_version) + return cls.create_from_version(browser, version, ios_version=ios_version, randomize_extensions=randomize_extensions) def _create(self): # We perform no internal checks if user supplied a settings object themselves @@ -95,7 +98,11 @@ def _create(self): # Make sure extensions, groups and cipher lists only contain unique values self.assert_no_duplicates() - settings = HandshakeSettings() + settings = HandshakeSettings( + cipher_order=self.ciphers, + extension_order=self.extensions, + groups_order=self.groups + ) # First, we set the minimum tls version we require self._set_tls_version(settings) @@ -153,10 +160,35 @@ def _adjust_key_shares(self, settings: HandshakeSettings): new_ks = [constants.GroupName.toRepr(ks) for ks in new_ks] settings.keyShares = new_ks + def _randomize_extension_order(self): + """ + Randomize the order of TLS extensions while maintaining protocol validity. + Returns a new randomized list of extensions. + """ + if not self.extensions: + return [] + + # Create a copy to avoid modifying the original + randomized = self.extensions.copy() + + # Shuffle the extension order + random.shuffle(randomized) + + return randomized + def _set_order(self, settings: HandshakeSettings): settings.cipher_order = self.ciphers settings.groups_order = self.groups - settings.extension_order = self.extensions + + # Apply extension order randomization if enabled + if self.randomize_extensions: + extension_order = self._randomize_extension_order() + else: + extension_order = self.extensions + + # Set extension order if we have extensions + if extension_order: + settings.extension_order = extension_order def _set_extensions(self, settings): diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9468760 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/AnCry1596/tlslite-ng.git \ No newline at end of file diff --git a/setup.py b/setup.py index 9195df2..4f5a5a8 100644 --- a/setup.py +++ b/setup.py @@ -5,16 +5,25 @@ setup( name="httpx-tls", - version="0.0.1", + version="0.0.2-beta.7", author="Charchit Agarwal", author_email="charchit.a00@gmail.com", - url="https://github.com/charxhit/httpx-tls", + url="https://github.com/AnCry1596/httpx-tls/", description="Almighty patch to expose and configure low-level connection details in httpx", long_description=long_description, long_description_content_type="text/markdown", classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Natural Language :: English", @@ -23,9 +32,8 @@ python_requires='>=3.5', packages=['httpx_tls', 'httpx_tls.patch'], install_requires=['httpx', - 'tlslite-ng @ git+https://github.com/charxhit/tlslite-ng.git@httpx-integration', + 'tlslite-ng @ git+https://github.com/AnCry1596/tlslite-ng.git', 'trio', - 'user-agents', 'h2', 'anyio'] ) \ No newline at end of file