From bda571336389b0194e5a561af1ad504cb7edd8e4 Mon Sep 17 00:00:00 2001 From: wj-Mcat <1436130236@qq.com> Date: Wed, 1 Sep 2021 22:28:18 +0800 Subject: [PATCH 1/5] complete file box chunk --- .../file_box_chunk_converter.py | 94 +++++++++++++++++++ src/wechaty_puppet_service/puppet.py | 8 +- 2 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 src/wechaty_puppet_service/file_box_chunk/file_box_chunk_converter.py diff --git a/src/wechaty_puppet_service/file_box_chunk/file_box_chunk_converter.py b/src/wechaty_puppet_service/file_box_chunk/file_box_chunk_converter.py new file mode 100644 index 0000000..f43b5fa --- /dev/null +++ b/src/wechaty_puppet_service/file_box_chunk/file_box_chunk_converter.py @@ -0,0 +1,94 @@ +""" +Python Wechaty - https://github.com/wechaty/python-wechaty + +Authors: Jingjing WU (吴京京) + +2020-now @ Copyright Wechaty + +Licensed under the Apache License, Version 2.0 (the 'License'); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an 'AS IS' BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import annotations +import os +import math +from typing import AsyncIterable, List, Optional +import tempfile +from uuid import uuid4 +from wechaty_grpc.wechaty.puppet import MessageSendFileStreamRequest +from wechaty_puppet import FileBox +from wechaty_grpc.wechaty.puppet import FileBoxChunk + +from wechaty_puppet_service.config import CHUNK_SIZE + + +async def pack_file_box_to_chunk(file_box: FileBox) -> AsyncIterable[FileBoxChunk]: + """pack file-box instance to async iterator chunks + + Args: + file_box (FileBox): the source file-box instance + + Returns: + AsyncIterator[FileBoxChunk]: Async Iterator FileBox Chunk + """ + # 1. save file-box as temp file, so that we can read stream from it. + temp_dir = tempfile.TemporaryDirectory() + file_name = "%s-%s" % (str(uuid4()), file_box.name) + file_path = os.path.join(temp_dir.name, file_name) + await file_box.to_file(file_path, overwrite=True) + + # 2. read stream from the temp file + streams: bytes = bytes() + with open(file_path, 'rb') as f: + for line in f: + streams += line + + # 3. split it to chunks + size = len(streams) + + for i in range(math.ceil(size / CHUNK_SIZE)): + stream = streams[i * CHUNK_SIZE: (i + 1) * CHUNK_SIZE] + yield FileBoxChunk( + data=stream, + name=file_box.name + ) + + +async def gen_file_stream_request( + conversation_id: str, file_box: FileBox) -> AsyncIterable[MessageSendFileStreamRequest]: + chunks = pack_file_box_to_chunk(file_box) + async for chunk in chunks: + request = MessageSendFileStreamRequest( + conversation_id=conversation_id, + file_box_chunk=chunk + ) + yield request + + +async def unpack_file_box_chunk(file_box_chunks: List[FileBoxChunk]) -> FileBox: + """unpack async iterator filebox chunks to file-box instance + + Args: + file_box_chunks (AsyncIterator[FileBox]): + + Returns: + FileBox: [description] + """ + # 1. merge chunks + buffers: bytes = bytes() + name: Optional[str] = None + for chunk in file_box_chunks: + buffers += chunk.data + name = chunk.name + + # 2. gen file-box + file_box: FileBox = FileBox.from_buffer(buffers, name) + return file_box diff --git a/src/wechaty_puppet_service/puppet.py b/src/wechaty_puppet_service/puppet.py index 0c6b498..adfd4d8 100644 --- a/src/wechaty_puppet_service/puppet.py +++ b/src/wechaty_puppet_service/puppet.py @@ -85,6 +85,7 @@ extract_host_and_port, test_endpoint ) +from wechaty_puppet_service.file_box_chunk.file_box_chunk_converter import gen_file_stream_request log = get_logger('PuppetService') @@ -359,9 +360,10 @@ async def message_send_file(self, conversation_id: str, :param file: :return: """ - response = await self.puppet_stub.message_send_file( - conversation_id=conversation_id, - filebox=file.to_json_str() + + request_stream = gen_file_stream_request(conversation_id, file) + response = await self.puppet_stub.message_send_file_stream( + request_iterator=request_stream ) return response.id From e52df78d994257f9c9c4f8e2a12c98a713f8f34a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AA=91=E9=A9=AC=E5=B0=8F=E7=8C=AB?= <1435130236@qq.com> Date: Tue, 17 May 2022 21:54:25 +0800 Subject: [PATCH 2/5] enable tls --- VERSION | 2 +- src/wechaty_puppet_service/config.py | 35 ++++++++++++++++++++++++++++ src/wechaty_puppet_service/puppet.py | 7 ++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 55485e1..3eefcb9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8.9 +1.0.0 diff --git a/src/wechaty_puppet_service/config.py b/src/wechaty_puppet_service/config.py index d6d64f5..14d27c6 100644 --- a/src/wechaty_puppet_service/config.py +++ b/src/wechaty_puppet_service/config.py @@ -28,6 +28,41 @@ CHUNK_SIZE = 1024 * 1024 +TLS_CA_CERT = """-----BEGIN CERTIFICATE----- +MIIFxTCCA62gAwIBAgIUYddLAoa8JnLzJ80l2u5vGuFsaEIwDQYJKoZIhvcNAQEL +BQAwcjELMAkGA1UEBhMCVVMxFjAUBgNVBAgMDVNhbiBGcmFuY2lzY28xEjAQBgNV +BAcMCVBhbG8gQWx0bzEQMA4GA1UECgwHV2VjaGF0eTELMAkGA1UECwwCQ0ExGDAW +BgNVBAMMD3dlY2hhdHktcm9vdC1jYTAeFw0yMTA4MDkxNTQ4NTJaFw0zMTA4MDcx +NTQ4NTJaMHIxCzAJBgNVBAYTAlVTMRYwFAYDVQQIDA1TYW4gRnJhbmNpc2NvMRIw +EAYDVQQHDAlQYWxvIEFsdG8xEDAOBgNVBAoMB1dlY2hhdHkxCzAJBgNVBAsMAkNB +MRgwFgYDVQQDDA93ZWNoYXR5LXJvb3QtY2EwggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQDulLjOZhzQ58TSQ7TfWNYgdtWhlc+5L9MnKb1nznVRhzAkZo3Q +rPLRW/HDjlv2OEbt4nFLaQgaMmc1oJTUVGDBDlrzesI/lJh7z4eA/B0z8eW7f6Cw +/TGc8lgzHvq7UIE507QYPhvfSejfW4Prw+90HJnuodriPdMGS0n9AR37JPdQm6sD +iMFeEvhHmM2SXRo/o7bll8UDZi81DoFu0XuTCx0esfCX1W5QWEmAJ5oAdjWxJ23C +lxI1+EjwBQKXGqp147VP9+pwpYW5Xxpy870kctPBHKjCAti8Bfo+Y6dyWz2UAd4w +4BFRD+18C/TgX+ECl1s9fsHMY15JitcSGgAIz8gQX1OelECaTMRTQfNaSnNW4LdS +sXMQEI9WxAU/W47GCQFmwcJeZvimqDF1QtflHSaARD3O8tlbduYqTR81LJ63bPoy +9e1pdB6w2bVOTlHunE0YaGSJERALVc1xz40QpPGcZ52mNCb3PBg462RQc77yv/QB +x/P2RC1y0zDUF2tP9J29gTatWq6+D4MhfEk2flZNyzAgJbDuT6KAIJGzOB1ZJ/MG +o1gS13eTuZYw24LElrhd1PrR6OHK+lkyYzqUPYMulUg4HzaZIDclfHKwAC4lecKm +zC5q9jJB4m4SKMKdzxvpIOfdahoqsZMg34l4AavWRqPTpwEU0C0dboNA/QIDAQAB +o1MwUTAdBgNVHQ4EFgQU0rey3QPklTOgdhMJ9VIA6KbZ5bAwHwYDVR0jBBgwFoAU +0rey3QPklTOgdhMJ9VIA6KbZ5bAwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAgEAx2uyShx9kLoB1AJ8x7Vf95v6PX95L/4JkJ1WwzJ9Dlf3BcCI7VH7 +Fp1dnQ6Ig7mFqSBDBAUUBWAptAnuqIDcgehI6XAEKxW8ZZRxD877pUNwZ/45tSC4 +b5U5y9uaiNK7oC3LlDCsB0291b3KSOtevMeDFoh12LcliXAkdIGGTccUxrH+Cyij +cBOc+EKGJFBdLqcjLDU4M6QdMMMFOdfXyAOSpYuWGYqrxqvxQjAjvianEyMpNZWM +lajggJqiPhfF67sZTB2yzvRTmtHdUq7x+iNOVonOBcCHu31aGxa9Py91XEr9jaIQ +EBdl6sycLxKo8mxF/5tyUOns9+919aWNqTOUBmI15D68bqhhOVNyvsb7aVURIt5y +6A7Sj4gSBR9P22Ba6iFZgbvfLn0zKLzjlBonUGlSPf3rSIYUkawICtDyYPvK5mi3 +mANgIChMiOw6LYCPmmUVVAWU/tDy36kr9ZV9YTIZRYAkWswsJB340whjuzvZUVaG +DgW45GPR6bGIwlFZeqCwXLput8Z3C8Sw9bE9vjlB2ZCpjPLmWV/WbDlH3J3uDjgt +9PoALW0sOPhHfYklH4/rrmsSWMYTUuGS/HqxrEER1vpIOOb0hIiAWENDT/mruq22 +VqO8MHX9ebjInSxPmhYOlrSZrOgEcogyMB4Z0SOtKVqPnkWmdR5hatU= +-----END CERTIFICATE-----""" + + def get_token() -> Optional[str]: """ get the token from environment variable diff --git a/src/wechaty_puppet_service/puppet.py b/src/wechaty_puppet_service/puppet.py index c9a7ce6..a60f099 100644 --- a/src/wechaty_puppet_service/puppet.py +++ b/src/wechaty_puppet_service/puppet.py @@ -21,6 +21,7 @@ from __future__ import annotations import json +import ssl from typing import Callable, Optional, List from functools import reduce from dataclasses import asdict @@ -79,6 +80,7 @@ from wechaty_puppet_service.config import ( get_endpoint, get_token, + TLS_CA_CERT ) from wechaty_puppet_service.utils import ( extract_host_and_port, @@ -373,6 +375,7 @@ async def message_send_file(self, conversation_id: str, :param file: :return: """ + self.puppet_stub.message_send_contact() response = await self.puppet_stub.message_send_file( conversation_id=conversation_id, filebox=file.to_json_str() @@ -907,6 +910,10 @@ def _init_puppet(self) -> None: ) host, port = extract_host_and_port(self.options.end_point) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(cadata=TLS_CA_CERT) + self.channel = Channel(host=host, port=port) # pylint: disable=W0212 From b01d11d8eecad8d3c031afab7ac2b96a51031740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AA=91=E9=A9=AC=E5=B0=8F=E7=8C=AB?= <1435130236@qq.com> Date: Tue, 17 May 2022 21:57:58 +0800 Subject: [PATCH 3/5] update version in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 191476e..a504a78 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,8 @@ def versioning(version: str) -> str: minor = sem_ver['minor'] patch = str(sem_ver['patch']) - if minor % 2: - patch = 'dev' + patch + # if minor % 2: + # patch = 'dev' + patch fin_ver = '%d.%d.%s' % ( major, From eb787647694eeb0831600c9680e299200ac06fd4 Mon Sep 17 00:00:00 2001 From: wj-Mcat <1436130236@qq.com> Date: Tue, 17 May 2022 22:09:20 +0800 Subject: [PATCH 4/5] fix linting issue --- requirements-dev.txt | 3 +- setup.py | 4 +- .../file_box_chunk_converter.py | 94 ------------------- src/wechaty_puppet_service/puppet.py | 8 +- 4 files changed, 7 insertions(+), 102 deletions(-) delete mode 100644 src/wechaty_puppet_service/file_box_chunk/file_box_chunk_converter.py diff --git a/requirements-dev.txt b/requirements-dev.txt index 565e903..15b81ec 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,8 +5,9 @@ pycodestyle pylint pylint-quotes pytest +pytest-asyncio pytype semver grpclib wechaty-puppet~=0.3dev2 -pre-commit \ No newline at end of file +pre-commit \ No newline at end of file diff --git a/setup.py b/setup.py index 191476e..a504a78 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,8 @@ def versioning(version: str) -> str: minor = sem_ver['minor'] patch = str(sem_ver['patch']) - if minor % 2: - patch = 'dev' + patch + # if minor % 2: + # patch = 'dev' + patch fin_ver = '%d.%d.%s' % ( major, diff --git a/src/wechaty_puppet_service/file_box_chunk/file_box_chunk_converter.py b/src/wechaty_puppet_service/file_box_chunk/file_box_chunk_converter.py deleted file mode 100644 index f43b5fa..0000000 --- a/src/wechaty_puppet_service/file_box_chunk/file_box_chunk_converter.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -Python Wechaty - https://github.com/wechaty/python-wechaty - -Authors: Jingjing WU (吴京京) - -2020-now @ Copyright Wechaty - -Licensed under the Apache License, Version 2.0 (the 'License'); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an 'AS IS' BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from __future__ import annotations -import os -import math -from typing import AsyncIterable, List, Optional -import tempfile -from uuid import uuid4 -from wechaty_grpc.wechaty.puppet import MessageSendFileStreamRequest -from wechaty_puppet import FileBox -from wechaty_grpc.wechaty.puppet import FileBoxChunk - -from wechaty_puppet_service.config import CHUNK_SIZE - - -async def pack_file_box_to_chunk(file_box: FileBox) -> AsyncIterable[FileBoxChunk]: - """pack file-box instance to async iterator chunks - - Args: - file_box (FileBox): the source file-box instance - - Returns: - AsyncIterator[FileBoxChunk]: Async Iterator FileBox Chunk - """ - # 1. save file-box as temp file, so that we can read stream from it. - temp_dir = tempfile.TemporaryDirectory() - file_name = "%s-%s" % (str(uuid4()), file_box.name) - file_path = os.path.join(temp_dir.name, file_name) - await file_box.to_file(file_path, overwrite=True) - - # 2. read stream from the temp file - streams: bytes = bytes() - with open(file_path, 'rb') as f: - for line in f: - streams += line - - # 3. split it to chunks - size = len(streams) - - for i in range(math.ceil(size / CHUNK_SIZE)): - stream = streams[i * CHUNK_SIZE: (i + 1) * CHUNK_SIZE] - yield FileBoxChunk( - data=stream, - name=file_box.name - ) - - -async def gen_file_stream_request( - conversation_id: str, file_box: FileBox) -> AsyncIterable[MessageSendFileStreamRequest]: - chunks = pack_file_box_to_chunk(file_box) - async for chunk in chunks: - request = MessageSendFileStreamRequest( - conversation_id=conversation_id, - file_box_chunk=chunk - ) - yield request - - -async def unpack_file_box_chunk(file_box_chunks: List[FileBoxChunk]) -> FileBox: - """unpack async iterator filebox chunks to file-box instance - - Args: - file_box_chunks (AsyncIterator[FileBox]): - - Returns: - FileBox: [description] - """ - # 1. merge chunks - buffers: bytes = bytes() - name: Optional[str] = None - for chunk in file_box_chunks: - buffers += chunk.data - name = chunk.name - - # 2. gen file-box - file_box: FileBox = FileBox.from_buffer(buffers, name) - return file_box diff --git a/src/wechaty_puppet_service/puppet.py b/src/wechaty_puppet_service/puppet.py index f2e14f3..c9a7ce6 100644 --- a/src/wechaty_puppet_service/puppet.py +++ b/src/wechaty_puppet_service/puppet.py @@ -85,7 +85,6 @@ ping_endpoint, message_emoticon ) -from wechaty_puppet_service.file_box_chunk.file_box_chunk_converter import gen_file_stream_request log = get_logger('PuppetService') @@ -374,10 +373,9 @@ async def message_send_file(self, conversation_id: str, :param file: :return: """ - - request_stream = gen_file_stream_request(conversation_id, file) - response = await self.puppet_stub.message_send_file_stream( - request_iterator=request_stream + response = await self.puppet_stub.message_send_file( + conversation_id=conversation_id, + filebox=file.to_json_str() ) return response.id From ffdfcfe236a021f7fc08922565720a83a3ce3564 Mon Sep 17 00:00:00 2001 From: wj-Mcat <1436130236@qq.com> Date: Tue, 17 May 2022 22:11:18 +0800 Subject: [PATCH 5/5] convert triple quote --- src/wechaty_puppet_service/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wechaty_puppet_service/config.py b/src/wechaty_puppet_service/config.py index 14d27c6..3a088be 100644 --- a/src/wechaty_puppet_service/config.py +++ b/src/wechaty_puppet_service/config.py @@ -28,7 +28,7 @@ CHUNK_SIZE = 1024 * 1024 -TLS_CA_CERT = """-----BEGIN CERTIFICATE----- +TLS_CA_CERT = '''-----BEGIN CERTIFICATE----- MIIFxTCCA62gAwIBAgIUYddLAoa8JnLzJ80l2u5vGuFsaEIwDQYJKoZIhvcNAQEL BQAwcjELMAkGA1UEBhMCVVMxFjAUBgNVBAgMDVNhbiBGcmFuY2lzY28xEjAQBgNV BAcMCVBhbG8gQWx0bzEQMA4GA1UECgwHV2VjaGF0eTELMAkGA1UECwwCQ0ExGDAW @@ -60,7 +60,7 @@ DgW45GPR6bGIwlFZeqCwXLput8Z3C8Sw9bE9vjlB2ZCpjPLmWV/WbDlH3J3uDjgt 9PoALW0sOPhHfYklH4/rrmsSWMYTUuGS/HqxrEER1vpIOOb0hIiAWENDT/mruq22 VqO8MHX9ebjInSxPmhYOlrSZrOgEcogyMB4Z0SOtKVqPnkWmdR5hatU= ------END CERTIFICATE-----""" +-----END CERTIFICATE-----''' def get_token() -> Optional[str]: