Skip to content

Commit 69c52fd

Browse files
VaghinakDevVaghinakDev
committed
Merge branch 'fix_upload' into friday
2 parents 1d901a9 + d5580d6 commit 69c52fd

File tree

11 files changed

+266
-7
lines changed

11 files changed

+266
-7
lines changed

‎CHANGELOG.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
# Changelog
22
All release highlights of this project will be documented in this file.
3+
## 4.4.2, 4.4.3 - August 21, 2022
4+
### Updated
5+
- the **schema** of `classes JSON` to support new values for the `"group_type"` key for a given attribute group. `"group_type": "radio" | "checklist" | "text" | "numeric"`.
6+
- the **schema** of `video annotation JSON` to support instances that have a `"tag"` type.
7+
### Fixed
8+
- `SAClient.get_annotations()` _method_ to address the issue of working with the large projects.
9+
- `SAClient.get_annotations_per_frame()` _method_ to address the issue of throwing an error on small videos when the fps is set to 1.
10+
- `SAClient.upload_annotations_from_folder_to_project()` to address the issue of timestamp values represented in seconds for the `"lastAction"`.
11+
- `SAClient.download_export()` _method_ to address the issue of empty logs.
12+
- `SAClient.clone_project()` _method_ to address the issue of having a corrupted project clone, when the source project has a keypoint workflow.
13+
###
314
## 4.4.1 - July 24, 2022
415
### Added
516
- `SAClient.create_custom_fields()` _method_ to create/add new custom fields to a project’s custom field schema.

‎src/superannotate/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import os
22
import sys
33

4-
__version__ = "4.4.2dev14"
4+
__version__ = "4.4.3"
55

66
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
77

‎src/superannotate/lib/core/usecases/annotations.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import os
77
import platform
88
import re
9-
import sys
109
import time
1110
from dataclasses import dataclass
1211
from datetime import datetime
@@ -81,6 +80,7 @@ class AnnotationToUpload:
8180
path: str
8281
data: io.StringIO = None
8382
mask: io.BytesIO = None
83+
size: int = None
8484

8585
def __init__(
8686
self,
@@ -348,20 +348,23 @@ async def upload(_chunk):
348348
self.reporter.log_debug(str(e))
349349
self._report.failed_annotations.extend([i.name for i in _chunk])
350350

351+
_size = 0
351352
while True:
352353
item = await self._small_files_queue.get()
353354
self._small_files_queue.task_done()
354355
if not item:
355356
self._small_files_queue.put_nowait(None)
356357
break
357-
chunk.append(item)
358358
if (
359-
sys.getsizeof(chunk) >= self.CHUNK_SIZE_MB
359+
_size + item.size >= self.CHUNK_SIZE_MB
360360
or sum([len(i.name) for i in chunk])
361-
>= self.URI_THRESHOLD - len(chunk) * 14
361+
>= self.URI_THRESHOLD - (len(chunk) + 1) * 14
362362
):
363363
await upload(chunk)
364364
chunk = []
365+
_size = 0
366+
chunk.append(item)
367+
_size += item.size
365368
if chunk:
366369
await upload(chunk)
367370

@@ -411,6 +414,7 @@ async def distribute_queues(self, items_to_upload: list):
411414
annotation_file.seek(0)
412415
t_item.data = annotation_file
413416
t_item.mask = mask
417+
t_item.size = size
414418
while True:
415419
if size > BIG_FILE_THRESHOLD:
416420
if self._big_files_queue.qsize() > 32:

‎src/superannotate/lib/infrastructure/services.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1369,6 +1369,7 @@ async def upload_annotations(
13691369
filename=key,
13701370
content_type="application/json",
13711371
)
1372+
13721373
_response = await session.post(
13731374
url,
13741375
params={

‎tests/data_set/sample_large_json_vector/aearth_mov_001.jpg.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎tests/data_set/sample_large_json_vector/aearth_mov_002.jpg.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎tests/data_set/sample_large_json_vector/aearth_mov_003.jpg.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎tests/data_set/sample_large_json_vector/aearth_mov_004.jpg.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎tests/data_set/sample_large_json_vector/aearth_mov_005.jpg.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎tests/data_set/sample_large_json_vector/classes/classes.json‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
[
2+
{
3+
"id": 1178455,
4+
"project_id": 1290,
5+
"name": "sdfg",
6+
"color": "#880901",
7+
"type": "tag",
8+
"createdAt": "2022-02-16T12:49:36.000Z",
9+
"updatedAt": "2022-07-14T06:32:21.000Z",
10+
"attribute_groups": [
11+
{
12+
"group_type": "radio",
13+
"id": 1815783,
14+
"class_id": 1178455,
15+
"name": "0",
16+
"createdAt": "2022-07-14T06:32:21.000Z",
17+
"updatedAt": "2022-07-14T06:32:21.000Z",
18+
"attributes": [
19+
{
20+
"id": 4143110,
21+
"group_id": 1815783,
22+
"project_id": 1290,
23+
"name": "s",
24+
"count": 0,
25+
"createdAt": "2022-07-14T06:32:21.000Z",
26+
"updatedAt": "2022-07-14T06:32:21.000Z"
27+
}
28+
],
29+
"default_value": null
30+
},
31+
{
32+
"group_type": "radio",
33+
"id": 1815784,
34+
"class_id": 1178455,
35+
"name": "1",
36+
"createdAt": "2022-07-14T06:32:21.000Z",
37+
"updatedAt": "2022-07-14T06:32:21.000Z",
38+
"attributes": [
39+
{
40+
"id": 4143111,
41+
"group_id": 1815784,
42+
"project_id": 1290,
43+
"name": "d",
44+
"count": 0,
45+
"createdAt": "2022-07-14T06:32:21.000Z",
46+
"updatedAt": "2022-07-14T06:32:21.000Z"
47+
}
48+
],
49+
"default_value": null
50+
},
51+
{
52+
"group_type": "radio",
53+
"id": 1815785,
54+
"class_id": 1178455,
55+
"name": "2",
56+
"createdAt": "2022-07-14T06:32:21.000Z",
57+
"updatedAt": "2022-07-14T06:32:21.000Z",
58+
"attributes": [
59+
{
60+
"id": 4143112,
61+
"group_id": 1815785,
62+
"project_id": 1290,
63+
"name": "d",
64+
"count": 0,
65+
"createdAt": "2022-07-14T06:32:21.000Z",
66+
"updatedAt": "2022-07-14T06:32:21.000Z"
67+
}
68+
],
69+
"default_value": null
70+
}
71+
]
72+
},
73+
{
74+
"id": 755215,
75+
"project_id": 1290,
76+
"name": "Plhayer",
77+
"color": "#0022c3",
78+
"type": "object",
79+
"createdAt": "2021-07-29T08:41:59.000Z",
80+
"updatedAt": "2021-07-29T08:42:38.000Z",
81+
"attribute_groups": []
82+
},
83+
{
84+
"id": 7399,
85+
"project_id": 1290,
86+
"name": "Shirt Number",
87+
"color": "#792568",
88+
"type": "object",
89+
"createdAt": "2020-06-03T07:33:51.000Z",
90+
"updatedAt": "2020-06-03T07:33:51.000Z",
91+
"attribute_groups": [
92+
{
93+
"group_type": "checklist",
94+
"id": 5791,
95+
"class_id": 7399,
96+
"name": "Type",
97+
"createdAt": "2020-06-03T07:33:51.000Z",
98+
"updatedAt": "2020-06-03T07:33:51.000Z",
99+
"attributes": [
100+
{
101+
"id": 13668,
102+
"group_id": 5791,
103+
"project_id": 1290,
104+
"name": "Illegible",
105+
"count": 0,
106+
"createdAt": "2020-06-03T07:33:51.000Z",
107+
"updatedAt": "2020-06-03T07:33:51.000Z"
108+
},
109+
{
110+
"id": 13667,
111+
"group_id": 5791,
112+
"project_id": 1290,
113+
"name": "Occluded",
114+
"count": 0,
115+
"createdAt": "2020-06-03T07:33:51.000Z",
116+
"updatedAt": "2020-06-03T07:33:51.000Z"
117+
}
118+
],
119+
"default_value": []
120+
}
121+
]
122+
},
123+
{
124+
"id": 7398,
125+
"project_id": 1290,
126+
"name": "Referee",
127+
"color": "#26eded",
128+
"type": "object",
129+
"createdAt": "2020-06-03T07:33:51.000Z",
130+
"updatedAt": "2020-10-01T11:06:57.000Z",
131+
"attribute_groups": [
132+
{
133+
"group_type": "radio",
134+
"id": 5790,
135+
"class_id": 7398,
136+
"name": "Type",
137+
"createdAt": "2020-06-03T07:33:51.000Z",
138+
"updatedAt": "2020-06-03T07:33:51.000Z",
139+
"attributes": [
140+
{
141+
"id": 13666,
142+
"group_id": 5790,
143+
"project_id": 1290,
144+
"name": "Occluded",
145+
"count": 0,
146+
"createdAt": "2020-06-03T07:33:51.000Z",
147+
"updatedAt": "2020-06-03T07:33:51.000Z"
148+
}
149+
],
150+
"default_value": null
151+
}
152+
]
153+
},
154+
{
155+
"id": 7397,
156+
"project_id": 1290,
157+
"name": "Goalkeeper",
158+
"color": "#d002f6",
159+
"type": "object",
160+
"createdAt": "2020-06-03T07:33:51.000Z",
161+
"updatedAt": "2020-11-09T13:01:12.000Z",
162+
"attribute_groups": [
163+
{
164+
"group_type": "radio",
165+
"id": 5789,
166+
"class_id": 7397,
167+
"name": "Type",
168+
"createdAt": "2020-06-03T07:33:51.000Z",
169+
"updatedAt": "2020-06-03T07:33:51.000Z",
170+
"attributes": [
171+
{
172+
"id": 13665,
173+
"group_id": 5789,
174+
"project_id": 1290,
175+
"name": "Occluded",
176+
"count": 0,
177+
"createdAt": "2020-06-03T07:33:51.000Z",
178+
"updatedAt": "2020-06-03T07:33:51.000Z"
179+
}
180+
],
181+
"default_value": null
182+
}
183+
]
184+
},
185+
{
186+
"id": 7396,
187+
"project_id": 1290,
188+
"name": "Player",
189+
"color": "#f90e35",
190+
"type": "object",
191+
"createdAt": "2020-06-03T07:33:51.000Z",
192+
"updatedAt": "2020-11-09T13:01:12.000Z",
193+
"attribute_groups": [
194+
{
195+
"group_type": "radio",
196+
"id": 5788,
197+
"class_id": 7396,
198+
"name": "Type",
199+
"createdAt": "2020-06-03T07:33:51.000Z",
200+
"updatedAt": "2020-06-03T07:33:51.000Z",
201+
"attributes": [
202+
{
203+
"id": 13664,
204+
"group_id": 5788,
205+
"project_id": 1290,
206+
"name": "Occluded",
207+
"count": 0,
208+
"createdAt": "2020-06-03T07:33:51.000Z",
209+
"updatedAt": "2020-11-09T13:01:12.000Z"
210+
}
211+
],
212+
"default_value": null
213+
}
214+
]
215+
}
216+
]

0 commit comments

Comments
 (0)