diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index d8d1277..850b3b7 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -67,6 +67,11 @@ def load_schema(entity): 'schema': load_schema('groups'), 'key_properties': ['id'], }, + 'merge_requests': { + 'url': '/projects/{}/merge_requests', + 'schema': load_schema('merge_requests'), + 'key_properties': ['id'], + }, } @@ -130,7 +135,7 @@ def gen_request(url): def format_timestamp(data, typ, schema): result = data - if typ == 'string' and schema.get('format') == 'date-time': + if typ == 'string' and schema.get('format') == 'date-time' and data is not None: rfc3339_ts = rfc3339_to_timestamp(data) utc_dt = datetime.datetime.utcfromtimestamp(rfc3339_ts).replace(tzinfo=pytz.UTC) result = utils.strftime(utc_dt) @@ -219,6 +224,25 @@ def sync_group(gid, pids): singer.write_record("groups", group, time_extracted=time_extracted) +def sync_merge_request(project): + url = get_url('merge_requests', project['id']) + with Transformer(pre_hook=format_timestamp) as transformer: + for row in gen_request(url): + flatten_id(row, 'merged_by') + flatten_id(row, 'closed_by') + flatten_id(row, 'merge_user') + flatten_id(row, 'assignee') + flatten_id(row, 'author') + transformed_row = transformer.transform( + row, RESOURCES['merge_requests']['schema'] + ) + + if row['updated_at'] >= get_start('project_{}'.format(project['id'])): + singer.write_record( + 'merge_requests', transformed_row, time_extracted=utils.now() + ) + + def sync_project(pid): url = get_url("projects", pid) data = request(url).json() @@ -246,6 +270,7 @@ def sync_project(pid): sync_issues(project) sync_milestones(project) sync_users(project) + sync_merge_request(project) singer.write_record("projects", project, time_extracted=time_extracted) utils.update_state(STATE, state_key, last_activity_at) diff --git a/tap_gitlab/schemas/merge_requests.json b/tap_gitlab/schemas/merge_requests.json new file mode 100644 index 0000000..a18d83d --- /dev/null +++ b/tap_gitlab/schemas/merge_requests.json @@ -0,0 +1,148 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "iid": { + "type": "integer" + }, + "project_id": { + "type": "integer" + }, + "title": { + "type": "string" + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": "string" + }, + "state": { + "type": "string" + }, + "created_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "merged_by_id": { + "type": [ + "null", + "integer" + ] + }, + "merge_user_id": { + "type": [ + "null", + "integer" + ] + }, + "merged_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "closed_by_id": { + "type": [ + "null", + "integer" + ] + }, + "closed_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "author_id": { + "type": [ + "null", + "integer" + ] + }, + "assignee_id": { + "type": [ + "null", + "integer" + ] + }, + "target_branch": { + "type": "string" + }, + "source_branch": { + "type": "string" + }, + "user_notes_count": { + "type": [ + "null", + "integer" + ] + }, + "upvotes": { + "type": [ + "null", + "integer" + ] + }, + "downvotes": { + "type": [ + "null", + "integer" + ] + }, + "merge_status": { + "type": "string" + }, + "has_conflicts": { + "type": [ + "null", + "boolean" + ] + }, + "squash": { + "type": [ + "null", + "boolean" + ] + }, + "merge_when_pipeline_succeeds": { + "type": [ + "null", + "boolean" + ] + } + } +} \ No newline at end of file